xref: /petsc/src/dm/impls/plex/plexcreate.c (revision 5f0c5ebc6c97b879c1d68dc86014b21841a44daa)
1 #define PETSCDM_DLL
2 #include <petsc/private/dmpleximpl.h> /*I   "petscdmplex.h"   I*/
3 #include <petsc/private/hashseti.h>
4 #include <petscsf.h>
5 #include <petscdmplextransform.h> /*I   "petscdmplextransform.h"   I*/
6 #include <petscdmlabelephemeral.h>
7 #include <petsc/private/kernels/blockmatmult.h>
8 #include <petsc/private/kernels/blockinvert.h>
9 
10 #ifdef PETSC_HAVE_UNISTD_H
11   #include <unistd.h>
12 #endif
13 #include <errno.h>
14 
15 PetscLogEvent DMPLEX_CreateFromFile, DMPLEX_CreateFromOptions, DMPLEX_BuildFromCellList, DMPLEX_BuildCoordinatesFromCellList;
16 
17 /* External function declarations here */
18 static PetscErrorCode DMInitialize_Plex(DM dm);
19 
20 PETSC_EXTERN PetscErrorCode DMPlexCheckEGADS_Private(DM dm)
21 {
22   PetscObject modelObj;
23 
24   PetscFunctionBegin;
25   PetscCall(PetscObjectQuery((PetscObject)dm, "EGADS Model", &modelObj));
26   PetscCheck(modelObj, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Input DM must have attached EGADS Geometry Model");
27   PetscFunctionReturn(PETSC_SUCCESS);
28 }
29 
30 static PetscErrorCode DMPlexCopyContext_Private(DM dmin, const char name[], DM dmout)
31 {
32   PetscObject obj;
33 
34   PetscFunctionBegin;
35   PetscCall(PetscObjectQuery((PetscObject)dmin, name, &obj));
36   if (obj) PetscCall(PetscObjectCompose((PetscObject)dmout, name, obj));
37   PetscFunctionReturn(PETSC_SUCCESS);
38 }
39 
40 static PetscErrorCode DMPlexSwapContext_Private(DM dmA, const char name[], DM dmB)
41 {
42   PetscObject objA, objB;
43 
44   PetscFunctionBegin;
45   PetscCall(PetscObjectQuery((PetscObject)dmA, name, &objA));
46   PetscCall(PetscObjectQuery((PetscObject)dmB, name, &objB));
47   PetscCall(PetscObjectReference(objA));
48   PetscCall(PetscObjectReference(objB));
49   PetscCall(PetscObjectCompose((PetscObject)dmA, name, objB));
50   PetscCall(PetscObjectCompose((PetscObject)dmB, name, objA));
51   PetscCall(PetscObjectDereference(objA));
52   PetscCall(PetscObjectDereference(objB));
53   PetscFunctionReturn(PETSC_SUCCESS);
54 }
55 
56 PetscErrorCode DMPlexCopyEGADSInfo_Internal(DM dmin, DM dmout)
57 {
58   PetscFunctionBegin;
59   PetscCall(DMPlexCopyContext_Private(dmin, "EGADS Model", dmout));
60   PetscCall(DMPlexCopyContext_Private(dmin, "EGADS Context", dmout));
61   PetscCall(DMPlexCopyContext_Private(dmin, "EGADSlite Model", dmout));
62   PetscCall(DMPlexCopyContext_Private(dmin, "EGADSlite Context", dmout));
63   PetscFunctionReturn(PETSC_SUCCESS);
64 }
65 
66 static PetscErrorCode DMPlexSwapEGADSInfo_Private(DM dmA, DM dmB)
67 {
68   PetscFunctionBegin;
69   PetscCall(DMPlexSwapContext_Private(dmA, "EGADS Model", dmB));
70   PetscCall(DMPlexSwapContext_Private(dmA, "EGADS Context", dmB));
71   PetscCall(DMPlexSwapContext_Private(dmA, "EGADSlite Model", dmB));
72   PetscCall(DMPlexSwapContext_Private(dmA, "EGADSlite Context", dmB));
73   PetscFunctionReturn(PETSC_SUCCESS);
74 }
75 
76 /* This copies internal things in the Plex structure that we generally want when making a new, related Plex */
77 PetscErrorCode DMPlexCopy_Internal(DM dmin, PetscBool copyPeriodicity, PetscBool copyOverlap, DM dmout)
78 {
79   const PetscReal     *maxCell, *Lstart, *L;
80   VecType              vecType;
81   MatType              matType;
82   PetscBool            dist, useCeed, balance_partition;
83   DMReorderDefaultFlag reorder;
84 
85   PetscFunctionBegin;
86   if (dmin == dmout) PetscFunctionReturn(PETSC_SUCCESS);
87   PetscCall(DMGetVecType(dmin, &vecType));
88   PetscCall(DMSetVecType(dmout, vecType));
89   PetscCall(DMGetMatType(dmin, &matType));
90   PetscCall(DMSetMatType(dmout, matType));
91   if (copyPeriodicity) {
92     PetscCall(DMGetPeriodicity(dmin, &maxCell, &Lstart, &L));
93     PetscCall(DMSetPeriodicity(dmout, maxCell, Lstart, L));
94     PetscCall(DMLocalizeCoordinates(dmout));
95   }
96   PetscCall(DMPlexDistributeGetDefault(dmin, &dist));
97   PetscCall(DMPlexDistributeSetDefault(dmout, dist));
98   PetscCall(DMPlexReorderGetDefault(dmin, &reorder));
99   PetscCall(DMPlexReorderSetDefault(dmout, reorder));
100   PetscCall(DMPlexGetUseCeed(dmin, &useCeed));
101   PetscCall(DMPlexSetUseCeed(dmout, useCeed));
102   PetscCall(DMPlexGetPartitionBalance(dmin, &balance_partition));
103   PetscCall(DMPlexSetPartitionBalance(dmout, balance_partition));
104   ((DM_Plex *)dmout->data)->useHashLocation = ((DM_Plex *)dmin->data)->useHashLocation;
105   ((DM_Plex *)dmout->data)->printSetValues  = ((DM_Plex *)dmin->data)->printSetValues;
106   ((DM_Plex *)dmout->data)->printFEM        = ((DM_Plex *)dmin->data)->printFEM;
107   ((DM_Plex *)dmout->data)->printFVM        = ((DM_Plex *)dmin->data)->printFVM;
108   ((DM_Plex *)dmout->data)->printL2         = ((DM_Plex *)dmin->data)->printL2;
109   ((DM_Plex *)dmout->data)->printLocate     = ((DM_Plex *)dmin->data)->printLocate;
110   ((DM_Plex *)dmout->data)->printProject    = ((DM_Plex *)dmin->data)->printProject;
111   ((DM_Plex *)dmout->data)->printTol        = ((DM_Plex *)dmin->data)->printTol;
112   if (copyOverlap) PetscCall(DMPlexSetOverlap_Plex(dmout, dmin, 0));
113   PetscCall(DMPlexCopyEGADSInfo_Internal(dmin, dmout));
114   PetscFunctionReturn(PETSC_SUCCESS);
115 }
116 
117 /* Replace dm with the contents of ndm, and then destroy ndm
118    - Share the DM_Plex structure
119    - Share the coordinates
120    - Share the SF
121 */
122 PetscErrorCode DMPlexReplace_Internal(DM dm, DM *ndm)
123 {
124   PetscSF          sf;
125   DM               dmNew = *ndm, coordDM, coarseDM;
126   Vec              coords;
127   const PetscReal *maxCell, *Lstart, *L;
128   PetscInt         dim, cdim;
129   PetscBool        use_natural;
130 
131   PetscFunctionBegin;
132   if (dm == dmNew) {
133     PetscCall(DMDestroy(ndm));
134     PetscFunctionReturn(PETSC_SUCCESS);
135   }
136   dm->setupcalled = dmNew->setupcalled;
137   if (!dm->hdr.name) {
138     const char *name;
139 
140     PetscCall(PetscObjectGetName((PetscObject)*ndm, &name));
141     PetscCall(PetscObjectSetName((PetscObject)dm, name));
142   }
143   PetscCall(DMGetDimension(dmNew, &dim));
144   PetscCall(DMSetDimension(dm, dim));
145   PetscCall(DMGetCoordinateDim(dmNew, &cdim));
146   PetscCall(DMSetCoordinateDim(dm, cdim));
147   PetscCall(DMGetPointSF(dmNew, &sf));
148   PetscCall(DMSetPointSF(dm, sf));
149   PetscCall(DMGetCoordinateDM(dmNew, &coordDM));
150   PetscCall(DMGetCoordinatesLocal(dmNew, &coords));
151   PetscCall(DMSetCoordinateDM(dm, coordDM));
152   PetscCall(DMSetCoordinatesLocal(dm, coords));
153   PetscCall(DMGetCellCoordinateDM(dmNew, &coordDM));
154   PetscCall(DMGetCellCoordinatesLocal(dmNew, &coords));
155   PetscCall(DMSetCellCoordinateDM(dm, coordDM));
156   PetscCall(DMSetCellCoordinatesLocal(dm, coords));
157   /* Do not want to create the coordinate field if it does not already exist, so do not call DMGetCoordinateField() */
158   PetscCall(DMFieldDestroy(&dm->coordinates[0].field));
159   dm->coordinates[0].field            = dmNew->coordinates[0].field;
160   ((DM_Plex *)dmNew->data)->coordFunc = ((DM_Plex *)dm->data)->coordFunc;
161   PetscCall(DMGetPeriodicity(dmNew, &maxCell, &Lstart, &L));
162   PetscCall(DMSetPeriodicity(dm, maxCell, Lstart, L));
163   PetscCall(DMGetNaturalSF(dmNew, &sf));
164   PetscCall(DMSetNaturalSF(dm, sf));
165   PetscCall(DMGetUseNatural(dmNew, &use_natural));
166   PetscCall(DMSetUseNatural(dm, use_natural));
167   PetscCall(DMDestroy_Plex(dm));
168   PetscCall(DMInitialize_Plex(dm));
169   dm->data = dmNew->data;
170   ((DM_Plex *)dmNew->data)->refct++;
171   {
172     PetscInt       num_face_sfs;
173     const PetscSF *sfs;
174     PetscCall(DMPlexGetIsoperiodicFaceSF(dm, &num_face_sfs, &sfs));
175     PetscCall(DMPlexSetIsoperiodicFaceSF(dm, num_face_sfs, (PetscSF *)sfs)); // for the compose function effect on dm
176   }
177   PetscCall(DMDestroyLabelLinkList_Internal(dm));
178   PetscCall(DMCopyLabels(dmNew, dm, PETSC_OWN_POINTER, PETSC_TRUE, DM_COPY_LABELS_FAIL));
179   PetscCall(DMGetCoarseDM(dmNew, &coarseDM));
180   PetscCall(DMSetCoarseDM(dm, coarseDM));
181   PetscCall(DMPlexCopyEGADSInfo_Internal(dmNew, dm));
182   PetscCall(DMDestroy(ndm));
183   PetscFunctionReturn(PETSC_SUCCESS);
184 }
185 
186 /* Swap dm with the contents of dmNew
187    - Swap the DM_Plex structure
188    - Swap the coordinates
189    - Swap the point PetscSF
190 */
191 static PetscErrorCode DMPlexSwap_Static(DM dmA, DM dmB)
192 {
193   DM          coordDMA, coordDMB;
194   Vec         coordsA, coordsB;
195   PetscSF     sfA, sfB;
196   DMField     fieldTmp;
197   void       *tmp;
198   DMLabelLink listTmp;
199   DMLabel     depthTmp;
200   PetscInt    tmpI;
201 
202   PetscFunctionBegin;
203   if (dmA == dmB) PetscFunctionReturn(PETSC_SUCCESS);
204   PetscCall(DMGetPointSF(dmA, &sfA));
205   PetscCall(DMGetPointSF(dmB, &sfB));
206   PetscCall(PetscObjectReference((PetscObject)sfA));
207   PetscCall(DMSetPointSF(dmA, sfB));
208   PetscCall(DMSetPointSF(dmB, sfA));
209   PetscCall(PetscObjectDereference((PetscObject)sfA));
210 
211   PetscCall(DMGetCoordinateDM(dmA, &coordDMA));
212   PetscCall(DMGetCoordinateDM(dmB, &coordDMB));
213   PetscCall(PetscObjectReference((PetscObject)coordDMA));
214   PetscCall(DMSetCoordinateDM(dmA, coordDMB));
215   PetscCall(DMSetCoordinateDM(dmB, coordDMA));
216   PetscCall(PetscObjectDereference((PetscObject)coordDMA));
217 
218   PetscCall(DMGetCoordinatesLocal(dmA, &coordsA));
219   PetscCall(DMGetCoordinatesLocal(dmB, &coordsB));
220   PetscCall(PetscObjectReference((PetscObject)coordsA));
221   PetscCall(DMSetCoordinatesLocal(dmA, coordsB));
222   PetscCall(DMSetCoordinatesLocal(dmB, coordsA));
223   PetscCall(PetscObjectDereference((PetscObject)coordsA));
224 
225   PetscCall(DMGetCellCoordinateDM(dmA, &coordDMA));
226   PetscCall(DMGetCellCoordinateDM(dmB, &coordDMB));
227   PetscCall(PetscObjectReference((PetscObject)coordDMA));
228   PetscCall(DMSetCellCoordinateDM(dmA, coordDMB));
229   PetscCall(DMSetCellCoordinateDM(dmB, coordDMA));
230   PetscCall(PetscObjectDereference((PetscObject)coordDMA));
231 
232   PetscCall(DMGetCellCoordinatesLocal(dmA, &coordsA));
233   PetscCall(DMGetCellCoordinatesLocal(dmB, &coordsB));
234   PetscCall(PetscObjectReference((PetscObject)coordsA));
235   PetscCall(DMSetCellCoordinatesLocal(dmA, coordsB));
236   PetscCall(DMSetCellCoordinatesLocal(dmB, coordsA));
237   PetscCall(PetscObjectDereference((PetscObject)coordsA));
238 
239   PetscCall(DMPlexSwapEGADSInfo_Private(dmA, dmB));
240 
241   fieldTmp                  = dmA->coordinates[0].field;
242   dmA->coordinates[0].field = dmB->coordinates[0].field;
243   dmB->coordinates[0].field = fieldTmp;
244   fieldTmp                  = dmA->coordinates[1].field;
245   dmA->coordinates[1].field = dmB->coordinates[1].field;
246   dmB->coordinates[1].field = fieldTmp;
247   tmp                       = dmA->data;
248   dmA->data                 = dmB->data;
249   dmB->data                 = tmp;
250   listTmp                   = dmA->labels;
251   dmA->labels               = dmB->labels;
252   dmB->labels               = listTmp;
253   depthTmp                  = dmA->depthLabel;
254   dmA->depthLabel           = dmB->depthLabel;
255   dmB->depthLabel           = depthTmp;
256   depthTmp                  = dmA->celltypeLabel;
257   dmA->celltypeLabel        = dmB->celltypeLabel;
258   dmB->celltypeLabel        = depthTmp;
259   tmpI                      = dmA->levelup;
260   dmA->levelup              = dmB->levelup;
261   dmB->levelup              = tmpI;
262   PetscFunctionReturn(PETSC_SUCCESS);
263 }
264 
265 PetscErrorCode DMPlexInterpolateInPlace_Internal(DM dm)
266 {
267   DM idm;
268 
269   PetscFunctionBegin;
270   PetscCall(DMPlexInterpolate(dm, &idm));
271   PetscCall(DMPlexCopyCoordinates(dm, idm));
272   PetscCall(DMPlexReplace_Internal(dm, &idm));
273   PetscFunctionReturn(PETSC_SUCCESS);
274 }
275 
276 /*@C
277   DMPlexCreateCoordinateSpace - Creates a finite element space for the coordinates
278 
279   Collective
280 
281   Input Parameters:
282 + dm        - The `DMPLEX`
283 . degree    - The degree of the finite element or `PETSC_DECIDE`
284 . project   - Flag to project current coordinates into the space
285 - coordFunc - An optional function to map new points from refinement to the surface
286 
287   Level: advanced
288 
289 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `PetscPointFunc`, `PetscFECreateLagrange()`, `DMGetCoordinateDM()`
290 @*/
291 PetscErrorCode DMPlexCreateCoordinateSpace(DM dm, PetscInt degree, PetscBool project, PetscPointFunc coordFunc)
292 {
293   DM_Plex *mesh = (DM_Plex *)dm->data;
294   PetscFE  fe   = NULL;
295   DM       cdm;
296   PetscInt dim, cdim, dE, qorder, height;
297 
298   PetscFunctionBegin;
299   PetscCall(DMGetDimension(dm, &dim));
300   cdim = dim;
301   PetscCall(DMGetCoordinateDim(dm, &dE));
302   qorder = degree;
303   PetscCall(DMGetCoordinateDM(dm, &cdm));
304   PetscObjectOptionsBegin((PetscObject)cdm);
305   PetscCall(PetscOptionsBoundedInt("-default_quadrature_order", "Quadrature order is one less than quadrature points per edge", "DMPlexCreateCoordinateSpace", qorder, &qorder, NULL, 0));
306   PetscCall(PetscOptionsBoundedInt("-dm_plex_coordinate_dim", "Set the coordinate dimension", "DMPlexCreateCoordinateSpace", cdim, &cdim, NULL, dim));
307   PetscOptionsEnd();
308   PetscCall(DMPlexGetVTKCellHeight(dm, &height));
309   if (cdim > dim) {
310     DM           cdm;
311     PetscSection cs, csNew;
312     Vec          coordinates, coordinatesNew;
313     VecType      vectype;
314     IS           idx;
315     PetscInt    *indices;
316     PetscInt     bs, n;
317 
318     // Recreate coordinate section
319     {
320       const char *fieldName = NULL, *compName = NULL;
321       PetscInt    Nc, pStart, pEnd;
322 
323       PetscCall(DMGetCoordinateDM(dm, &cdm));
324       PetscCall(DMGetLocalSection(cdm, &cs));
325       PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)cs), &csNew));
326       PetscCall(PetscSectionSetNumFields(csNew, 1));
327       PetscCall(PetscSectionGetFieldName(cs, 0, &fieldName));
328       PetscCall(PetscSectionSetFieldName(csNew, 0, fieldName));
329       PetscCall(PetscSectionGetFieldComponents(cs, 0, &Nc));
330       PetscCall(PetscSectionSetFieldComponents(csNew, 0, cdim));
331       for (PetscInt c = 0; c < Nc; ++c) {
332         PetscCall(PetscSectionGetComponentName(cs, 0, c, &compName));
333         PetscCall(PetscSectionSetComponentName(csNew, 0, c, compName));
334       }
335       PetscCall(PetscSectionGetChart(cs, &pStart, &pEnd));
336       PetscCall(PetscSectionSetChart(csNew, pStart, pEnd));
337       for (PetscInt p = pStart; p < pEnd; ++p) {
338         PetscInt dof;
339 
340         PetscCall(PetscSectionGetDof(cs, p, &dof));
341         if (dof) {
342           PetscCall(PetscSectionSetDof(csNew, p, cdim));
343           PetscCall(PetscSectionSetFieldDof(csNew, p, 0, cdim));
344         }
345       }
346       PetscCall(PetscSectionSetUp(csNew));
347     }
348     PetscCall(DMSetLocalSection(cdm, csNew));
349     PetscCall(PetscSectionDestroy(&csNew));
350     // Reset coordinate dimension for coordinate DM
351     PetscCall(DMSetCoordinateDim(cdm, cdim));
352     PetscCall(DMSetCoordinateField(cdm, NULL));
353     // Inject coordinates into higher dimension
354     PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
355     PetscCall(VecGetBlockSize(coordinates, &bs));
356     PetscCheck(bs == dim, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "We can only inject simple coordinates into a higher dimension");
357     PetscCall(VecCreate(PetscObjectComm((PetscObject)coordinates), &coordinatesNew));
358     PetscCall(VecGetType(coordinates, &vectype));
359     PetscCall(VecSetType(coordinatesNew, vectype));
360     PetscCall(VecGetLocalSize(coordinates, &n));
361     PetscCheck(!(n % bs), PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "We can only inject simple coordinates into a higher dimension");
362     n /= bs;
363     PetscCall(VecSetSizes(coordinatesNew, n * cdim, PETSC_DETERMINE));
364     PetscCall(VecSetUp(coordinatesNew));
365     PetscCall(PetscMalloc1(n * bs, &indices));
366     for (PetscInt i = 0; i < n; ++i)
367       for (PetscInt b = 0; b < bs; ++b) indices[i * bs + b] = i * cdim + b;
368     PetscCall(ISCreateGeneral(PETSC_COMM_SELF, n * bs, indices, PETSC_OWN_POINTER, &idx));
369     PetscCall(VecISCopy(coordinatesNew, idx, SCATTER_FORWARD, coordinates));
370     PetscCall(ISDestroy(&idx));
371     PetscCall(DMSetCoordinatesLocal(dm, coordinatesNew));
372     PetscCall(VecDestroy(&coordinatesNew));
373     PetscCall(DMSetCoordinateDim(dm, cdim));
374     {
375       PetscInt gn;
376 
377       PetscCall(DMGetCoordinates(dm, &coordinatesNew));
378       PetscCall(VecGetLocalSize(coordinatesNew, &gn));
379       PetscCheck(gn == n * cdim, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Global coordinate size %" PetscInt_FMT " != %" PetscInt_FMT "local coordinate size", gn, n * cdim);
380     }
381     dE      = cdim;
382     project = PETSC_FALSE;
383   }
384   if (degree >= 0) {
385     DMPolytopeType ct = DM_POLYTOPE_UNKNOWN;
386     PetscInt       cStart, cEnd, gct;
387 
388     PetscCall(DMPlexGetHeightStratum(dm, height, &cStart, &cEnd));
389     if (cEnd > cStart) PetscCall(DMPlexGetCellType(dm, cStart, &ct));
390     gct = (PetscInt)ct;
391     PetscCallMPI(MPIU_Allreduce(MPI_IN_PLACE, &gct, 1, MPIU_INT, MPI_MIN, PetscObjectComm((PetscObject)dm)));
392     ct = (DMPolytopeType)gct;
393     // Work around current bug in PetscDualSpaceSetUp_Lagrange()
394     //   Can be seen in plex_tutorials-ex10_1
395     if (ct != DM_POLYTOPE_SEG_PRISM_TENSOR && ct != DM_POLYTOPE_TRI_PRISM_TENSOR && ct != DM_POLYTOPE_QUAD_PRISM_TENSOR) PetscCall(PetscFECreateLagrangeByCell(PETSC_COMM_SELF, dim, dE, ct, degree, qorder, &fe));
396   }
397   PetscCall(DMSetCoordinateDisc(dm, fe, project));
398   PetscCall(PetscFEDestroy(&fe));
399   mesh->coordFunc = coordFunc;
400   PetscFunctionReturn(PETSC_SUCCESS);
401 }
402 
403 /*@
404   DMPlexCreateDoublet - Creates a mesh of two cells of the specified type, optionally with later refinement.
405 
406   Collective
407 
408   Input Parameters:
409 + comm            - The communicator for the `DM` object
410 . dim             - The spatial dimension
411 . simplex         - Flag for simplicial cells, otherwise they are tensor product cells
412 . interpolate     - Flag to create intermediate mesh pieces (edges, faces)
413 - refinementLimit - A nonzero number indicates the largest admissible volume for a refined cell
414 
415   Output Parameter:
416 . newdm - The `DM` object
417 
418   Level: beginner
419 
420 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMSetType()`, `DMCreate()`
421 @*/
422 PetscErrorCode DMPlexCreateDoublet(MPI_Comm comm, PetscInt dim, PetscBool simplex, PetscBool interpolate, PetscReal refinementLimit, DM *newdm)
423 {
424   DM          dm;
425   PetscMPIInt rank;
426 
427   PetscFunctionBegin;
428   PetscCall(DMCreate(comm, &dm));
429   PetscCall(DMSetType(dm, DMPLEX));
430   PetscCall(DMSetDimension(dm, dim));
431   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
432   PetscCallMPI(MPI_Comm_rank(comm, &rank));
433   switch (dim) {
434   case 2:
435     if (simplex) PetscCall(PetscObjectSetName((PetscObject)dm, "triangular"));
436     else PetscCall(PetscObjectSetName((PetscObject)dm, "quadrilateral"));
437     break;
438   case 3:
439     if (simplex) PetscCall(PetscObjectSetName((PetscObject)dm, "tetrahedral"));
440     else PetscCall(PetscObjectSetName((PetscObject)dm, "hexahedral"));
441     break;
442   default:
443     SETERRQ(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %" PetscInt_FMT, dim);
444   }
445   if (rank) {
446     PetscInt numPoints[2] = {0, 0};
447     PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, NULL, NULL, NULL, NULL));
448   } else {
449     switch (dim) {
450     case 2:
451       if (simplex) {
452         PetscInt    numPoints[2]        = {4, 2};
453         PetscInt    coneSize[6]         = {3, 3, 0, 0, 0, 0};
454         PetscInt    cones[6]            = {2, 3, 4, 5, 4, 3};
455         PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
456         PetscScalar vertexCoords[8]     = {-0.5, 0.5, 0.0, 0.0, 0.0, 1.0, 0.5, 0.5};
457 
458         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
459       } else {
460         PetscInt    numPoints[2]        = {6, 2};
461         PetscInt    coneSize[8]         = {4, 4, 0, 0, 0, 0, 0, 0};
462         PetscInt    cones[8]            = {2, 3, 4, 5, 3, 6, 7, 4};
463         PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
464         PetscScalar vertexCoords[12]    = {-1.0, -0.5, 0.0, -0.5, 0.0, 0.5, -1.0, 0.5, 1.0, -0.5, 1.0, 0.5};
465 
466         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
467       }
468       break;
469     case 3:
470       if (simplex) {
471         PetscInt    numPoints[2]        = {5, 2};
472         PetscInt    coneSize[7]         = {4, 4, 0, 0, 0, 0, 0};
473         PetscInt    cones[8]            = {4, 3, 5, 2, 5, 3, 4, 6};
474         PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
475         PetscScalar vertexCoords[15]    = {-1.0, 0.0, 0.0, 0.0, -1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 1.0, 0.0, 1.0, 0.0, 0.0};
476 
477         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
478       } else {
479         PetscInt    numPoints[2]         = {12, 2};
480         PetscInt    coneSize[14]         = {8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
481         PetscInt    cones[16]            = {2, 3, 4, 5, 6, 7, 8, 9, 5, 4, 10, 11, 7, 12, 13, 8};
482         PetscInt    coneOrientations[16] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
483         PetscScalar vertexCoords[36]     = {-1.0, -0.5, -0.5, -1.0, 0.5, -0.5, 0.0, 0.5, -0.5, 0.0, -0.5, -0.5, -1.0, -0.5, 0.5, 0.0, -0.5, 0.5, 0.0, 0.5, 0.5, -1.0, 0.5, 0.5, 1.0, 0.5, -0.5, 1.0, -0.5, -0.5, 1.0, -0.5, 0.5, 1.0, 0.5, 0.5};
484 
485         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
486       }
487       break;
488     default:
489       SETERRQ(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %" PetscInt_FMT, dim);
490     }
491   }
492   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
493   *newdm = dm;
494   if (refinementLimit > 0.0) {
495     DM          rdm;
496     const char *name;
497 
498     PetscCall(DMPlexSetRefinementUniform(*newdm, PETSC_FALSE));
499     PetscCall(DMPlexSetRefinementLimit(*newdm, refinementLimit));
500     PetscCall(DMRefine(*newdm, comm, &rdm));
501     PetscCall(PetscObjectGetName((PetscObject)*newdm, &name));
502     PetscCall(PetscObjectSetName((PetscObject)rdm, name));
503     PetscCall(DMDestroy(newdm));
504     *newdm = rdm;
505   }
506   if (interpolate) {
507     DM idm;
508 
509     PetscCall(DMPlexInterpolate(*newdm, &idm));
510     PetscCall(DMDestroy(newdm));
511     *newdm = idm;
512   }
513   PetscFunctionReturn(PETSC_SUCCESS);
514 }
515 
516 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_1D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[])
517 {
518   const PetscInt numVertices    = 2;
519   PetscInt       markerRight    = 1;
520   PetscInt       markerLeft     = 1;
521   PetscBool      markerSeparate = PETSC_FALSE;
522   Vec            coordinates;
523   PetscSection   coordSection;
524   PetscScalar   *coords;
525   PetscInt       coordSize;
526   PetscMPIInt    rank;
527   PetscInt       cdim = 1, v;
528 
529   PetscFunctionBegin;
530   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
531   if (markerSeparate) {
532     markerRight = 2;
533     markerLeft  = 1;
534   }
535   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
536   if (rank == 0) {
537     PetscCall(DMPlexSetChart(dm, 0, numVertices));
538     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
539     PetscCall(DMSetLabelValue(dm, "marker", 0, markerLeft));
540     PetscCall(DMSetLabelValue(dm, "marker", 1, markerRight));
541   }
542   PetscCall(DMPlexSymmetrize(dm));
543   PetscCall(DMPlexStratify(dm));
544   /* Build coordinates */
545   PetscCall(DMSetCoordinateDim(dm, cdim));
546   PetscCall(DMGetCoordinateSection(dm, &coordSection));
547   PetscCall(PetscSectionSetNumFields(coordSection, 1));
548   PetscCall(PetscSectionSetChart(coordSection, 0, numVertices));
549   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, cdim));
550   for (v = 0; v < numVertices; ++v) {
551     PetscCall(PetscSectionSetDof(coordSection, v, cdim));
552     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, cdim));
553   }
554   PetscCall(PetscSectionSetUp(coordSection));
555   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
556   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
557   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
558   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
559   PetscCall(VecSetBlockSize(coordinates, cdim));
560   PetscCall(VecSetType(coordinates, VECSTANDARD));
561   PetscCall(VecGetArray(coordinates, &coords));
562   coords[0] = lower[0];
563   coords[1] = upper[0];
564   PetscCall(VecRestoreArray(coordinates, &coords));
565   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
566   PetscCall(VecDestroy(&coordinates));
567   PetscFunctionReturn(PETSC_SUCCESS);
568 }
569 
570 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_2D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[])
571 {
572   const PetscInt numVertices    = (edges[0] + 1) * (edges[1] + 1);
573   const PetscInt numEdges       = edges[0] * (edges[1] + 1) + (edges[0] + 1) * edges[1];
574   PetscInt       markerTop      = 1;
575   PetscInt       markerBottom   = 1;
576   PetscInt       markerRight    = 1;
577   PetscInt       markerLeft     = 1;
578   PetscBool      markerSeparate = PETSC_FALSE;
579   Vec            coordinates;
580   PetscSection   coordSection;
581   PetscScalar   *coords;
582   PetscInt       coordSize;
583   PetscMPIInt    rank;
584   PetscInt       v, vx, vy;
585 
586   PetscFunctionBegin;
587   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
588   if (markerSeparate) {
589     markerTop    = 3;
590     markerBottom = 1;
591     markerRight  = 2;
592     markerLeft   = 4;
593   }
594   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
595   if (rank == 0) {
596     PetscInt e, ex, ey;
597 
598     PetscCall(DMPlexSetChart(dm, 0, numEdges + numVertices));
599     for (e = 0; e < numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
600     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
601     for (vx = 0; vx <= edges[0]; vx++) {
602       for (ey = 0; ey < edges[1]; ey++) {
603         PetscInt edge   = vx * edges[1] + ey + edges[0] * (edges[1] + 1);
604         PetscInt vertex = ey * (edges[0] + 1) + vx + numEdges;
605         PetscInt cone[2];
606 
607         cone[0] = vertex;
608         cone[1] = vertex + edges[0] + 1;
609         PetscCall(DMPlexSetCone(dm, edge, cone));
610         if (vx == edges[0]) {
611           PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
612           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
613           if (ey == edges[1] - 1) {
614             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
615             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerRight));
616           }
617         } else if (vx == 0) {
618           PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
619           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
620           if (ey == edges[1] - 1) {
621             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
622             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerLeft));
623           }
624         }
625       }
626     }
627     for (vy = 0; vy <= edges[1]; vy++) {
628       for (ex = 0; ex < edges[0]; ex++) {
629         PetscInt edge   = vy * edges[0] + ex;
630         PetscInt vertex = vy * (edges[0] + 1) + ex + numEdges;
631         PetscInt cone[2];
632 
633         cone[0] = vertex;
634         cone[1] = vertex + 1;
635         PetscCall(DMPlexSetCone(dm, edge, cone));
636         if (vy == edges[1]) {
637           PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
638           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
639           if (ex == edges[0] - 1) {
640             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
641             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerTop));
642           }
643         } else if (vy == 0) {
644           PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
645           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
646           if (ex == edges[0] - 1) {
647             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
648             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerBottom));
649           }
650         }
651       }
652     }
653   }
654   PetscCall(DMPlexSymmetrize(dm));
655   PetscCall(DMPlexStratify(dm));
656   /* Build coordinates */
657   PetscCall(DMSetCoordinateDim(dm, 2));
658   PetscCall(DMGetCoordinateSection(dm, &coordSection));
659   PetscCall(PetscSectionSetNumFields(coordSection, 1));
660   PetscCall(PetscSectionSetChart(coordSection, numEdges, numEdges + numVertices));
661   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, 2));
662   for (v = numEdges; v < numEdges + numVertices; ++v) {
663     PetscCall(PetscSectionSetDof(coordSection, v, 2));
664     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, 2));
665   }
666   PetscCall(PetscSectionSetUp(coordSection));
667   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
668   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
669   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
670   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
671   PetscCall(VecSetBlockSize(coordinates, 2));
672   PetscCall(VecSetType(coordinates, VECSTANDARD));
673   PetscCall(VecGetArray(coordinates, &coords));
674   for (vy = 0; vy <= edges[1]; ++vy) {
675     for (vx = 0; vx <= edges[0]; ++vx) {
676       coords[(vy * (edges[0] + 1) + vx) * 2 + 0] = lower[0] + ((upper[0] - lower[0]) / edges[0]) * vx;
677       coords[(vy * (edges[0] + 1) + vx) * 2 + 1] = lower[1] + ((upper[1] - lower[1]) / edges[1]) * vy;
678     }
679   }
680   PetscCall(VecRestoreArray(coordinates, &coords));
681   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
682   PetscCall(VecDestroy(&coordinates));
683   PetscFunctionReturn(PETSC_SUCCESS);
684 }
685 
686 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_3D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt faces[])
687 {
688   PetscInt     vertices[3], numVertices;
689   PetscInt     numFaces       = 2 * faces[0] * faces[1] + 2 * faces[1] * faces[2] + 2 * faces[0] * faces[2];
690   PetscInt     markerTop      = 1;
691   PetscInt     markerBottom   = 1;
692   PetscInt     markerFront    = 1;
693   PetscInt     markerBack     = 1;
694   PetscInt     markerRight    = 1;
695   PetscInt     markerLeft     = 1;
696   PetscBool    markerSeparate = PETSC_FALSE;
697   Vec          coordinates;
698   PetscSection coordSection;
699   PetscScalar *coords;
700   PetscInt     coordSize;
701   PetscMPIInt  rank;
702   PetscInt     v, vx, vy, vz;
703   PetscInt     voffset, iface = 0, cone[4];
704 
705   PetscFunctionBegin;
706   PetscCheck(faces[0] >= 1 && faces[1] >= 1 && faces[2] >= 1, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Must have at least 1 face per side");
707   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
708   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
709   if (markerSeparate) {
710     markerBottom = 1;
711     markerTop    = 2;
712     markerFront  = 3;
713     markerBack   = 4;
714     markerRight  = 5;
715     markerLeft   = 6;
716   }
717   vertices[0] = faces[0] + 1;
718   vertices[1] = faces[1] + 1;
719   vertices[2] = faces[2] + 1;
720   numVertices = vertices[0] * vertices[1] * vertices[2];
721   if (rank == 0) {
722     PetscInt f;
723 
724     PetscCall(DMPlexSetChart(dm, 0, numFaces + numVertices));
725     for (f = 0; f < numFaces; ++f) PetscCall(DMPlexSetConeSize(dm, f, 4));
726     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
727 
728     /* Side 0 (Top) */
729     for (vy = 0; vy < faces[1]; vy++) {
730       for (vx = 0; vx < faces[0]; vx++) {
731         voffset = numFaces + vertices[0] * vertices[1] * (vertices[2] - 1) + vy * vertices[0] + vx;
732         cone[0] = voffset;
733         cone[1] = voffset + 1;
734         cone[2] = voffset + vertices[0] + 1;
735         cone[3] = voffset + vertices[0];
736         PetscCall(DMPlexSetCone(dm, iface, cone));
737         PetscCall(DMSetLabelValue(dm, "marker", iface, markerTop));
738         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerTop));
739         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerTop));
740         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerTop));
741         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 1, markerTop));
742         iface++;
743       }
744     }
745 
746     /* Side 1 (Bottom) */
747     for (vy = 0; vy < faces[1]; vy++) {
748       for (vx = 0; vx < faces[0]; vx++) {
749         voffset = numFaces + vy * (faces[0] + 1) + vx;
750         cone[0] = voffset + 1;
751         cone[1] = voffset;
752         cone[2] = voffset + vertices[0];
753         cone[3] = voffset + vertices[0] + 1;
754         PetscCall(DMPlexSetCone(dm, iface, cone));
755         PetscCall(DMSetLabelValue(dm, "marker", iface, markerBottom));
756         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerBottom));
757         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerBottom));
758         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerBottom));
759         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 1, markerBottom));
760         iface++;
761       }
762     }
763 
764     /* Side 2 (Front) */
765     for (vz = 0; vz < faces[2]; vz++) {
766       for (vx = 0; vx < faces[0]; vx++) {
767         voffset = numFaces + vz * vertices[0] * vertices[1] + vx;
768         cone[0] = voffset;
769         cone[1] = voffset + 1;
770         cone[2] = voffset + vertices[0] * vertices[1] + 1;
771         cone[3] = voffset + vertices[0] * vertices[1];
772         PetscCall(DMPlexSetCone(dm, iface, cone));
773         PetscCall(DMSetLabelValue(dm, "marker", iface, markerFront));
774         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerFront));
775         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerFront));
776         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 0, markerFront));
777         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 1, markerFront));
778         iface++;
779       }
780     }
781 
782     /* Side 3 (Back) */
783     for (vz = 0; vz < faces[2]; vz++) {
784       for (vx = 0; vx < faces[0]; vx++) {
785         voffset = numFaces + vz * vertices[0] * vertices[1] + vertices[0] * (vertices[1] - 1) + vx;
786         cone[0] = voffset + vertices[0] * vertices[1];
787         cone[1] = voffset + vertices[0] * vertices[1] + 1;
788         cone[2] = voffset + 1;
789         cone[3] = voffset;
790         PetscCall(DMPlexSetCone(dm, iface, cone));
791         PetscCall(DMSetLabelValue(dm, "marker", iface, markerBack));
792         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerBack));
793         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerBack));
794         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 0, markerBack));
795         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 1, markerBack));
796         iface++;
797       }
798     }
799 
800     /* Side 4 (Left) */
801     for (vz = 0; vz < faces[2]; vz++) {
802       for (vy = 0; vy < faces[1]; vy++) {
803         voffset = numFaces + vz * vertices[0] * vertices[1] + vy * vertices[0];
804         cone[0] = voffset;
805         cone[1] = voffset + vertices[0] * vertices[1];
806         cone[2] = voffset + vertices[0] * vertices[1] + vertices[0];
807         cone[3] = voffset + vertices[0];
808         PetscCall(DMPlexSetCone(dm, iface, cone));
809         PetscCall(DMSetLabelValue(dm, "marker", iface, markerLeft));
810         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerLeft));
811         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerLeft));
812         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[1] + 0, markerLeft));
813         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + vertices[0], markerLeft));
814         iface++;
815       }
816     }
817 
818     /* Side 5 (Right) */
819     for (vz = 0; vz < faces[2]; vz++) {
820       for (vy = 0; vy < faces[1]; vy++) {
821         voffset = numFaces + vz * vertices[0] * vertices[1] + vy * vertices[0] + faces[0];
822         cone[0] = voffset + vertices[0] * vertices[1];
823         cone[1] = voffset;
824         cone[2] = voffset + vertices[0];
825         cone[3] = voffset + vertices[0] * vertices[1] + vertices[0];
826         PetscCall(DMPlexSetCone(dm, iface, cone));
827         PetscCall(DMSetLabelValue(dm, "marker", iface, markerRight));
828         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerRight));
829         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerRight));
830         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 0, markerRight));
831         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + vertices[0], markerRight));
832         iface++;
833       }
834     }
835   }
836   PetscCall(DMPlexSymmetrize(dm));
837   PetscCall(DMPlexStratify(dm));
838   /* Build coordinates */
839   PetscCall(DMSetCoordinateDim(dm, 3));
840   PetscCall(DMGetCoordinateSection(dm, &coordSection));
841   PetscCall(PetscSectionSetNumFields(coordSection, 1));
842   PetscCall(PetscSectionSetChart(coordSection, numFaces, numFaces + numVertices));
843   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, 3));
844   for (v = numFaces; v < numFaces + numVertices; ++v) {
845     PetscCall(PetscSectionSetDof(coordSection, v, 3));
846     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, 3));
847   }
848   PetscCall(PetscSectionSetUp(coordSection));
849   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
850   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
851   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
852   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
853   PetscCall(VecSetBlockSize(coordinates, 3));
854   PetscCall(VecSetType(coordinates, VECSTANDARD));
855   PetscCall(VecGetArray(coordinates, &coords));
856   for (vz = 0; vz <= faces[2]; ++vz) {
857     for (vy = 0; vy <= faces[1]; ++vy) {
858       for (vx = 0; vx <= faces[0]; ++vx) {
859         coords[((vz * (faces[1] + 1) + vy) * (faces[0] + 1) + vx) * 3 + 0] = lower[0] + ((upper[0] - lower[0]) / faces[0]) * vx;
860         coords[((vz * (faces[1] + 1) + vy) * (faces[0] + 1) + vx) * 3 + 1] = lower[1] + ((upper[1] - lower[1]) / faces[1]) * vy;
861         coords[((vz * (faces[1] + 1) + vy) * (faces[0] + 1) + vx) * 3 + 2] = lower[2] + ((upper[2] - lower[2]) / faces[2]) * vz;
862       }
863     }
864   }
865   PetscCall(VecRestoreArray(coordinates, &coords));
866   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
867   PetscCall(VecDestroy(&coordinates));
868   PetscFunctionReturn(PETSC_SUCCESS);
869 }
870 
871 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], PetscBool interpolate)
872 {
873   PetscFunctionBegin;
874   PetscValidLogicalCollectiveInt(dm, dim, 2);
875   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
876   PetscCall(DMSetDimension(dm, dim - 1));
877   PetscCall(DMSetCoordinateDim(dm, dim));
878   switch (dim) {
879   case 1:
880     PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_1D_Internal(dm, lower, upper, faces));
881     break;
882   case 2:
883     PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_2D_Internal(dm, lower, upper, faces));
884     break;
885   case 3:
886     PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_3D_Internal(dm, lower, upper, faces));
887     break;
888   default:
889     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Dimension not supported: %" PetscInt_FMT, dim);
890   }
891   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
892   if (interpolate) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
893   PetscFunctionReturn(PETSC_SUCCESS);
894 }
895 
896 /*@C
897   DMPlexCreateBoxSurfaceMesh - Creates a mesh on the surface of the tensor product of unit intervals (box) using tensor cells (hexahedra).
898 
899   Collective
900 
901   Input Parameters:
902 + comm        - The communicator for the `DM` object
903 . dim         - The spatial dimension of the box, so the resulting mesh is has dimension `dim`-1
904 . faces       - Number of faces per dimension, or `NULL` for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
905 . lower       - The lower left corner, or `NULL` for (0, 0, 0)
906 . upper       - The upper right corner, or `NULL` for (1, 1, 1)
907 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
908 
909   Output Parameter:
910 . dm - The `DM` object
911 
912   Level: beginner
913 
914 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMSetFromOptions()`, `DMPlexCreateBoxMesh()`, `DMPlexCreateFromFile()`, `DMSetType()`, `DMCreate()`
915 @*/
916 PetscErrorCode DMPlexCreateBoxSurfaceMesh(MPI_Comm comm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], PetscBool interpolate, DM *dm)
917 {
918   PetscInt  fac[3] = {1, 1, 1};
919   PetscReal low[3] = {0, 0, 0};
920   PetscReal upp[3] = {1, 1, 1};
921 
922   PetscFunctionBegin;
923   PetscCall(DMCreate(comm, dm));
924   PetscCall(DMSetType(*dm, DMPLEX));
925   PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(*dm, dim, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, interpolate));
926   PetscFunctionReturn(PETSC_SUCCESS);
927 }
928 
929 static PetscErrorCode DMPlexCreateLineMesh_Internal(DM dm, PetscInt segments, PetscReal lower, PetscReal upper, DMBoundaryType bd)
930 {
931   PetscInt     i, fStart, fEnd, numCells = 0, numVerts = 0;
932   PetscInt     numPoints[2], *coneSize, *cones, *coneOrientations;
933   PetscScalar *vertexCoords;
934   PetscReal    L, maxCell;
935   PetscBool    markerSeparate = PETSC_FALSE;
936   PetscInt     markerLeft = 1, faceMarkerLeft = 1;
937   PetscInt     markerRight = 1, faceMarkerRight = 2;
938   PetscBool    wrap = (bd == DM_BOUNDARY_PERIODIC || bd == DM_BOUNDARY_TWIST) ? PETSC_TRUE : PETSC_FALSE;
939   PetscMPIInt  rank;
940 
941   PetscFunctionBegin;
942   PetscAssertPointer(dm, 1);
943 
944   PetscCall(DMSetDimension(dm, 1));
945   PetscCall(DMCreateLabel(dm, "marker"));
946   PetscCall(DMCreateLabel(dm, "Face Sets"));
947 
948   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
949   if (rank == 0) numCells = segments;
950   if (rank == 0) numVerts = segments + (wrap ? 0 : 1);
951 
952   numPoints[0] = numVerts;
953   numPoints[1] = numCells;
954   PetscCall(PetscMalloc4(numCells + numVerts, &coneSize, numCells * 2, &cones, numCells + numVerts, &coneOrientations, numVerts, &vertexCoords));
955   PetscCall(PetscArrayzero(coneOrientations, numCells + numVerts));
956   for (i = 0; i < numCells; ++i) coneSize[i] = 2;
957   for (i = 0; i < numVerts; ++i) coneSize[numCells + i] = 0;
958   for (i = 0; i < numCells; ++i) {
959     cones[2 * i]     = numCells + i % numVerts;
960     cones[2 * i + 1] = numCells + (i + 1) % numVerts;
961   }
962   for (i = 0; i < numVerts; ++i) vertexCoords[i] = lower + (upper - lower) * ((PetscReal)i / (PetscReal)numCells);
963   PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
964   PetscCall(PetscFree4(coneSize, cones, coneOrientations, vertexCoords));
965 
966   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
967   if (markerSeparate) {
968     markerLeft  = faceMarkerLeft;
969     markerRight = faceMarkerRight;
970   }
971   if (!wrap && rank == 0) {
972     PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
973     PetscCall(DMSetLabelValue(dm, "marker", fStart, markerLeft));
974     PetscCall(DMSetLabelValue(dm, "marker", fEnd - 1, markerRight));
975     PetscCall(DMSetLabelValue(dm, "Face Sets", fStart, faceMarkerLeft));
976     PetscCall(DMSetLabelValue(dm, "Face Sets", fEnd - 1, faceMarkerRight));
977   }
978   if (wrap) {
979     L       = upper - lower;
980     maxCell = (PetscReal)1.1 * (L / (PetscReal)PetscMax(1, segments));
981     PetscCall(DMSetPeriodicity(dm, &maxCell, &lower, &L));
982   }
983   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
984   PetscFunctionReturn(PETSC_SUCCESS);
985 }
986 
987 // Creates "Face Sets" label based on the standard box labeling conventions
988 static PetscErrorCode DMPlexSetBoxLabel_Internal(DM dm, const DMBoundaryType periodicity[])
989 {
990   DM              cdm;
991   PetscSection    csection;
992   Vec             coordinates;
993   DMLabel         label;
994   IS              faces_is;
995   PetscInt        dim, num_face = 0;
996   const PetscInt *faces;
997   PetscInt        faceMarkerBottom, faceMarkerTop, faceMarkerFront, faceMarkerBack, faceMarkerRight, faceMarkerLeft;
998 
999   PetscFunctionBeginUser;
1000   PetscCall(DMGetDimension(dm, &dim));
1001   PetscCheck((dim == 2) || (dim == 3), PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "DMPlex box labeling only supports 2D and 3D meshes, received DM of dimension %" PetscInt_FMT, dim);
1002   // Get Face Sets label
1003   PetscCall(DMGetLabel(dm, "Face Sets", &label));
1004   if (label) {
1005     PetscCall(DMLabelReset(label));
1006   } else {
1007     PetscCall(DMCreateLabel(dm, "Face Sets"));
1008     PetscCall(DMGetLabel(dm, "Face Sets", &label));
1009   }
1010   PetscCall(DMPlexMarkBoundaryFaces(dm, 1, label));
1011   PetscCall(DMGetStratumIS(dm, "Face Sets", 1, &faces_is));
1012 
1013   switch (dim) {
1014   case 2:
1015     faceMarkerTop    = 3;
1016     faceMarkerBottom = 1;
1017     faceMarkerRight  = 2;
1018     faceMarkerLeft   = 4;
1019     break;
1020   case 3:
1021     faceMarkerBottom = 1;
1022     faceMarkerTop    = 2;
1023     faceMarkerFront  = 3;
1024     faceMarkerBack   = 4;
1025     faceMarkerRight  = 5;
1026     faceMarkerLeft   = 6;
1027     break;
1028   default:
1029     SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Dimension %" PetscInt_FMT " not supported", dim);
1030   }
1031 
1032   if (faces_is) PetscCall(ISGetLocalSize(faces_is, &num_face));
1033   if (faces_is) PetscCall(ISGetIndices(faces_is, &faces));
1034   PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
1035   PetscCall(DMGetCoordinateDM(dm, &cdm));
1036   PetscCall(DMGetLocalSection(cdm, &csection));
1037   for (PetscInt f = 0; f < num_face; ++f) {
1038     PetscScalar *coords = NULL;
1039     PetscInt     face = faces[f], flip = 1, label_value = -1, coords_size;
1040 
1041     { // Determine if orientation of face is flipped
1042       PetscInt        num_cells_support, num_faces, start = -1;
1043       const PetscInt *orients, *cell_faces, *cells;
1044 
1045       PetscCall(DMPlexGetSupport(dm, face, &cells));
1046       PetscCall(DMPlexGetSupportSize(dm, face, &num_cells_support));
1047       PetscCheck(num_cells_support == 1, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "Expected one cell in support of exterior face, but got %" PetscInt_FMT " cells", num_cells_support);
1048       PetscCall(DMPlexGetCone(dm, cells[0], &cell_faces));
1049       PetscCall(DMPlexGetConeSize(dm, cells[0], &num_faces));
1050       for (PetscInt i = 0; i < num_faces; i++) {
1051         if (cell_faces[i] == face) start = i;
1052       }
1053       PetscCheck(start >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_CORRUPT, "Could not find face %" PetscInt_FMT " in cone of its support", face);
1054       PetscCall(DMPlexGetConeOrientation(dm, cells[0], &orients));
1055       if (orients[start] < 0) flip = -1;
1056     }
1057 
1058     // Cannot use DMPlexComputeCellGeometryFVM() for high-order geometry, so must calculate normal vectors manually
1059     // Use the vertices (depth 0) of coordinate DM to calculate normal vector
1060     PetscCall(DMPlexVecGetClosureAtDepth_Internal(cdm, csection, coordinates, face, 0, &coords_size, &coords));
1061     switch (dim) {
1062     case 2: {
1063       PetscScalar vec[2];
1064 
1065       for (PetscInt d = 0; d < dim; ++d) vec[d] = flip * (PetscRealPart(coords[1 * dim + d]) - PetscRealPart(coords[0 * dim + d]));
1066       PetscScalar normal[] = {vec[1], -vec[0]};
1067       if (PetscAbsScalar(normal[0]) > PetscAbsScalar(normal[1])) {
1068         label_value = PetscRealPart(normal[0]) > 0 ? faceMarkerRight : faceMarkerLeft;
1069       } else {
1070         label_value = PetscRealPart(normal[1]) > 0 ? faceMarkerTop : faceMarkerBottom;
1071       }
1072     } break;
1073     case 3: {
1074       PetscScalar vec1[3], vec2[3], normal[3];
1075 
1076       for (PetscInt d = 0; d < dim; ++d) {
1077         vec1[d] = PetscRealPart(coords[1 * dim + d]) - PetscRealPart(coords[0 * dim + d]);
1078         vec2[d] = PetscRealPart(coords[2 * dim + d]) - PetscRealPart(coords[1 * dim + d]);
1079       }
1080 
1081       // Calculate normal vector via cross-product
1082       normal[0] = flip * ((vec1[1] * vec2[2]) - (vec1[2] * vec2[1]));
1083       normal[1] = flip * ((vec1[2] * vec2[0]) - (vec1[0] * vec2[2]));
1084       normal[2] = flip * ((vec1[0] * vec2[1]) - (vec1[1] * vec2[0]));
1085 
1086       if (PetscAbsScalar(normal[0]) > PetscAbsScalar(normal[1])) {
1087         if (PetscAbsScalar(normal[0]) > PetscAbsScalar(normal[2])) {
1088           label_value = PetscRealPart(normal[0]) > 0 ? faceMarkerRight : faceMarkerLeft;
1089         } else {
1090           label_value = PetscRealPart(normal[2]) > 0 ? faceMarkerTop : faceMarkerBottom;
1091         }
1092       } else {
1093         if (PetscAbsScalar(normal[1]) > PetscAbsScalar(normal[2])) {
1094           label_value = PetscRealPart(normal[1]) > 0 ? faceMarkerBack : faceMarkerFront;
1095         } else {
1096           label_value = PetscRealPart(normal[2]) > 0 ? faceMarkerTop : faceMarkerBottom;
1097         }
1098       }
1099     } break;
1100     }
1101 
1102     PetscInt previous_label_value; // always 1 due to DMPlexMarkBoundaryFaces call above
1103     PetscCall(DMGetLabelValue(dm, "Face Sets", face, &previous_label_value));
1104     PetscCall(DMClearLabelValue(dm, "Face Sets", face, previous_label_value));
1105     PetscCall(DMSetLabelValue(dm, "Face Sets", face, label_value));
1106     PetscCall(DMPlexVecRestoreClosure(cdm, csection, coordinates, face, &coords_size, &coords));
1107   }
1108   if (faces_is) PetscCall(ISRestoreIndices(faces_is, &faces));
1109   PetscCall(ISDestroy(&faces_is));
1110 
1111   // Create Isoperiodic SF from newly-created face labels
1112   PetscSF     periodicsfs[3];
1113   PetscInt    periodic_sf_index  = 0;
1114   PetscScalar transform[3][4][4] = {{{0.}}};
1115   for (PetscInt d = 0; d < dim; d++) {
1116     IS              donor_is, periodic_is;
1117     const PetscInt *donor_faces = NULL, *periodic_faces = NULL;
1118     PetscInt        num_donor = 0, num_periodic = 0;
1119     PetscSF         centroidsf;
1120     PetscReal       donor_to_periodic_distance;
1121     const PetscInt  face_pairings[2][3][2] = {
1122       // 2D face pairings, {donor, periodic}
1123       {{4, 2}, {1, 3}},
1124       // 3D face pairings
1125       {{5, 6}, {3, 4}, {1, 2}}
1126     };
1127 
1128     if (periodicity[d] != DM_BOUNDARY_PERIODIC) continue;
1129     {
1130       // Compute centroidsf, which is the mapping from donor faces to periodic faces
1131       // Matches the centroid of the faces together, ignoring the periodic direction component (which should not match between donor and periodic face)
1132       PetscInt     coords_size, centroid_comps = dim - 1;
1133       PetscScalar *coords = NULL;
1134       PetscReal   *donor_centroids, *periodic_centroids;
1135       PetscReal    loc_periodic[2] = {PETSC_MIN_REAL, PETSC_MIN_REAL}, loc_periodic_global[2]; // Location of donor (0) and periodic (1) faces in periodic direction
1136 
1137       PetscCall(DMGetStratumIS(dm, "Face Sets", face_pairings[dim - 2][d][0], &donor_is));
1138       PetscCall(DMGetStratumIS(dm, "Face Sets", face_pairings[dim - 2][d][1], &periodic_is));
1139       if (donor_is) {
1140         PetscCall(ISGetLocalSize(donor_is, &num_donor));
1141         PetscCall(ISGetIndices(donor_is, &donor_faces));
1142       }
1143       if (periodic_is) {
1144         PetscCall(ISGetLocalSize(periodic_is, &num_periodic));
1145         PetscCall(ISGetIndices(periodic_is, &periodic_faces));
1146       }
1147       PetscCall(PetscCalloc2(num_donor * centroid_comps, &donor_centroids, num_periodic * centroid_comps, &periodic_centroids));
1148       for (PetscInt f = 0; f < num_donor; f++) {
1149         PetscInt face = donor_faces[f], num_coords;
1150         PetscCall(DMPlexVecGetClosureAtDepth_Internal(cdm, csection, coordinates, face, 0, &coords_size, &coords));
1151         num_coords = coords_size / dim;
1152         for (PetscInt c = 0; c < num_coords; c++) {
1153           PetscInt comp_index = 0;
1154           loc_periodic[0]     = PetscRealPart(coords[c * dim + d]);
1155           for (PetscInt i = 0; i < dim; i++) {
1156             if (i == d) continue; // Periodic direction not used for centroid calculation
1157             donor_centroids[f * centroid_comps + comp_index] += PetscRealPart(coords[c * dim + i]) / num_coords;
1158             comp_index++;
1159           }
1160         }
1161         PetscCall(DMPlexVecRestoreClosure(cdm, csection, coordinates, face, &coords_size, &coords));
1162       }
1163 
1164       for (PetscInt f = 0; f < num_periodic; f++) {
1165         PetscInt face = periodic_faces[f], num_coords;
1166         PetscCall(DMPlexVecGetClosureAtDepth_Internal(cdm, csection, coordinates, face, 0, &coords_size, &coords));
1167         num_coords = coords_size / dim;
1168         for (PetscInt c = 0; c < num_coords; c++) {
1169           PetscInt comp_index = 0;
1170           loc_periodic[1]     = PetscRealPart(coords[c * dim + d]);
1171           for (PetscInt i = 0; i < dim; i++) {
1172             if (i == d) continue; // Periodic direction not used for centroid calculation
1173             periodic_centroids[f * centroid_comps + comp_index] += PetscRealPart(coords[c * dim + i]) / num_coords;
1174             comp_index++;
1175           }
1176         }
1177         PetscCall(DMPlexVecRestoreClosure(cdm, csection, coordinates, face, &coords_size, &coords));
1178       }
1179       PetscCallMPI(MPIU_Allreduce(loc_periodic, loc_periodic_global, 2, MPIU_REAL, MPIU_MAX, PetscObjectComm((PetscObject)dm)));
1180       donor_to_periodic_distance = loc_periodic_global[1] - loc_periodic_global[0];
1181 
1182       PetscCall(PetscSFCreate(PetscObjectComm((PetscObject)dm), &centroidsf));
1183       PetscCall(PetscSFSetGraphFromCoordinates(centroidsf, num_donor, num_periodic, centroid_comps, 1e-10, donor_centroids, periodic_centroids));
1184       PetscCall(PetscSFViewFromOptions(centroidsf, NULL, "-dm_plex_box_label_centroid_sf_view"));
1185       PetscCall(PetscFree2(donor_centroids, periodic_centroids));
1186     }
1187 
1188     { // Create Isoperiodic SF using centroidsSF
1189       PetscInt           pStart, pEnd;
1190       PetscInt          *leaf_faces;
1191       const PetscSFNode *firemote;
1192       PetscSFNode       *isoperiodic_leaves;
1193 
1194       PetscCall(PetscMalloc1(num_periodic, &leaf_faces));
1195       PetscCall(PetscSFBcastBegin(centroidsf, MPIU_INT, donor_faces, leaf_faces, MPI_REPLACE));
1196       PetscCall(PetscSFBcastEnd(centroidsf, MPIU_INT, donor_faces, leaf_faces, MPI_REPLACE));
1197 
1198       PetscCall(PetscMalloc1(num_periodic, &isoperiodic_leaves));
1199       PetscCall(PetscSFGetGraph(centroidsf, NULL, NULL, NULL, &firemote));
1200       for (PetscInt l = 0; l < num_periodic; ++l) {
1201         isoperiodic_leaves[l].index = leaf_faces[l];
1202         isoperiodic_leaves[l].rank  = firemote[l].rank;
1203       }
1204 
1205       PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
1206       PetscCall(PetscSFCreate(PetscObjectComm((PetscObject)dm), &periodicsfs[periodic_sf_index]));
1207       PetscCall(PetscSFSetGraph(periodicsfs[periodic_sf_index], pEnd - pStart, num_periodic, (PetscInt *)periodic_faces, PETSC_COPY_VALUES, isoperiodic_leaves, PETSC_OWN_POINTER));
1208       PetscCall(PetscSFViewFromOptions(periodicsfs[periodic_sf_index], NULL, "-dm_plex_box_label_periodic_sf_view"));
1209       PetscCall(PetscFree(leaf_faces));
1210     }
1211 
1212     transform[periodic_sf_index][0][0] = 1;
1213     transform[periodic_sf_index][1][1] = 1;
1214     transform[periodic_sf_index][2][2] = 1;
1215     transform[periodic_sf_index][3][3] = 1;
1216     transform[periodic_sf_index][d][3] = donor_to_periodic_distance;
1217 
1218     periodic_sf_index++;
1219     PetscCall(PetscSFDestroy(&centroidsf));
1220     if (donor_is) {
1221       PetscCall(ISRestoreIndices(donor_is, &donor_faces));
1222       PetscCall(ISDestroy(&donor_is));
1223     }
1224     if (periodic_is) {
1225       PetscCall(ISRestoreIndices(periodic_is, &periodic_faces));
1226       PetscCall(ISDestroy(&periodic_is));
1227     }
1228     PetscCall(DMClearLabelStratum(dm, "Face Sets", face_pairings[dim - 2][d][0]));
1229     PetscCall(DMClearLabelStratum(dm, "Face Sets", face_pairings[dim - 2][d][1]));
1230   }
1231   PetscCall(DMPlexSetIsoperiodicFaceSF(dm, periodic_sf_index, periodicsfs));
1232   PetscCall(DMPlexSetIsoperiodicFaceTransform(dm, periodic_sf_index, (const PetscScalar *)transform));
1233   for (PetscInt p = 0; p < periodic_sf_index; p++) PetscCall(PetscSFDestroy(&periodicsfs[p]));
1234 
1235   { // Update coordinate DM with new Face Sets label
1236     DM      cdm;
1237     DMLabel oldFaceSets, newFaceSets;
1238     PetscCall(DMGetCoordinateDM(dm, &cdm));
1239     PetscCall(DMGetLabel(cdm, "Face Sets", &oldFaceSets));
1240     if (oldFaceSets) PetscCall(DMRemoveLabelBySelf(cdm, &oldFaceSets, PETSC_FALSE));
1241     PetscCall(DMLabelDuplicate(label, &newFaceSets));
1242     PetscCall(DMAddLabel(cdm, newFaceSets));
1243     PetscCall(DMLabelDestroy(&newFaceSets));
1244   }
1245   PetscFunctionReturn(PETSC_SUCCESS);
1246 }
1247 
1248 static PetscErrorCode DMPlexCreateSquareMesh_Simplex_CrissCross(DM dm, const PetscInt edges[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType bd[])
1249 {
1250   PetscInt       markerTop = 1, faceMarkerTop = 3;
1251   PetscInt       markerBottom = 1, faceMarkerBottom = 1;
1252   PetscInt       markerRight = 1, faceMarkerRight = 2;
1253   PetscInt       markerLeft = 1, faceMarkerLeft = 4;
1254   PetscBool      markerSeparate = PETSC_FALSE;
1255   DMBoundaryType bdX = bd[0], bdY = bd[1];
1256   PetscMPIInt    rank;
1257 
1258   PetscFunctionBegin;
1259   PetscCheck(bdX == DM_BOUNDARY_NONE || bdX == DM_BOUNDARY_PERIODIC, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Not implemented for boundary type %s", DMBoundaryTypes[bdX]);
1260   PetscCheck(bdY == DM_BOUNDARY_NONE || bdY == DM_BOUNDARY_PERIODIC, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Not implemented for boundary type %s", DMBoundaryTypes[bdY]);
1261   PetscCall(DMSetDimension(dm, 2));
1262   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
1263   PetscCall(DMCreateLabel(dm, "marker"));
1264   PetscCall(DMCreateLabel(dm, "Face Sets"));
1265   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
1266   if (markerSeparate) {
1267     markerBottom = faceMarkerBottom;
1268     markerTop    = faceMarkerTop;
1269     markerRight  = faceMarkerRight;
1270     markerLeft   = faceMarkerLeft;
1271   }
1272   {
1273     const PetscInt numXEdges    = rank == 0 ? edges[0] : 0;
1274     const PetscInt numYEdges    = rank == 0 ? edges[1] : 0;
1275     const PetscInt numZEdges    = rank == 0 ? 4 * edges[0] * edges[1] : 0; /* Z-edges are the 4 internal edges per cell */
1276     const PetscInt numXVertices = rank == 0 ? (bdX == DM_BOUNDARY_PERIODIC ? edges[0] : edges[0] + 1) : 0;
1277     const PetscInt numYVertices = rank == 0 ? (bdY == DM_BOUNDARY_PERIODIC ? edges[1] : edges[1] + 1) : 0;
1278     const PetscInt numZVertices = rank == 0 ? edges[0] * edges[1] : 0;
1279     const PetscInt numCells     = 4 * numXEdges * numYEdges;
1280     const PetscInt numTotXEdges = numXEdges * numYVertices;
1281     const PetscInt numTotYEdges = numYEdges * numXVertices;
1282     const PetscInt numVertices  = numXVertices * numYVertices + numZVertices;
1283     const PetscInt numEdges     = numTotXEdges + numTotYEdges + numZEdges;
1284     const PetscInt firstVertex  = numCells;
1285     const PetscInt firstXEdge   = numCells + numVertices;
1286     const PetscInt firstYEdge   = firstXEdge + numTotXEdges;
1287     const PetscInt firstZEdge   = firstYEdge + numTotYEdges;
1288     Vec            coordinates;
1289     PetscSection   coordSection;
1290     PetscScalar   *coords;
1291     PetscInt       coordSize;
1292     PetscInt       v, vx, vy;
1293     PetscInt       c, e, ex, ey;
1294 
1295     PetscCall(DMPlexSetChart(dm, 0, numCells + numEdges + numVertices));
1296     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 3));
1297     for (e = firstXEdge; e < firstXEdge + numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
1298     PetscCall(DMSetUp(dm));
1299 
1300     /* Build cells and Z-edges */
1301     for (ey = 0; ey < numYEdges; ++ey) {
1302       for (ex = 0; ex < numXEdges; ++ex) {
1303         const PetscInt exp1 = (ex + 1) % numXVertices;
1304         const PetscInt eyp1 = (ey + 1) % numYVertices;
1305         const PetscInt ez   = firstZEdge + 4 * (ey * numXEdges + ex);
1306         const PetscInt vc   = firstVertex + numXVertices * numYVertices + ey * numXEdges + ex;
1307         const PetscInt v0   = firstVertex + ey * numXVertices + ex;
1308         const PetscInt v1   = firstVertex + ey * numXVertices + exp1;
1309         const PetscInt v2   = firstVertex + eyp1 * numXVertices + exp1;
1310         const PetscInt v3   = firstVertex + eyp1 * numXVertices + ex;
1311         const PetscInt e0   = firstXEdge + ey * numXEdges + ex;
1312         const PetscInt e1   = firstYEdge + exp1 * numYEdges + ey;
1313         const PetscInt e2   = firstXEdge + eyp1 * numXEdges + ex;
1314         const PetscInt e3   = firstYEdge + ex * numYEdges + ey;
1315 
1316         const PetscInt cones[] = {ez, e0, ez + 1, ez + 1, e1, ez + 2, ez + 2, e2, ez + 3, ez + 3, e3, ez};
1317         const PetscInt ornts[] = {-1, 0, 0, -1, 0, 0, -1, -1, 0, -1, -1, 0};
1318         const PetscInt verts[] = {v0, vc, v1, vc, v2, vc, v3, vc};
1319 
1320         for (c = 0; c < 4; c++) {
1321           PetscInt cell = 4 * (ey * numXEdges + ex) + c;
1322           PetscInt edge = ez + c;
1323 
1324           PetscCall(DMPlexSetCone(dm, cell, cones + 3 * c));
1325           PetscCall(DMPlexSetConeOrientation(dm, cell, ornts + 3 * c));
1326           PetscCall(DMPlexSetCone(dm, edge, verts + 2 * c));
1327         }
1328       }
1329     }
1330 
1331     /* Build Y edges*/
1332     for (vx = 0; vx < numXVertices; vx++) {
1333       for (ey = 0; ey < numYEdges; ey++) {
1334         const PetscInt edge   = firstYEdge + vx * numYEdges + ey;
1335         const PetscInt v0     = firstVertex + ey * numXVertices + vx;
1336         const PetscInt v1     = firstVertex + ((ey + 1) % numYVertices) * numXVertices + vx;
1337         const PetscInt cone[] = {v0, v1};
1338 
1339         PetscCall(DMPlexSetCone(dm, edge, cone));
1340         if ((bdX != DM_BOUNDARY_PERIODIC) && (bdX != DM_BOUNDARY_TWIST)) {
1341           if (vx == numXVertices - 1) {
1342             PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerRight));
1343             PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1344             PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1345             if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1346           } else if (vx == 0) {
1347             PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerLeft));
1348             PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1349             PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1350             if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1351           }
1352         }
1353       }
1354     }
1355 
1356     /* Build X edges*/
1357     for (vy = 0; vy < numYVertices; vy++) {
1358       for (ex = 0; ex < numXEdges; ex++) {
1359         const PetscInt edge   = firstXEdge + vy * numXEdges + ex;
1360         const PetscInt v0     = firstVertex + vy * numXVertices + ex;
1361         const PetscInt v1     = firstVertex + vy * numXVertices + (ex + 1) % numXVertices;
1362         const PetscInt cone[] = {v0, v1};
1363 
1364         PetscCall(DMPlexSetCone(dm, edge, cone));
1365         if ((bdY != DM_BOUNDARY_PERIODIC) && (bdY != DM_BOUNDARY_TWIST)) {
1366           if (vy == numYVertices - 1) {
1367             PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerTop));
1368             PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1369             PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1370             if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1371           } else if (vy == 0) {
1372             PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerBottom));
1373             PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1374             PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1375             if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1376           }
1377         }
1378       }
1379     }
1380 
1381     /* Compute support, stratify, and celltype label */
1382     PetscCall(DMPlexSymmetrize(dm));
1383     PetscCall(DMPlexStratify(dm));
1384     PetscCall(DMPlexComputeCellTypes(dm));
1385 
1386     /* Build coordinates */
1387     PetscCall(DMGetCoordinateSection(dm, &coordSection));
1388     PetscCall(PetscSectionSetNumFields(coordSection, 1));
1389     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, 2));
1390     PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numVertices));
1391     for (v = firstVertex; v < firstVertex + numVertices; ++v) {
1392       PetscCall(PetscSectionSetDof(coordSection, v, 2));
1393       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, 2));
1394     }
1395     PetscCall(PetscSectionSetUp(coordSection));
1396     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
1397     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
1398     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
1399     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
1400     PetscCall(VecSetBlockSize(coordinates, 2));
1401     PetscCall(VecSetType(coordinates, VECSTANDARD));
1402     PetscCall(VecGetArray(coordinates, &coords));
1403     for (vy = 0; vy < numYVertices; ++vy) {
1404       for (vx = 0; vx < numXVertices; ++vx) {
1405         coords[2 * (vy * numXVertices + vx) + 0] = lower[0] + ((upper[0] - lower[0]) / numXEdges) * vx;
1406         coords[2 * (vy * numXVertices + vx) + 1] = lower[1] + ((upper[1] - lower[1]) / numYEdges) * vy;
1407       }
1408     }
1409     for (ey = 0; ey < numYEdges; ++ey) {
1410       for (ex = 0; ex < numXEdges; ++ex) {
1411         const PetscInt c = ey * numXEdges + ex + numYVertices * numXVertices;
1412 
1413         coords[2 * c + 0] = lower[0] + ((upper[0] - lower[0]) / numXEdges) * (ex + 0.5);
1414         coords[2 * c + 1] = lower[1] + ((upper[1] - lower[1]) / numYEdges) * (ey + 0.5);
1415       }
1416     }
1417     PetscCall(VecRestoreArray(coordinates, &coords));
1418     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
1419     PetscCall(VecDestroy(&coordinates));
1420 
1421     /* handle periodic BC */
1422     if (bdX == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_PERIODIC) {
1423       PetscReal L[2]       = {-1., -1.};
1424       PetscReal maxCell[2] = {-1., -1.};
1425 
1426       for (PetscInt d = 0; d < 2; ++d) {
1427         if (bd[d] != DM_BOUNDARY_NONE) {
1428           L[d]       = upper[d] - lower[d];
1429           maxCell[d] = 1.1 * (L[d] / PetscMax(1, edges[d]));
1430         }
1431       }
1432       PetscCall(DMSetPeriodicity(dm, maxCell, lower, L));
1433     }
1434   }
1435   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
1436   PetscFunctionReturn(PETSC_SUCCESS);
1437 }
1438 
1439 static PetscErrorCode DMPlexCreateBoxMesh_Simplex_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate)
1440 {
1441   DM        boundary, vol;
1442   DMLabel   bdlabel;
1443   PetscBool crisscross = PETSC_FALSE;
1444 
1445   PetscFunctionBegin;
1446   PetscAssertPointer(dm, 1);
1447   if (dim == 2) PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_box_crisscross", &crisscross, NULL));
1448   if (crisscross) {
1449     PetscCall(DMPlexCreateSquareMesh_Simplex_CrissCross(dm, faces, lower, upper, periodicity));
1450   } else {
1451     for (PetscInt i = 0; i < dim; ++i) PetscCheck(periodicity[i] == DM_BOUNDARY_NONE, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Periodicity is not supported for simplex meshes");
1452     PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &boundary));
1453     PetscCall(DMSetType(boundary, DMPLEX));
1454     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)boundary, ((PetscObject)dm)->prefix));
1455     PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(boundary, dim, faces, lower, upper, PETSC_FALSE));
1456     PetscCall(DMPlexGenerate(boundary, NULL, interpolate, &vol));
1457     PetscCall(DMGetLabel(vol, "marker", &bdlabel));
1458     if (bdlabel) PetscCall(DMPlexLabelComplete(vol, bdlabel));
1459     PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_FALSE, vol));
1460     PetscCall(DMPlexReplace_Internal(dm, &vol));
1461     PetscCall(DMDestroy(&boundary));
1462   }
1463   if (interpolate) {
1464     PetscCall(DMPlexInterpolateInPlace_Internal(dm));
1465     PetscCall(DMPlexSetBoxLabel_Internal(dm, periodicity));
1466   }
1467   PetscFunctionReturn(PETSC_SUCCESS);
1468 }
1469 
1470 static PetscErrorCode DMPlexCreateCubeMesh_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[], DMBoundaryType bdX, DMBoundaryType bdY, DMBoundaryType bdZ)
1471 {
1472   DMLabel     cutLabel  = NULL;
1473   PetscInt    markerTop = 1, faceMarkerTop = 1;
1474   PetscInt    markerBottom = 1, faceMarkerBottom = 1;
1475   PetscInt    markerFront = 1, faceMarkerFront = 1;
1476   PetscInt    markerBack = 1, faceMarkerBack = 1;
1477   PetscInt    markerRight = 1, faceMarkerRight = 1;
1478   PetscInt    markerLeft = 1, faceMarkerLeft = 1;
1479   PetscInt    dim;
1480   PetscBool   markerSeparate = PETSC_FALSE, cutMarker = PETSC_FALSE;
1481   PetscMPIInt rank;
1482 
1483   PetscFunctionBegin;
1484   PetscCall(DMGetDimension(dm, &dim));
1485   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
1486   PetscCall(DMCreateLabel(dm, "marker"));
1487   PetscCall(DMCreateLabel(dm, "Face Sets"));
1488   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_periodic_cut", &cutMarker, NULL));
1489   if (bdX == DM_BOUNDARY_PERIODIC || bdX == DM_BOUNDARY_TWIST || bdY == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_TWIST || bdZ == DM_BOUNDARY_PERIODIC || bdZ == DM_BOUNDARY_TWIST) {
1490     if (cutMarker) {
1491       PetscCall(DMCreateLabel(dm, "periodic_cut"));
1492       PetscCall(DMGetLabel(dm, "periodic_cut", &cutLabel));
1493     }
1494   }
1495   switch (dim) {
1496   case 2:
1497     faceMarkerTop    = 3;
1498     faceMarkerBottom = 1;
1499     faceMarkerRight  = 2;
1500     faceMarkerLeft   = 4;
1501     break;
1502   case 3:
1503     faceMarkerBottom = 1;
1504     faceMarkerTop    = 2;
1505     faceMarkerFront  = 3;
1506     faceMarkerBack   = 4;
1507     faceMarkerRight  = 5;
1508     faceMarkerLeft   = 6;
1509     break;
1510   default:
1511     SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Dimension %" PetscInt_FMT " not supported", dim);
1512   }
1513   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
1514   if (markerSeparate) {
1515     markerBottom = faceMarkerBottom;
1516     markerTop    = faceMarkerTop;
1517     markerFront  = faceMarkerFront;
1518     markerBack   = faceMarkerBack;
1519     markerRight  = faceMarkerRight;
1520     markerLeft   = faceMarkerLeft;
1521   }
1522   {
1523     const PetscInt numXEdges    = rank == 0 ? edges[0] : 0;
1524     const PetscInt numYEdges    = rank == 0 ? edges[1] : 0;
1525     const PetscInt numZEdges    = rank == 0 ? edges[2] : 0;
1526     const PetscInt numXVertices = rank == 0 ? (bdX == DM_BOUNDARY_PERIODIC || bdX == DM_BOUNDARY_TWIST ? edges[0] : edges[0] + 1) : 0;
1527     const PetscInt numYVertices = rank == 0 ? (bdY == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_TWIST ? edges[1] : edges[1] + 1) : 0;
1528     const PetscInt numZVertices = rank == 0 ? (bdZ == DM_BOUNDARY_PERIODIC || bdZ == DM_BOUNDARY_TWIST ? edges[2] : edges[2] + 1) : 0;
1529     const PetscInt numCells     = numXEdges * numYEdges * numZEdges;
1530     const PetscInt numXFaces    = numYEdges * numZEdges;
1531     const PetscInt numYFaces    = numXEdges * numZEdges;
1532     const PetscInt numZFaces    = numXEdges * numYEdges;
1533     const PetscInt numTotXFaces = numXVertices * numXFaces;
1534     const PetscInt numTotYFaces = numYVertices * numYFaces;
1535     const PetscInt numTotZFaces = numZVertices * numZFaces;
1536     const PetscInt numFaces     = numTotXFaces + numTotYFaces + numTotZFaces;
1537     const PetscInt numTotXEdges = numXEdges * numYVertices * numZVertices;
1538     const PetscInt numTotYEdges = numYEdges * numXVertices * numZVertices;
1539     const PetscInt numTotZEdges = numZEdges * numXVertices * numYVertices;
1540     const PetscInt numVertices  = numXVertices * numYVertices * numZVertices;
1541     const PetscInt numEdges     = numTotXEdges + numTotYEdges + numTotZEdges;
1542     const PetscInt firstVertex  = (dim == 2) ? numFaces : numCells;
1543     const PetscInt firstXFace   = (dim == 2) ? 0 : numCells + numVertices;
1544     const PetscInt firstYFace   = firstXFace + numTotXFaces;
1545     const PetscInt firstZFace   = firstYFace + numTotYFaces;
1546     const PetscInt firstXEdge   = numCells + numFaces + numVertices;
1547     const PetscInt firstYEdge   = firstXEdge + numTotXEdges;
1548     const PetscInt firstZEdge   = firstYEdge + numTotYEdges;
1549     Vec            coordinates;
1550     PetscSection   coordSection;
1551     PetscScalar   *coords;
1552     PetscInt       coordSize;
1553     PetscInt       v, vx, vy, vz;
1554     PetscInt       c, f, fx, fy, fz, e, ex, ey, ez;
1555 
1556     PetscCall(DMPlexSetChart(dm, 0, numCells + numFaces + numEdges + numVertices));
1557     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 6));
1558     for (f = firstXFace; f < firstXFace + numFaces; ++f) PetscCall(DMPlexSetConeSize(dm, f, 4));
1559     for (e = firstXEdge; e < firstXEdge + numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
1560     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
1561     /* Build cells */
1562     for (fz = 0; fz < numZEdges; ++fz) {
1563       for (fy = 0; fy < numYEdges; ++fy) {
1564         for (fx = 0; fx < numXEdges; ++fx) {
1565           PetscInt cell  = (fz * numYEdges + fy) * numXEdges + fx;
1566           PetscInt faceB = firstZFace + (fy * numXEdges + fx) * numZVertices + fz;
1567           PetscInt faceT = firstZFace + (fy * numXEdges + fx) * numZVertices + ((fz + 1) % numZVertices);
1568           PetscInt faceF = firstYFace + (fz * numXEdges + fx) * numYVertices + fy;
1569           PetscInt faceK = firstYFace + (fz * numXEdges + fx) * numYVertices + ((fy + 1) % numYVertices);
1570           PetscInt faceL = firstXFace + (fz * numYEdges + fy) * numXVertices + fx;
1571           PetscInt faceR = firstXFace + (fz * numYEdges + fy) * numXVertices + ((fx + 1) % numXVertices);
1572           /* B,  T,  F,  K,  R,  L */
1573           PetscInt ornt[6] = {-2, 0, 0, -3, 0, -2}; /* ??? */
1574           PetscInt cone[6];
1575 
1576           /* no boundary twisting in 3D */
1577           cone[0] = faceB;
1578           cone[1] = faceT;
1579           cone[2] = faceF;
1580           cone[3] = faceK;
1581           cone[4] = faceR;
1582           cone[5] = faceL;
1583           PetscCall(DMPlexSetCone(dm, cell, cone));
1584           PetscCall(DMPlexSetConeOrientation(dm, cell, ornt));
1585           if (bdX != DM_BOUNDARY_NONE && fx == numXEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
1586           if (bdY != DM_BOUNDARY_NONE && fy == numYEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
1587           if (bdZ != DM_BOUNDARY_NONE && fz == numZEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
1588         }
1589       }
1590     }
1591     /* Build x faces */
1592     for (fz = 0; fz < numZEdges; ++fz) {
1593       for (fy = 0; fy < numYEdges; ++fy) {
1594         for (fx = 0; fx < numXVertices; ++fx) {
1595           PetscInt face    = firstXFace + (fz * numYEdges + fy) * numXVertices + fx;
1596           PetscInt edgeL   = firstZEdge + (fy * numXVertices + fx) * numZEdges + fz;
1597           PetscInt edgeR   = firstZEdge + (((fy + 1) % numYVertices) * numXVertices + fx) * numZEdges + fz;
1598           PetscInt edgeB   = firstYEdge + (fz * numXVertices + fx) * numYEdges + fy;
1599           PetscInt edgeT   = firstYEdge + (((fz + 1) % numZVertices) * numXVertices + fx) * numYEdges + fy;
1600           PetscInt ornt[4] = {0, 0, -1, -1};
1601           PetscInt cone[4];
1602 
1603           if (dim == 3) {
1604             /* markers */
1605             if (bdX != DM_BOUNDARY_PERIODIC) {
1606               if (fx == numXVertices - 1) {
1607                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerRight));
1608                 PetscCall(DMSetLabelValue(dm, "marker", face, markerRight));
1609               } else if (fx == 0) {
1610                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerLeft));
1611                 PetscCall(DMSetLabelValue(dm, "marker", face, markerLeft));
1612               }
1613             }
1614           }
1615           cone[0] = edgeB;
1616           cone[1] = edgeR;
1617           cone[2] = edgeT;
1618           cone[3] = edgeL;
1619           PetscCall(DMPlexSetCone(dm, face, cone));
1620           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
1621         }
1622       }
1623     }
1624     /* Build y faces */
1625     for (fz = 0; fz < numZEdges; ++fz) {
1626       for (fx = 0; fx < numXEdges; ++fx) {
1627         for (fy = 0; fy < numYVertices; ++fy) {
1628           PetscInt face    = firstYFace + (fz * numXEdges + fx) * numYVertices + fy;
1629           PetscInt edgeL   = firstZEdge + (fy * numXVertices + fx) * numZEdges + fz;
1630           PetscInt edgeR   = firstZEdge + (fy * numXVertices + ((fx + 1) % numXVertices)) * numZEdges + fz;
1631           PetscInt edgeB   = firstXEdge + (fz * numYVertices + fy) * numXEdges + fx;
1632           PetscInt edgeT   = firstXEdge + (((fz + 1) % numZVertices) * numYVertices + fy) * numXEdges + fx;
1633           PetscInt ornt[4] = {0, 0, -1, -1};
1634           PetscInt cone[4];
1635 
1636           if (dim == 3) {
1637             /* markers */
1638             if (bdY != DM_BOUNDARY_PERIODIC) {
1639               if (fy == numYVertices - 1) {
1640                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerBack));
1641                 PetscCall(DMSetLabelValue(dm, "marker", face, markerBack));
1642               } else if (fy == 0) {
1643                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerFront));
1644                 PetscCall(DMSetLabelValue(dm, "marker", face, markerFront));
1645               }
1646             }
1647           }
1648           cone[0] = edgeB;
1649           cone[1] = edgeR;
1650           cone[2] = edgeT;
1651           cone[3] = edgeL;
1652           PetscCall(DMPlexSetCone(dm, face, cone));
1653           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
1654         }
1655       }
1656     }
1657     /* Build z faces */
1658     for (fy = 0; fy < numYEdges; ++fy) {
1659       for (fx = 0; fx < numXEdges; ++fx) {
1660         for (fz = 0; fz < numZVertices; fz++) {
1661           PetscInt face    = firstZFace + (fy * numXEdges + fx) * numZVertices + fz;
1662           PetscInt edgeL   = firstYEdge + (fz * numXVertices + fx) * numYEdges + fy;
1663           PetscInt edgeR   = firstYEdge + (fz * numXVertices + ((fx + 1) % numXVertices)) * numYEdges + fy;
1664           PetscInt edgeB   = firstXEdge + (fz * numYVertices + fy) * numXEdges + fx;
1665           PetscInt edgeT   = firstXEdge + (fz * numYVertices + ((fy + 1) % numYVertices)) * numXEdges + fx;
1666           PetscInt ornt[4] = {0, 0, -1, -1};
1667           PetscInt cone[4];
1668 
1669           if (dim == 2) {
1670             if (bdX == DM_BOUNDARY_TWIST && fx == numXEdges - 1) {
1671               edgeR += numYEdges - 1 - 2 * fy;
1672               ornt[1] = -1;
1673             }
1674             if (bdY == DM_BOUNDARY_TWIST && fy == numYEdges - 1) {
1675               edgeT += numXEdges - 1 - 2 * fx;
1676               ornt[2] = 0;
1677             }
1678             if (bdX != DM_BOUNDARY_NONE && fx == numXEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, face, 2));
1679             if (bdY != DM_BOUNDARY_NONE && fy == numYEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, face, 2));
1680           } else {
1681             /* markers */
1682             if (bdZ != DM_BOUNDARY_PERIODIC) {
1683               if (fz == numZVertices - 1) {
1684                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerTop));
1685                 PetscCall(DMSetLabelValue(dm, "marker", face, markerTop));
1686               } else if (fz == 0) {
1687                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerBottom));
1688                 PetscCall(DMSetLabelValue(dm, "marker", face, markerBottom));
1689               }
1690             }
1691           }
1692           cone[0] = edgeB;
1693           cone[1] = edgeR;
1694           cone[2] = edgeT;
1695           cone[3] = edgeL;
1696           PetscCall(DMPlexSetCone(dm, face, cone));
1697           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
1698         }
1699       }
1700     }
1701     /* Build Z edges*/
1702     for (vy = 0; vy < numYVertices; vy++) {
1703       for (vx = 0; vx < numXVertices; vx++) {
1704         for (ez = 0; ez < numZEdges; ez++) {
1705           const PetscInt edge    = firstZEdge + (vy * numXVertices + vx) * numZEdges + ez;
1706           const PetscInt vertexB = firstVertex + (ez * numYVertices + vy) * numXVertices + vx;
1707           const PetscInt vertexT = firstVertex + (((ez + 1) % numZVertices) * numYVertices + vy) * numXVertices + vx;
1708           PetscInt       cone[2];
1709 
1710           cone[0] = vertexB;
1711           cone[1] = vertexT;
1712           PetscCall(DMPlexSetCone(dm, edge, cone));
1713           if (dim == 3) {
1714             if (bdX != DM_BOUNDARY_PERIODIC) {
1715               if (vx == numXVertices - 1) {
1716                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1717                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1718                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1719               } else if (vx == 0) {
1720                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1721                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1722                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1723               }
1724             }
1725             if (bdY != DM_BOUNDARY_PERIODIC) {
1726               if (vy == numYVertices - 1) {
1727                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBack));
1728                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBack));
1729                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBack));
1730               } else if (vy == 0) {
1731                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerFront));
1732                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerFront));
1733                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerFront));
1734               }
1735             }
1736           }
1737         }
1738       }
1739     }
1740     /* Build Y edges*/
1741     for (vz = 0; vz < numZVertices; vz++) {
1742       for (vx = 0; vx < numXVertices; vx++) {
1743         for (ey = 0; ey < numYEdges; ey++) {
1744           const PetscInt nextv   = (dim == 2 && bdY == DM_BOUNDARY_TWIST && ey == numYEdges - 1) ? (numXVertices - vx - 1) : (vz * numYVertices + ((ey + 1) % numYVertices)) * numXVertices + vx;
1745           const PetscInt edge    = firstYEdge + (vz * numXVertices + vx) * numYEdges + ey;
1746           const PetscInt vertexF = firstVertex + (vz * numYVertices + ey) * numXVertices + vx;
1747           const PetscInt vertexK = firstVertex + nextv;
1748           PetscInt       cone[2];
1749 
1750           cone[0] = vertexF;
1751           cone[1] = vertexK;
1752           PetscCall(DMPlexSetCone(dm, edge, cone));
1753           if (dim == 2) {
1754             if ((bdX != DM_BOUNDARY_PERIODIC) && (bdX != DM_BOUNDARY_TWIST)) {
1755               if (vx == numXVertices - 1) {
1756                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerRight));
1757                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1758                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1759                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1760               } else if (vx == 0) {
1761                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerLeft));
1762                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1763                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1764                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1765               }
1766             } else {
1767               if (vx == 0 && cutLabel) {
1768                 PetscCall(DMLabelSetValue(cutLabel, edge, 1));
1769                 PetscCall(DMLabelSetValue(cutLabel, cone[0], 1));
1770                 if (ey == numYEdges - 1) PetscCall(DMLabelSetValue(cutLabel, cone[1], 1));
1771               }
1772             }
1773           } else {
1774             if (bdX != DM_BOUNDARY_PERIODIC) {
1775               if (vx == numXVertices - 1) {
1776                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1777                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1778                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1779               } else if (vx == 0) {
1780                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1781                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1782                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1783               }
1784             }
1785             if (bdZ != DM_BOUNDARY_PERIODIC) {
1786               if (vz == numZVertices - 1) {
1787                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1788                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1789                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1790               } else if (vz == 0) {
1791                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1792                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1793                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1794               }
1795             }
1796           }
1797         }
1798       }
1799     }
1800     /* Build X edges*/
1801     for (vz = 0; vz < numZVertices; vz++) {
1802       for (vy = 0; vy < numYVertices; vy++) {
1803         for (ex = 0; ex < numXEdges; ex++) {
1804           const PetscInt nextv   = (dim == 2 && bdX == DM_BOUNDARY_TWIST && ex == numXEdges - 1) ? (numYVertices - vy - 1) * numXVertices : (vz * numYVertices + vy) * numXVertices + (ex + 1) % numXVertices;
1805           const PetscInt edge    = firstXEdge + (vz * numYVertices + vy) * numXEdges + ex;
1806           const PetscInt vertexL = firstVertex + (vz * numYVertices + vy) * numXVertices + ex;
1807           const PetscInt vertexR = firstVertex + nextv;
1808           PetscInt       cone[2];
1809 
1810           cone[0] = vertexL;
1811           cone[1] = vertexR;
1812           PetscCall(DMPlexSetCone(dm, edge, cone));
1813           if (dim == 2) {
1814             if ((bdY != DM_BOUNDARY_PERIODIC) && (bdY != DM_BOUNDARY_TWIST)) {
1815               if (vy == numYVertices - 1) {
1816                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerTop));
1817                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1818                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1819                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1820               } else if (vy == 0) {
1821                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerBottom));
1822                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1823                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1824                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1825               }
1826             } else {
1827               if (vy == 0 && cutLabel) {
1828                 PetscCall(DMLabelSetValue(cutLabel, edge, 1));
1829                 PetscCall(DMLabelSetValue(cutLabel, cone[0], 1));
1830                 if (ex == numXEdges - 1) PetscCall(DMLabelSetValue(cutLabel, cone[1], 1));
1831               }
1832             }
1833           } else {
1834             if (bdY != DM_BOUNDARY_PERIODIC) {
1835               if (vy == numYVertices - 1) {
1836                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBack));
1837                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBack));
1838                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBack));
1839               } else if (vy == 0) {
1840                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerFront));
1841                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerFront));
1842                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerFront));
1843               }
1844             }
1845             if (bdZ != DM_BOUNDARY_PERIODIC) {
1846               if (vz == numZVertices - 1) {
1847                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1848                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1849                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1850               } else if (vz == 0) {
1851                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1852                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1853                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1854               }
1855             }
1856           }
1857         }
1858       }
1859     }
1860     PetscCall(DMPlexSymmetrize(dm));
1861     PetscCall(DMPlexStratify(dm));
1862     /* Build coordinates */
1863     PetscCall(DMGetCoordinateSection(dm, &coordSection));
1864     PetscCall(PetscSectionSetNumFields(coordSection, 1));
1865     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
1866     PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numVertices));
1867     for (v = firstVertex; v < firstVertex + numVertices; ++v) {
1868       PetscCall(PetscSectionSetDof(coordSection, v, dim));
1869       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
1870     }
1871     PetscCall(PetscSectionSetUp(coordSection));
1872     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
1873     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
1874     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
1875     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
1876     PetscCall(VecSetBlockSize(coordinates, dim));
1877     PetscCall(VecSetType(coordinates, VECSTANDARD));
1878     PetscCall(VecGetArray(coordinates, &coords));
1879     for (vz = 0; vz < numZVertices; ++vz) {
1880       for (vy = 0; vy < numYVertices; ++vy) {
1881         for (vx = 0; vx < numXVertices; ++vx) {
1882           coords[((vz * numYVertices + vy) * numXVertices + vx) * dim + 0] = lower[0] + ((upper[0] - lower[0]) / numXEdges) * vx;
1883           coords[((vz * numYVertices + vy) * numXVertices + vx) * dim + 1] = lower[1] + ((upper[1] - lower[1]) / numYEdges) * vy;
1884           if (dim == 3) coords[((vz * numYVertices + vy) * numXVertices + vx) * dim + 2] = lower[2] + ((upper[2] - lower[2]) / numZEdges) * vz;
1885         }
1886       }
1887     }
1888     PetscCall(VecRestoreArray(coordinates, &coords));
1889     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
1890     PetscCall(VecDestroy(&coordinates));
1891   }
1892   PetscFunctionReturn(PETSC_SUCCESS);
1893 }
1894 
1895 static PetscErrorCode DMPlexCreateBoxMesh_Tensor_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[])
1896 {
1897   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1898   PetscInt       fac[3] = {0, 0, 0}, d;
1899 
1900   PetscFunctionBegin;
1901   PetscAssertPointer(dm, 1);
1902   PetscValidLogicalCollectiveInt(dm, dim, 2);
1903   PetscCall(DMSetDimension(dm, dim));
1904   for (d = 0; d < dim; ++d) {
1905     fac[d] = faces[d];
1906     bdt[d] = periodicity[d];
1907   }
1908   PetscCall(DMPlexCreateCubeMesh_Internal(dm, lower, upper, fac, bdt[0], bdt[1], bdt[2]));
1909   if (periodicity[0] == DM_BOUNDARY_PERIODIC || periodicity[0] == DM_BOUNDARY_TWIST || periodicity[1] == DM_BOUNDARY_PERIODIC || periodicity[1] == DM_BOUNDARY_TWIST || (dim > 2 && (periodicity[2] == DM_BOUNDARY_PERIODIC || periodicity[2] == DM_BOUNDARY_TWIST))) {
1910     PetscReal L[3]       = {-1., -1., 0.};
1911     PetscReal maxCell[3] = {-1., -1., 0.};
1912 
1913     for (d = 0; d < dim; ++d) {
1914       if (periodicity[d] != DM_BOUNDARY_NONE) {
1915         L[d]       = upper[d] - lower[d];
1916         maxCell[d] = 1.1 * (L[d] / PetscMax(1, faces[d]));
1917       }
1918     }
1919     PetscCall(DMSetPeriodicity(dm, maxCell, lower, L));
1920   }
1921   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
1922   PetscFunctionReturn(PETSC_SUCCESS);
1923 }
1924 
1925 static PetscErrorCode DMPlexCreateBoxMesh_Internal(DM dm, DMPlexShape shape, PetscInt dim, PetscBool simplex, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate)
1926 {
1927   PetscFunctionBegin;
1928   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
1929   if (shape == DM_SHAPE_ZBOX) PetscCall(DMPlexCreateBoxMesh_Tensor_SFC_Internal(dm, dim, faces, lower, upper, periodicity, interpolate));
1930   else if (dim == 1) PetscCall(DMPlexCreateLineMesh_Internal(dm, faces[0], lower[0], upper[0], periodicity[0]));
1931   else if (simplex) PetscCall(DMPlexCreateBoxMesh_Simplex_Internal(dm, dim, faces, lower, upper, periodicity, interpolate));
1932   else PetscCall(DMPlexCreateBoxMesh_Tensor_Internal(dm, dim, faces, lower, upper, periodicity));
1933   if (!interpolate && dim > 1 && !simplex) {
1934     DM udm;
1935 
1936     PetscCall(DMPlexUninterpolate(dm, &udm));
1937     PetscCall(DMPlexCopyCoordinates(dm, udm));
1938     PetscCall(DMPlexReplace_Internal(dm, &udm));
1939   }
1940   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
1941   PetscFunctionReturn(PETSC_SUCCESS);
1942 }
1943 
1944 /*@
1945   DMPlexCreateBoxMesh - Creates a mesh on the tensor product of unit intervals (box) using simplices or tensor cells (hexahedra).
1946 
1947   Collective
1948 
1949   Input Parameters:
1950 + comm               - The communicator for the `DM` object
1951 . dim                - The spatial dimension
1952 . simplex            - `PETSC_TRUE` for simplices, `PETSC_FALSE` for tensor cells
1953 . faces              - Number of faces per dimension, or `NULL` for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
1954 . lower              - The lower left corner, or `NULL` for (0, 0, 0)
1955 . upper              - The upper right corner, or `NULL` for (1, 1, 1)
1956 . periodicity        - The boundary type for the X,Y,Z direction, or `NULL` for `DM_BOUNDARY_NONE`
1957 . interpolate        - Flag to create intermediate mesh pieces (edges, faces)
1958 . localizationHeight - Flag to localize edges and faces in addition to cells; only significant for periodic meshes
1959 - sparseLocalize     - Flag to localize coordinates only for cells near the periodic boundary; only significant for periodic meshes
1960 
1961   Output Parameter:
1962 . dm - The `DM` object
1963 
1964   Level: beginner
1965 
1966   Note:
1967   To customize this mesh using options, use
1968 .vb
1969   DMCreate(comm, &dm);
1970   DMSetType(dm, DMPLEX);
1971   DMSetFromOptions(dm);
1972 .ve
1973   and use the options in `DMSetFromOptions()`.
1974 
1975   Here is the numbering returned for 2 faces in each direction for tensor cells\:
1976 .vb
1977  10---17---11---18----12
1978   |         |         |
1979   |         |         |
1980  20    2   22    3    24
1981   |         |         |
1982   |         |         |
1983   7---15----8---16----9
1984   |         |         |
1985   |         |         |
1986  19    0   21    1   23
1987   |         |         |
1988   |         |         |
1989   4---13----5---14----6
1990 .ve
1991   and for simplicial cells
1992 .vb
1993  14----8---15----9----16
1994   |\     5  |\      7 |
1995   | \       | \       |
1996  13   2    14    3    15
1997   | 4   \   | 6   \   |
1998   |       \ |       \ |
1999  11----6---12----7----13
2000   |\        |\        |
2001   | \    1  | \     3 |
2002  10   0    11    1    12
2003   | 0   \   | 2   \   |
2004   |       \ |       \ |
2005   8----4----9----5----10
2006 .ve
2007 
2008 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMSetFromOptions()`, `DMPlexCreateFromFile()`, `DMPlexCreateHexCylinderMesh()`, `DMSetType()`, `DMCreate()`
2009 @*/
2010 PetscErrorCode DMPlexCreateBoxMesh(MPI_Comm comm, PetscInt dim, PetscBool simplex, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate, PetscInt localizationHeight, PetscBool sparseLocalize, DM *dm)
2011 {
2012   PetscInt       fac[3] = {1, 1, 1};
2013   PetscReal      low[3] = {0, 0, 0};
2014   PetscReal      upp[3] = {1, 1, 1};
2015   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
2016 
2017   PetscFunctionBegin;
2018   PetscCall(DMCreate(comm, dm));
2019   PetscCall(DMSetType(*dm, DMPLEX));
2020   PetscCall(DMPlexCreateBoxMesh_Internal(*dm, DM_SHAPE_BOX, dim, simplex, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, periodicity ? periodicity : bdt, interpolate));
2021   if (periodicity) {
2022     DM cdm;
2023 
2024     PetscCall(DMGetCoordinateDM(*dm, &cdm));
2025     PetscCall(DMPlexSetMaxProjectionHeight(cdm, localizationHeight));
2026     PetscCall(DMSetSparseLocalize(*dm, sparseLocalize));
2027     PetscCall(DMLocalizeCoordinates(*dm));
2028   }
2029   PetscFunctionReturn(PETSC_SUCCESS);
2030 }
2031 
2032 static PetscErrorCode DMPlexCreateWedgeBoxMesh_Internal(DM dm, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[])
2033 {
2034   DM       bdm, vol;
2035   PetscInt i;
2036 
2037   PetscFunctionBegin;
2038   // TODO Now we can support periodicity
2039   for (i = 0; i < 3; ++i) PetscCheck(periodicity[i] == DM_BOUNDARY_NONE, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Periodicity not yet supported");
2040   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &bdm));
2041   PetscCall(DMSetType(bdm, DMPLEX));
2042   PetscCall(DMSetDimension(bdm, 2));
2043   PetscCall(PetscLogEventBegin(DMPLEX_Generate, bdm, 0, 0, 0));
2044   PetscCall(DMPlexCreateBoxMesh_Simplex_Internal(bdm, 2, faces, lower, upper, periodicity, PETSC_TRUE));
2045   PetscCall(DMPlexExtrude(bdm, faces[2], upper[2] - lower[2], PETSC_TRUE, PETSC_FALSE, PETSC_FALSE, NULL, NULL, &vol));
2046   PetscCall(PetscLogEventEnd(DMPLEX_Generate, bdm, 0, 0, 0));
2047   PetscCall(DMDestroy(&bdm));
2048   PetscCall(DMPlexReplace_Internal(dm, &vol));
2049   if (lower[2] != 0.0) {
2050     Vec          v;
2051     PetscScalar *x;
2052     PetscInt     cDim, n;
2053 
2054     PetscCall(DMGetCoordinatesLocal(dm, &v));
2055     PetscCall(VecGetBlockSize(v, &cDim));
2056     PetscCall(VecGetLocalSize(v, &n));
2057     PetscCall(VecGetArray(v, &x));
2058     x += cDim;
2059     for (i = 0; i < n; i += cDim) x[i] += lower[2];
2060     PetscCall(VecRestoreArray(v, &x));
2061     PetscCall(DMSetCoordinatesLocal(dm, v));
2062   }
2063   PetscFunctionReturn(PETSC_SUCCESS);
2064 }
2065 
2066 /*@
2067   DMPlexCreateWedgeBoxMesh - Creates a 3-D mesh tessellating the (x,y) plane and extruding in the third direction using wedge cells.
2068 
2069   Collective
2070 
2071   Input Parameters:
2072 + comm        - The communicator for the `DM` object
2073 . faces       - Number of faces per dimension, or `NULL` for (1, 1, 1)
2074 . lower       - The lower left corner, or `NULL` for (0, 0, 0)
2075 . upper       - The upper right corner, or `NULL` for (1, 1, 1)
2076 . periodicity - The boundary type for the X,Y,Z direction, or `NULL` for `DM_BOUNDARY_NONE`
2077 . orderHeight - If `PETSC_TRUE`, orders the extruded cells in the height first. Otherwise, orders the cell on the layers first
2078 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
2079 
2080   Output Parameter:
2081 . dm - The `DM` object
2082 
2083   Level: beginner
2084 
2085 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateHexCylinderMesh()`, `DMPlexCreateWedgeCylinderMesh()`, `DMExtrude()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
2086 @*/
2087 PetscErrorCode DMPlexCreateWedgeBoxMesh(MPI_Comm comm, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool orderHeight, PetscBool interpolate, DM *dm)
2088 {
2089   PetscInt       fac[3] = {1, 1, 1};
2090   PetscReal      low[3] = {0, 0, 0};
2091   PetscReal      upp[3] = {1, 1, 1};
2092   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
2093 
2094   PetscFunctionBegin;
2095   PetscCall(DMCreate(comm, dm));
2096   PetscCall(DMSetType(*dm, DMPLEX));
2097   PetscCall(DMPlexCreateWedgeBoxMesh_Internal(*dm, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, periodicity ? periodicity : bdt));
2098   if (!interpolate) {
2099     DM udm;
2100 
2101     PetscCall(DMPlexUninterpolate(*dm, &udm));
2102     PetscCall(DMPlexReplace_Internal(*dm, &udm));
2103   }
2104   if (periodicity) PetscCall(DMLocalizeCoordinates(*dm));
2105   PetscFunctionReturn(PETSC_SUCCESS);
2106 }
2107 
2108 /*
2109   DMPlexTensorPointLexicographic_Private - Returns all tuples of size 'len' with nonnegative integers that are all less than or equal to 'max' for that dimension.
2110 
2111   Input Parameters:
2112 + len - The length of the tuple
2113 . max - The maximum for each dimension, so values are in [0, max)
2114 - tup - A tuple of length len+1: tup[len] > 0 indicates a stopping condition
2115 
2116   Output Parameter:
2117 . tup - A tuple of `len` integers whose entries are at most `max`
2118 
2119   Level: developer
2120 
2121   Note:
2122   Ordering is lexicographic with lowest index as least significant in ordering.
2123   e.g. for len == 2 and max == 2, this will return, in order, {0,0}, {1,0}, {2,0}, {0,1}, {1,1}, {2,1}, {0,2}, {1,2}, {2,2}.
2124 
2125 .seealso: PetscDualSpaceTensorPointLexicographic_Internal(), PetscDualSpaceLatticePointLexicographic_Internal()
2126 */
2127 static PetscErrorCode DMPlexTensorPointLexicographic_Private(PetscInt len, const PetscInt max[], PetscInt tup[])
2128 {
2129   PetscInt i;
2130 
2131   PetscFunctionBegin;
2132   for (i = 0; i < len; ++i) {
2133     if (tup[i] < max[i] - 1) {
2134       break;
2135     } else {
2136       tup[i] = 0;
2137     }
2138   }
2139   if (i == len) tup[i - 1] = max[i - 1];
2140   else ++tup[i];
2141   PetscFunctionReturn(PETSC_SUCCESS);
2142 }
2143 
2144 static PetscInt TupleToIndex_Private(PetscInt len, const PetscInt max[], const PetscInt tup[])
2145 {
2146   PetscInt idx = tup[len - 1];
2147 
2148   for (PetscInt i = len - 2; i >= 0; --i) {
2149     idx *= max[i];
2150     idx += tup[i];
2151   }
2152   return idx;
2153 }
2154 
2155 static void IndexToTuple_Private(PetscInt len, const PetscInt max[], PetscInt idx, PetscInt tup[])
2156 {
2157   for (PetscInt i = 0; i < len; ++i) {
2158     tup[i] = idx % max[i];
2159     idx    = (idx - tup[i]) / max[i];
2160   }
2161 }
2162 
2163 static void TupleToRanks_Private(PetscInt len, const PetscInt max[], const PetscInt procs[], const PetscInt tup[], PetscInt ranks[])
2164 {
2165   for (PetscInt i = 0; i < len; ++i) {
2166     const PetscInt div = max[i] / procs[i];
2167     const PetscInt rem = max[i] % procs[i];
2168     const PetscInt idx = (tup[i] < 0 ? max[i] + tup[i] : tup[i]) % max[i];
2169 
2170     if (idx < rem * (div + 1)) ranks[i] = idx / (div + 1);
2171     else ranks[i] = rem + (idx - rem * (div + 1)) / div;
2172   }
2173 }
2174 
2175 static void RanksToSizes_Private(PetscInt len, const PetscInt max[], const PetscInt procs[], const PetscInt ranks[], PetscInt sizes[])
2176 {
2177   for (PetscInt i = 0; i < len; ++i) {
2178     const PetscInt div = max[i] / procs[i];
2179     const PetscInt rem = max[i] % procs[i];
2180 
2181     sizes[i] = ranks[i] < rem ? div + 1 : div;
2182   }
2183 }
2184 
2185 /*
2186   In serial, the mesh is completely periodic. In parallel, we will include a layer of ghost vertices around the patch, so our edges will not wrap around in parallel. This will instead be handled by the SF.
2187 
2188   The cone for all edges will always be complete. Even in the presence of boundaries, we will keep all support sizes constant. When an edge would not exist in the support, we will create one to wrap back periodically. This allows for uniform stencils, and that edge will not be used for computation.
2189 
2190   All point which do not attain the vertex lower or upper bound in any dimension are owned, the rest are leaves owned by another process and present in the SF.
2191 
2192   Parallel Layout:
2193 
2194   We create a cubic process grid of dimension P^{1/d} on each side. The IndexToTuple_Private() function maps the global rank to the local rank in each dimension. We divide edges using the PETSc distribution rule in each dimension, and then add overlap. TupleToRanks_Private() returns the local rank in ech dimension for a tuple. RanksToSizes_Private() gives the size in each dimension for the domain with those local ranks.
2195 */
2196 static PetscErrorCode DMPlexCreateHypercubicMesh_Internal(DM dm, PetscInt dim, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[], PetscInt overlap, const DMBoundaryType bd[])
2197 {
2198   const PetscInt debug = ((DM_Plex *)dm->data)->printAdj;
2199   PetscSF        sf;
2200   Vec            coordinates;
2201   PetscSection   coordSection;
2202   DMLabel        cutLabel    = NULL;
2203   PetscBool      cutMarker   = PETSC_FALSE;
2204   PetscBool      periodic    = PETSC_FALSE;
2205   PetscInt       numCells    = 1;
2206   PetscInt       numVertices = 1;
2207   PetscSFNode   *remotes;
2208   PetscScalar   *coords;
2209   PetscInt      *procs;     // The number of processes along each dimension
2210   PetscInt      *lrank;     // Rank in each dimension, lrank[d] \in [0, procs[d])
2211   PetscInt      *ledges;    // The number of edges along each dimension for this process
2212   PetscInt      *vstart;    // The first vertex along each dimension on this processes
2213   PetscInt      *vertices;  // The number of vertices along each dimension on this process
2214   PetscInt      *rvert;     // The global (not local) vertex number along each dimension
2215   PetscInt      *rrank;     // The rank along each dimension for the process owning rvert[]
2216   PetscInt      *rvertices; // The number of vertices along each dimension for the process rrank[]
2217   PetscInt      *vert, *vtmp, *supp, cone[2], *leaves;
2218   PetscInt       cell = 0, coordSize, Nl = 0, Nl2 = 0;
2219   PetscMPIInt    rank, size;
2220   MPI_Comm       comm;
2221 
2222   PetscFunctionBegin;
2223   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
2224   PetscCallMPI(MPI_Comm_rank(comm, &rank));
2225   PetscCallMPI(MPI_Comm_size(comm, &size));
2226   PetscCall(DMSetDimension(dm, dim));
2227   PetscCall(DMPlexDistributeSetDefault(dm, PETSC_FALSE));
2228   PetscCall(PetscCalloc4(dim, &procs, dim, &lrank, dim, &rrank, 2 * dim, &supp));
2229   PetscCall(PetscCalloc7(dim, &ledges, dim, &vertices, dim, &rvertices, dim, &vert, dim, &rvert, dim, &vstart, dim, &vtmp));
2230   PetscCall(DMCreateLabel(dm, "marker"));
2231   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_periodic_cut", &cutMarker, NULL));
2232   for (PetscInt d = 0; d < dim; ++d) periodic = (periodic || bd[d] == DM_BOUNDARY_PERIODIC) ? PETSC_TRUE : PETSC_FALSE;
2233   if (periodic && cutMarker) {
2234     PetscCall(DMCreateLabel(dm, "periodic_cut"));
2235     PetscCall(DMGetLabel(dm, "periodic_cut", &cutLabel));
2236   }
2237   for (PetscInt d = 0; d < dim; ++d) PetscCheck(bd[d] == DM_BOUNDARY_PERIODIC, comm, PETSC_ERR_SUP, "Hypercubic mesh must be periodic now");
2238   overlap = overlap == PETSC_DETERMINE ? 1 : overlap;
2239   PetscCheck(overlap >= 1, comm, PETSC_ERR_SUP, "Overlap %" PetscInt_FMT " must be greater than 0", overlap);
2240   if (size > 1) {
2241     PetscInt Npr = 1;
2242 
2243     // Make process grid
2244     if (debug) PetscCall(PetscPrintf(comm, "Process grid:"));
2245     for (PetscInt d = 0; d < dim; ++d) {
2246       procs[d] = PetscRintReal(PetscPowReal(size, 1. / dim));
2247       Npr *= procs[d];
2248       if (debug) PetscCall(PetscPrintf(comm, " %" PetscInt_FMT, procs[d]));
2249     }
2250     if (debug) PetscCall(PetscPrintf(comm, "\n"));
2251     PetscCheck(Npr == size, comm, PETSC_ERR_PLIB, "Process grid size %" PetscInt_FMT " != %d comm size", Npr, size);
2252     IndexToTuple_Private(dim, procs, rank, lrank);
2253     for (PetscInt d = 0; d < dim; ++d) {
2254       ledges[d] = edges[d] / procs[d] + (edges[d] % procs[d] > lrank[d] ? 1 : 0);
2255       vstart[d] = 0;
2256       for (PetscInt r = 0; r < lrank[d]; ++r) vstart[d] += edges[d] / procs[d] + (edges[d] % procs[d] > r ? 1 : 0);
2257       vstart[d] -= overlap; // For halo
2258     }
2259   } else {
2260     for (PetscInt d = 0; d < dim; ++d) {
2261       procs[d]  = 1;
2262       ledges[d] = edges[d];
2263     }
2264   }
2265   // Calculate local patch size
2266   for (PetscInt d = 0; d < dim; ++d) {
2267     vertices[d] = ledges[d] + (procs[d] > 1 ? 2 * overlap : 0);
2268     numVertices *= vertices[d];
2269   }
2270   numCells = numVertices * dim;
2271   PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
2272   for (PetscInt c = 0; c < numCells; ++c) PetscCall(DMPlexSetConeSize(dm, c, 2));
2273   for (PetscInt v = numCells; v < numCells + numVertices; ++v) PetscCall(DMPlexSetSupportSize(dm, v, 2 * dim));
2274   PetscCall(DMSetUp(dm)); /* Allocate space for cones and supports */
2275   /* Build cell cones and vertex supports */
2276   PetscCall(DMCreateLabel(dm, "celltype"));
2277   if (debug) PetscCall(PetscSynchronizedPrintf(comm, "Topology for rank %d:\n", rank));
2278   while (vert[dim - 1] < vertices[dim - 1]) {
2279     const PetscInt vertex = TupleToIndex_Private(dim, vertices, vert) + numCells;
2280     PetscInt       s      = 0;
2281     PetscBool      leaf   = PETSC_FALSE;
2282 
2283     if (debug) {
2284       PetscCall(PetscSynchronizedPrintf(comm, "Vertex %" PetscInt_FMT ":", vertex));
2285       for (PetscInt d = 0; d < dim; ++d) PetscCall(PetscSynchronizedPrintf(comm, " %" PetscInt_FMT, vert[d]));
2286       PetscCall(PetscSynchronizedPrintf(comm, "\n"));
2287     }
2288     PetscCall(DMPlexSetCellType(dm, vertex, DM_POLYTOPE_POINT));
2289     // Define edge cones
2290     for (PetscInt d = 0; d < dim; ++d) {
2291       for (PetscInt e = 0; e < dim; ++e) vtmp[e] = vert[e];
2292       vtmp[d] = (vert[d] + 1) % vertices[d];
2293       cone[0] = vertex;
2294       cone[1] = TupleToIndex_Private(dim, vertices, vtmp) + numCells;
2295       if (debug) {
2296         PetscCall(PetscSynchronizedPrintf(comm, "  Vertex %" PetscInt_FMT ":", cone[1]));
2297         for (PetscInt e = 0; e < dim; ++e) PetscCall(PetscSynchronizedPrintf(comm, " %" PetscInt_FMT, vtmp[e]));
2298         PetscCall(PetscSynchronizedPrintf(comm, "\n"));
2299       }
2300       PetscCall(DMPlexSetCone(dm, cell, cone));
2301       PetscCall(DMPlexSetCellType(dm, cell, DM_POLYTOPE_SEGMENT));
2302       if (debug) PetscCall(PetscSynchronizedPrintf(comm, "  Edge %" PetscInt_FMT " (%" PetscInt_FMT " %" PetscInt_FMT ")\n", cell, cone[0], cone[1]));
2303       ++cell;
2304       // Shared vertices are any in the first or last overlap layers
2305       if (vert[d] < overlap || vert[d] >= vertices[d] - overlap) leaf = PETSC_TRUE;
2306     }
2307     if (size > 1 && leaf) ++Nl;
2308     // Define vertex supports
2309     for (PetscInt d = 0; d < dim; ++d) {
2310       for (PetscInt e = 0; e < dim; ++e) vtmp[e] = vert[e];
2311       vtmp[d]   = (vert[d] + vertices[d] - 1) % vertices[d];
2312       supp[s++] = TupleToIndex_Private(dim, vertices, vtmp) * dim + d;
2313       supp[s++] = (vertex - numCells) * dim + d;
2314       PetscCall(DMPlexSetSupport(dm, vertex, supp));
2315     }
2316     PetscCall(DMPlexTensorPointLexicographic_Private(dim, vertices, vert));
2317   }
2318   if (debug) PetscCall(PetscSynchronizedFlush(comm, NULL));
2319   PetscCall(DMPlexStratify(dm));
2320   // Allocate for SF
2321   PetscCall(PetscMalloc1(Nl, &leaves));
2322   PetscCall(PetscMalloc1(Nl, &remotes));
2323   // Build coordinates
2324   PetscCall(DMGetCoordinateSection(dm, &coordSection));
2325   PetscCall(PetscSectionSetNumFields(coordSection, 1));
2326   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
2327   PetscCall(PetscSectionSetChart(coordSection, numCells, numCells + numVertices));
2328   for (PetscInt v = numCells; v < numCells + numVertices; ++v) {
2329     PetscCall(PetscSectionSetDof(coordSection, v, dim));
2330     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
2331   }
2332   PetscCall(PetscSectionSetUp(coordSection));
2333   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
2334   PetscCall(VecCreate(comm, &coordinates));
2335   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
2336   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
2337   PetscCall(VecSetBlockSize(coordinates, dim));
2338   PetscCall(VecSetType(coordinates, VECSTANDARD));
2339   PetscCall(VecGetArray(coordinates, &coords));
2340   if (debug) PetscCall(PetscSynchronizedPrintf(comm, "Geometry for rank %d:\n", rank));
2341   for (PetscInt d = 0; d < dim; ++d) vert[d] = 0;
2342   while (vert[dim - 1] < vertices[dim - 1]) {
2343     const PetscInt vertex = TupleToIndex_Private(dim, vertices, vert);
2344     PetscBool      leaf   = PETSC_FALSE;
2345 
2346     for (PetscInt d = 0; d < dim; ++d) {
2347       coords[vertex * dim + d] = lower[d] + ((upper[d] - lower[d]) / edges[d]) * (vert[d] + vstart[d]);
2348       if (vert[d] < overlap || vert[d] >= vertices[d] - overlap) leaf = PETSC_TRUE;
2349     }
2350     if (size > 1 && leaf) {
2351       PetscInt rnumCells = 1;
2352 
2353       for (PetscInt d = 0; d < dim; ++d) rvert[d] = vert[d] + vstart[d];
2354       TupleToRanks_Private(dim, edges, procs, rvert, rrank);
2355       leaves[Nl2]       = vertex + numCells;
2356       remotes[Nl2].rank = TupleToIndex_Private(dim, procs, rrank);
2357       RanksToSizes_Private(dim, edges, procs, rrank, rvertices);
2358       for (PetscInt d = 0; d < dim; ++d) {
2359         rvertices[d] += 2 * overlap; // Add halo
2360         rnumCells *= rvertices[d];
2361       }
2362       rnumCells *= dim;
2363       for (PetscInt d = 0; d < dim; ++d) {
2364         const PetscInt diff = rrank[d] - lrank[d];
2365 
2366         if (!diff) rvert[d] = vert[d];                                     // Vertex is local
2367         else if (rvert[d] < 0) rvert[d] = rvertices[d] - 1 + rvert[d];     // Wrap around at the bottom
2368         else if (rvert[d] >= edges[d]) rvert[d] = rvert[d] - edges[d] + 1; // Wrap around at the top
2369         else if (diff == -1) rvert[d] = rvertices[d] - 1 + (vert[d] - overlap);
2370         else if (diff == 1) rvert[d] = (vertices[d] - vert[d] - 1) + overlap;
2371         else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Process distance %" PetscInt_FMT " in direction %" PetscInt_FMT " should not be possible", diff, d);
2372       }
2373       remotes[Nl2].index = TupleToIndex_Private(dim, rvertices, rvert) + rnumCells;
2374       if (debug) PetscCall(PetscSynchronizedPrintf(comm, "Shared Vertex %" PetscInt_FMT " (%" PetscInt_FMT ", %" PetscInt_FMT ")\n", leaves[Nl2], remotes[Nl2].rank, remotes[Nl2].index));
2375       ++Nl2;
2376     }
2377     if (debug) {
2378       PetscCall(PetscSynchronizedPrintf(comm, "Vertex %" PetscInt_FMT ":", vertex));
2379       for (PetscInt d = 0; d < dim; ++d) PetscCall(PetscSynchronizedPrintf(comm, " %" PetscInt_FMT, vert[d] + vstart[d]));
2380       for (PetscInt d = 0; d < dim; ++d) PetscCall(PetscSynchronizedPrintf(comm, " %g", (double)PetscRealPart(coords[vertex * dim + d])));
2381       PetscCall(PetscSynchronizedPrintf(comm, "\n"));
2382     }
2383     PetscCall(DMPlexTensorPointLexicographic_Private(dim, vertices, vert));
2384   }
2385   if (debug) PetscCall(PetscSynchronizedFlush(comm, NULL));
2386   PetscCall(VecRestoreArray(coordinates, &coords));
2387   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
2388   PetscCall(VecDestroy(&coordinates));
2389   // Build SF
2390   PetscCheck(Nl == Nl2, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Initial number of leaves %" PetscInt_FMT " != %" PetscInt_FMT " final number", Nl, Nl2);
2391   PetscCall(DMGetPointSF(dm, &sf));
2392   PetscCall(PetscSFSetGraph(sf, numCells + numVertices, Nl, leaves, PETSC_OWN_POINTER, remotes, PETSC_OWN_POINTER));
2393   if (debug) PetscCall(PetscSFView(sf, PETSC_VIEWER_STDOUT_WORLD));
2394   //PetscCall(DMSetPeriodicity(dm, NULL, lower, upper));
2395   // Attach the extent
2396   {
2397     PetscContainer c;
2398     PetscInt      *extent, *lextent;
2399 
2400     PetscCall(PetscMalloc1(dim, &extent));
2401     PetscCall(PetscMalloc1(dim, &lextent));
2402     for (PetscInt d = 0; d < dim; ++d) {
2403       extent[d]  = edges[d];
2404       lextent[d] = ledges[d];
2405     }
2406     PetscCall(PetscContainerCreate(PETSC_COMM_SELF, &c));
2407     PetscCall(PetscContainerSetCtxDestroy(c, PetscCtxDestroyDefault));
2408     PetscCall(PetscContainerSetPointer(c, extent));
2409     PetscCall(PetscObjectCompose((PetscObject)dm, "_extent", (PetscObject)c));
2410     PetscCall(PetscContainerDestroy(&c));
2411     PetscCall(PetscContainerCreate(PETSC_COMM_SELF, &c));
2412     PetscCall(PetscContainerSetCtxDestroy(c, PetscCtxDestroyDefault));
2413     PetscCall(PetscContainerSetPointer(c, lextent));
2414     PetscCall(PetscObjectCompose((PetscObject)dm, "_lextent", (PetscObject)c));
2415     PetscCall(PetscContainerDestroy(&c));
2416   }
2417   PetscCall(PetscFree4(procs, lrank, rrank, supp));
2418   PetscCall(PetscFree7(ledges, vertices, rvertices, vert, rvert, vstart, vtmp));
2419   PetscFunctionReturn(PETSC_SUCCESS);
2420 }
2421 
2422 /*@C
2423   DMPlexCreateHypercubicMesh - Creates a periodic mesh on the tensor product of unit intervals using only vertices and edges.
2424 
2425   Collective
2426 
2427   Input Parameters:
2428 + comm    - The communicator for the `DM` object
2429 . dim     - The spatial dimension
2430 . edges   - Number of edges per dimension, or `NULL` for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
2431 . lower   - The lower left corner, or `NULL` for (0, 0, 0)
2432 . upper   - The upper right corner, or `NULL` for (1, 1, 1)
2433 - overlap - The number of vertices in each direction to include in the overlap (default is 1)
2434 
2435   Output Parameter:
2436 . dm - The DM object
2437 
2438   Level: beginner
2439 
2440   Note:
2441   If you want to customize this mesh using options, you just need to
2442 .vb
2443   DMCreate(comm, &dm);
2444   DMSetType(dm, DMPLEX);
2445   DMSetFromOptions(dm);
2446 .ve
2447   and use the options on the `DMSetFromOptions()` page.
2448 
2449   The vertices are numbered is lexicographic order, and the dim edges exiting a vertex in the positive orthant are number consecutively,
2450 .vb
2451  18--0-19--2-20--4-18
2452   |     |     |     |
2453  13    15    17    13
2454   |     |     |     |
2455  24-12-25-14-26-16-24
2456   |     |     |     |
2457   7     9    11     7
2458   |     |     |     |
2459  21--6-22--8-23-10-21
2460   |     |     |     |
2461   1     3     5     1
2462   |     |     |     |
2463  18--0-19--2-20--4-18
2464 .ve
2465 
2466 .seealso: `DMSetFromOptions()`, `DMPlexCreateFromFile()`, `DMPlexCreateHexCylinderMesh()`, `DMSetType()`, `DMCreate()`
2467 @*/
2468 PetscErrorCode DMPlexCreateHypercubicMesh(MPI_Comm comm, PetscInt dim, const PetscInt edges[], const PetscReal lower[], const PetscReal upper[], PetscInt overlap, DM *dm)
2469 {
2470   PetscInt       *edg;
2471   PetscReal      *low, *upp;
2472   DMBoundaryType *bdt;
2473   PetscInt        d;
2474 
2475   PetscFunctionBegin;
2476   PetscCall(DMCreate(comm, dm));
2477   PetscCall(DMSetType(*dm, DMPLEX));
2478   PetscCall(PetscMalloc4(dim, &edg, dim, &low, dim, &upp, dim, &bdt));
2479   for (d = 0; d < dim; ++d) {
2480     edg[d] = edges ? edges[d] : 1;
2481     low[d] = lower ? lower[d] : 0.;
2482     upp[d] = upper ? upper[d] : 1.;
2483     bdt[d] = DM_BOUNDARY_PERIODIC;
2484   }
2485   PetscCall(DMPlexCreateHypercubicMesh_Internal(*dm, dim, low, upp, edg, overlap, bdt));
2486   PetscCall(PetscFree4(edg, low, upp, bdt));
2487   PetscFunctionReturn(PETSC_SUCCESS);
2488 }
2489 
2490 /*@
2491   DMPlexSetOptionsPrefix - Sets the prefix used for searching for all `DM` options in the database.
2492 
2493   Logically Collective
2494 
2495   Input Parameters:
2496 + dm     - the `DM` context
2497 - prefix - the prefix to prepend to all option names
2498 
2499   Level: advanced
2500 
2501   Note:
2502   A hyphen (-) must NOT be given at the beginning of the prefix name.
2503   The first character of all runtime options is AUTOMATICALLY the hyphen.
2504 
2505 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `SNESSetFromOptions()`
2506 @*/
2507 PetscErrorCode DMPlexSetOptionsPrefix(DM dm, const char prefix[])
2508 {
2509   DM_Plex *mesh = (DM_Plex *)dm->data;
2510 
2511   PetscFunctionBegin;
2512   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2513   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)dm, prefix));
2514   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)mesh->partitioner, prefix));
2515   PetscFunctionReturn(PETSC_SUCCESS);
2516 }
2517 
2518 /* Remap geometry to cylinder
2519    TODO: This only works for a single refinement, then it is broken
2520 
2521      Interior square: Linear interpolation is correct
2522      The other cells all have vertices on rays from the origin. We want to uniformly expand the spacing
2523      such that the last vertex is on the unit circle. So the closest and farthest vertices are at distance
2524 
2525        phi     = arctan(y/x)
2526        d_close = sqrt(1/8 + 1/4 sin^2(phi))
2527        d_far   = sqrt(1/2 + sin^2(phi))
2528 
2529      so we remap them using
2530 
2531        x_new = x_close + (x - x_close) (1 - d_close) / (d_far - d_close)
2532        y_new = y_close + (y - y_close) (1 - d_close) / (d_far - d_close)
2533 
2534      If pi/4 < phi < 3pi/4 or -3pi/4 < phi < -pi/4, then we switch x and y.
2535 */
2536 static void snapToCylinder(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
2537 {
2538   const PetscReal dis = 1.0 / PetscSqrtReal(2.0);
2539   const PetscReal ds2 = 0.5 * dis;
2540 
2541   if ((PetscAbsScalar(u[0]) <= ds2) && (PetscAbsScalar(u[1]) <= ds2)) {
2542     f0[0] = u[0];
2543     f0[1] = u[1];
2544   } else {
2545     PetscReal phi, sinp, cosp, dc, df, x, y, xc, yc;
2546 
2547     x    = PetscRealPart(u[0]);
2548     y    = PetscRealPart(u[1]);
2549     phi  = PetscAtan2Real(y, x);
2550     sinp = PetscSinReal(phi);
2551     cosp = PetscCosReal(phi);
2552     if ((PetscAbsReal(phi) > PETSC_PI / 4.0) && (PetscAbsReal(phi) < 3.0 * PETSC_PI / 4.0)) {
2553       dc = PetscAbsReal(ds2 / sinp);
2554       df = PetscAbsReal(dis / sinp);
2555       xc = ds2 * x / PetscAbsReal(y);
2556       yc = ds2 * PetscSignReal(y);
2557     } else {
2558       dc = PetscAbsReal(ds2 / cosp);
2559       df = PetscAbsReal(dis / cosp);
2560       xc = ds2 * PetscSignReal(x);
2561       yc = ds2 * y / PetscAbsReal(x);
2562     }
2563     f0[0] = xc + (u[0] - xc) * (1.0 - dc) / (df - dc);
2564     f0[1] = yc + (u[1] - yc) * (1.0 - dc) / (df - dc);
2565   }
2566   f0[2] = u[2];
2567 }
2568 
2569 static PetscErrorCode DMPlexCreateHexCylinderMesh_Internal(DM dm, DMBoundaryType periodicZ, PetscInt Nr)
2570 {
2571   const PetscInt dim = 3;
2572   PetscInt       numCells, numVertices;
2573   PetscMPIInt    rank;
2574 
2575   PetscFunctionBegin;
2576   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
2577   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
2578   PetscCall(DMSetDimension(dm, dim));
2579   /* Create topology */
2580   {
2581     PetscInt cone[8], c;
2582 
2583     numCells    = rank == 0 ? 5 : 0;
2584     numVertices = rank == 0 ? 16 : 0;
2585     if (periodicZ == DM_BOUNDARY_PERIODIC) {
2586       numCells *= 3;
2587       numVertices = rank == 0 ? 24 : 0;
2588     }
2589     PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
2590     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 8));
2591     PetscCall(DMSetUp(dm));
2592     if (rank == 0) {
2593       if (periodicZ == DM_BOUNDARY_PERIODIC) {
2594         cone[0] = 15;
2595         cone[1] = 18;
2596         cone[2] = 17;
2597         cone[3] = 16;
2598         cone[4] = 31;
2599         cone[5] = 32;
2600         cone[6] = 33;
2601         cone[7] = 34;
2602         PetscCall(DMPlexSetCone(dm, 0, cone));
2603         cone[0] = 16;
2604         cone[1] = 17;
2605         cone[2] = 24;
2606         cone[3] = 23;
2607         cone[4] = 32;
2608         cone[5] = 36;
2609         cone[6] = 37;
2610         cone[7] = 33; /* 22 25 26 21 */
2611         PetscCall(DMPlexSetCone(dm, 1, cone));
2612         cone[0] = 18;
2613         cone[1] = 27;
2614         cone[2] = 24;
2615         cone[3] = 17;
2616         cone[4] = 34;
2617         cone[5] = 33;
2618         cone[6] = 37;
2619         cone[7] = 38;
2620         PetscCall(DMPlexSetCone(dm, 2, cone));
2621         cone[0] = 29;
2622         cone[1] = 27;
2623         cone[2] = 18;
2624         cone[3] = 15;
2625         cone[4] = 35;
2626         cone[5] = 31;
2627         cone[6] = 34;
2628         cone[7] = 38;
2629         PetscCall(DMPlexSetCone(dm, 3, cone));
2630         cone[0] = 29;
2631         cone[1] = 15;
2632         cone[2] = 16;
2633         cone[3] = 23;
2634         cone[4] = 35;
2635         cone[5] = 36;
2636         cone[6] = 32;
2637         cone[7] = 31;
2638         PetscCall(DMPlexSetCone(dm, 4, cone));
2639 
2640         cone[0] = 31;
2641         cone[1] = 34;
2642         cone[2] = 33;
2643         cone[3] = 32;
2644         cone[4] = 19;
2645         cone[5] = 22;
2646         cone[6] = 21;
2647         cone[7] = 20;
2648         PetscCall(DMPlexSetCone(dm, 5, cone));
2649         cone[0] = 32;
2650         cone[1] = 33;
2651         cone[2] = 37;
2652         cone[3] = 36;
2653         cone[4] = 22;
2654         cone[5] = 25;
2655         cone[6] = 26;
2656         cone[7] = 21;
2657         PetscCall(DMPlexSetCone(dm, 6, cone));
2658         cone[0] = 34;
2659         cone[1] = 38;
2660         cone[2] = 37;
2661         cone[3] = 33;
2662         cone[4] = 20;
2663         cone[5] = 21;
2664         cone[6] = 26;
2665         cone[7] = 28;
2666         PetscCall(DMPlexSetCone(dm, 7, cone));
2667         cone[0] = 35;
2668         cone[1] = 38;
2669         cone[2] = 34;
2670         cone[3] = 31;
2671         cone[4] = 30;
2672         cone[5] = 19;
2673         cone[6] = 20;
2674         cone[7] = 28;
2675         PetscCall(DMPlexSetCone(dm, 8, cone));
2676         cone[0] = 35;
2677         cone[1] = 31;
2678         cone[2] = 32;
2679         cone[3] = 36;
2680         cone[4] = 30;
2681         cone[5] = 25;
2682         cone[6] = 22;
2683         cone[7] = 19;
2684         PetscCall(DMPlexSetCone(dm, 9, cone));
2685 
2686         cone[0] = 19;
2687         cone[1] = 20;
2688         cone[2] = 21;
2689         cone[3] = 22;
2690         cone[4] = 15;
2691         cone[5] = 16;
2692         cone[6] = 17;
2693         cone[7] = 18;
2694         PetscCall(DMPlexSetCone(dm, 10, cone));
2695         cone[0] = 22;
2696         cone[1] = 21;
2697         cone[2] = 26;
2698         cone[3] = 25;
2699         cone[4] = 16;
2700         cone[5] = 23;
2701         cone[6] = 24;
2702         cone[7] = 17;
2703         PetscCall(DMPlexSetCone(dm, 11, cone));
2704         cone[0] = 20;
2705         cone[1] = 28;
2706         cone[2] = 26;
2707         cone[3] = 21;
2708         cone[4] = 18;
2709         cone[5] = 17;
2710         cone[6] = 24;
2711         cone[7] = 27;
2712         PetscCall(DMPlexSetCone(dm, 12, cone));
2713         cone[0] = 30;
2714         cone[1] = 28;
2715         cone[2] = 20;
2716         cone[3] = 19;
2717         cone[4] = 29;
2718         cone[5] = 15;
2719         cone[6] = 18;
2720         cone[7] = 27;
2721         PetscCall(DMPlexSetCone(dm, 13, cone));
2722         cone[0] = 30;
2723         cone[1] = 19;
2724         cone[2] = 22;
2725         cone[3] = 25;
2726         cone[4] = 29;
2727         cone[5] = 23;
2728         cone[6] = 16;
2729         cone[7] = 15;
2730         PetscCall(DMPlexSetCone(dm, 14, cone));
2731       } else {
2732         cone[0] = 5;
2733         cone[1] = 8;
2734         cone[2] = 7;
2735         cone[3] = 6;
2736         cone[4] = 9;
2737         cone[5] = 12;
2738         cone[6] = 11;
2739         cone[7] = 10;
2740         PetscCall(DMPlexSetCone(dm, 0, cone));
2741         cone[0] = 6;
2742         cone[1] = 7;
2743         cone[2] = 14;
2744         cone[3] = 13;
2745         cone[4] = 12;
2746         cone[5] = 15;
2747         cone[6] = 16;
2748         cone[7] = 11;
2749         PetscCall(DMPlexSetCone(dm, 1, cone));
2750         cone[0] = 8;
2751         cone[1] = 17;
2752         cone[2] = 14;
2753         cone[3] = 7;
2754         cone[4] = 10;
2755         cone[5] = 11;
2756         cone[6] = 16;
2757         cone[7] = 18;
2758         PetscCall(DMPlexSetCone(dm, 2, cone));
2759         cone[0] = 19;
2760         cone[1] = 17;
2761         cone[2] = 8;
2762         cone[3] = 5;
2763         cone[4] = 20;
2764         cone[5] = 9;
2765         cone[6] = 10;
2766         cone[7] = 18;
2767         PetscCall(DMPlexSetCone(dm, 3, cone));
2768         cone[0] = 19;
2769         cone[1] = 5;
2770         cone[2] = 6;
2771         cone[3] = 13;
2772         cone[4] = 20;
2773         cone[5] = 15;
2774         cone[6] = 12;
2775         cone[7] = 9;
2776         PetscCall(DMPlexSetCone(dm, 4, cone));
2777       }
2778     }
2779     PetscCall(DMPlexSymmetrize(dm));
2780     PetscCall(DMPlexStratify(dm));
2781   }
2782   /* Create cube geometry */
2783   {
2784     Vec             coordinates;
2785     PetscSection    coordSection;
2786     PetscScalar    *coords;
2787     PetscInt        coordSize, v;
2788     const PetscReal dis = 1.0 / PetscSqrtReal(2.0);
2789     const PetscReal ds2 = dis / 2.0;
2790 
2791     /* Build coordinates */
2792     PetscCall(DMGetCoordinateSection(dm, &coordSection));
2793     PetscCall(PetscSectionSetNumFields(coordSection, 1));
2794     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
2795     PetscCall(PetscSectionSetChart(coordSection, numCells, numCells + numVertices));
2796     for (v = numCells; v < numCells + numVertices; ++v) {
2797       PetscCall(PetscSectionSetDof(coordSection, v, dim));
2798       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
2799     }
2800     PetscCall(PetscSectionSetUp(coordSection));
2801     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
2802     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
2803     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
2804     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
2805     PetscCall(VecSetBlockSize(coordinates, dim));
2806     PetscCall(VecSetType(coordinates, VECSTANDARD));
2807     PetscCall(VecGetArray(coordinates, &coords));
2808     if (rank == 0) {
2809       coords[0 * dim + 0]  = -ds2;
2810       coords[0 * dim + 1]  = -ds2;
2811       coords[0 * dim + 2]  = 0.0;
2812       coords[1 * dim + 0]  = ds2;
2813       coords[1 * dim + 1]  = -ds2;
2814       coords[1 * dim + 2]  = 0.0;
2815       coords[2 * dim + 0]  = ds2;
2816       coords[2 * dim + 1]  = ds2;
2817       coords[2 * dim + 2]  = 0.0;
2818       coords[3 * dim + 0]  = -ds2;
2819       coords[3 * dim + 1]  = ds2;
2820       coords[3 * dim + 2]  = 0.0;
2821       coords[4 * dim + 0]  = -ds2;
2822       coords[4 * dim + 1]  = -ds2;
2823       coords[4 * dim + 2]  = 1.0;
2824       coords[5 * dim + 0]  = -ds2;
2825       coords[5 * dim + 1]  = ds2;
2826       coords[5 * dim + 2]  = 1.0;
2827       coords[6 * dim + 0]  = ds2;
2828       coords[6 * dim + 1]  = ds2;
2829       coords[6 * dim + 2]  = 1.0;
2830       coords[7 * dim + 0]  = ds2;
2831       coords[7 * dim + 1]  = -ds2;
2832       coords[7 * dim + 2]  = 1.0;
2833       coords[8 * dim + 0]  = dis;
2834       coords[8 * dim + 1]  = -dis;
2835       coords[8 * dim + 2]  = 0.0;
2836       coords[9 * dim + 0]  = dis;
2837       coords[9 * dim + 1]  = dis;
2838       coords[9 * dim + 2]  = 0.0;
2839       coords[10 * dim + 0] = dis;
2840       coords[10 * dim + 1] = -dis;
2841       coords[10 * dim + 2] = 1.0;
2842       coords[11 * dim + 0] = dis;
2843       coords[11 * dim + 1] = dis;
2844       coords[11 * dim + 2] = 1.0;
2845       coords[12 * dim + 0] = -dis;
2846       coords[12 * dim + 1] = dis;
2847       coords[12 * dim + 2] = 0.0;
2848       coords[13 * dim + 0] = -dis;
2849       coords[13 * dim + 1] = dis;
2850       coords[13 * dim + 2] = 1.0;
2851       coords[14 * dim + 0] = -dis;
2852       coords[14 * dim + 1] = -dis;
2853       coords[14 * dim + 2] = 0.0;
2854       coords[15 * dim + 0] = -dis;
2855       coords[15 * dim + 1] = -dis;
2856       coords[15 * dim + 2] = 1.0;
2857       if (periodicZ == DM_BOUNDARY_PERIODIC) {
2858         /* 15 31 19 */ coords[16 * dim + 0] = -ds2;
2859         coords[16 * dim + 1]                = -ds2;
2860         coords[16 * dim + 2]                = 0.5;
2861         /* 16 32 22 */ coords[17 * dim + 0] = ds2;
2862         coords[17 * dim + 1]                = -ds2;
2863         coords[17 * dim + 2]                = 0.5;
2864         /* 17 33 21 */ coords[18 * dim + 0] = ds2;
2865         coords[18 * dim + 1]                = ds2;
2866         coords[18 * dim + 2]                = 0.5;
2867         /* 18 34 20 */ coords[19 * dim + 0] = -ds2;
2868         coords[19 * dim + 1]                = ds2;
2869         coords[19 * dim + 2]                = 0.5;
2870         /* 29 35 30 */ coords[20 * dim + 0] = -dis;
2871         coords[20 * dim + 1]                = -dis;
2872         coords[20 * dim + 2]                = 0.5;
2873         /* 23 36 25 */ coords[21 * dim + 0] = dis;
2874         coords[21 * dim + 1]                = -dis;
2875         coords[21 * dim + 2]                = 0.5;
2876         /* 24 37 26 */ coords[22 * dim + 0] = dis;
2877         coords[22 * dim + 1]                = dis;
2878         coords[22 * dim + 2]                = 0.5;
2879         /* 27 38 28 */ coords[23 * dim + 0] = -dis;
2880         coords[23 * dim + 1]                = dis;
2881         coords[23 * dim + 2]                = 0.5;
2882       }
2883     }
2884     PetscCall(VecRestoreArray(coordinates, &coords));
2885     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
2886     PetscCall(VecDestroy(&coordinates));
2887   }
2888   /* Create periodicity */
2889   if (periodicZ == DM_BOUNDARY_PERIODIC || periodicZ == DM_BOUNDARY_TWIST) {
2890     PetscReal L[3]       = {-1., -1., 0.};
2891     PetscReal maxCell[3] = {-1., -1., 0.};
2892     PetscReal lower[3]   = {0.0, 0.0, 0.0};
2893     PetscReal upper[3]   = {1.0, 1.0, 1.5};
2894     PetscInt  numZCells  = 3;
2895 
2896     L[2]       = upper[2] - lower[2];
2897     maxCell[2] = 1.1 * (L[2] / numZCells);
2898     PetscCall(DMSetPeriodicity(dm, maxCell, lower, L));
2899   }
2900   {
2901     DM          cdm;
2902     PetscDS     cds;
2903     PetscScalar c[2] = {1.0, 1.0};
2904 
2905     PetscCall(DMPlexCreateCoordinateSpace(dm, 1, PETSC_TRUE, NULL));
2906     PetscCall(DMGetCoordinateDM(dm, &cdm));
2907     PetscCall(DMGetDS(cdm, &cds));
2908     PetscCall(PetscDSSetConstants(cds, 2, c));
2909   }
2910   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
2911 
2912   /* Wait for coordinate creation before doing in-place modification */
2913   PetscCall(DMPlexInterpolateInPlace_Internal(dm));
2914 
2915   char        oldprefix[PETSC_MAX_PATH_LEN];
2916   const char *prefix;
2917 
2918   PetscCall(PetscObjectGetOptionsPrefix((PetscObject)dm, &prefix));
2919   PetscCall(PetscStrncpy(oldprefix, prefix, PETSC_MAX_PATH_LEN));
2920   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)dm, "petsc_cyl_ref_"));
2921   for (PetscInt r = 0; r < PetscMax(0, Nr); ++r) {
2922     DM rdm;
2923 
2924     PetscCall(DMRefine(dm, PetscObjectComm((PetscObject)dm), &rdm));
2925     PetscCall(DMPlexReplace_Internal(dm, &rdm));
2926   }
2927   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)dm, oldprefix));
2928   PetscCall(DMPlexRemapGeometry(dm, 0.0, snapToCylinder));
2929 
2930   DMLabel         bdlabel, edgelabel;
2931   IS              faceIS;
2932   const PetscInt *faces;
2933   PetscInt        Nf;
2934 
2935   PetscCall(DMCreateLabel(dm, "marker"));
2936   PetscCall(DMGetLabel(dm, "marker", &bdlabel));
2937   PetscCall(DMCreateLabel(dm, "generatrix"));
2938   PetscCall(DMGetLabel(dm, "generatrix", &edgelabel));
2939   PetscCall(DMPlexMarkBoundaryFaces(dm, PETSC_DETERMINE, bdlabel));
2940   // Remove faces on top and bottom
2941   PetscCall(DMLabelGetStratumIS(bdlabel, 1, &faceIS));
2942   if (faceIS) {
2943     PetscCall(ISGetLocalSize(faceIS, &Nf));
2944     PetscCall(ISGetIndices(faceIS, &faces));
2945     for (PetscInt f = 0; f < Nf; ++f) {
2946       PetscReal vol, normal[3];
2947 
2948       PetscCall(DMPlexComputeCellGeometryFVM(dm, faces[f], &vol, NULL, normal));
2949       if (PetscAbsReal(normal[2]) < PETSC_SMALL) PetscCall(DMLabelSetValue(edgelabel, faces[f], 1));
2950     }
2951     PetscCall(ISRestoreIndices(faceIS, &faces));
2952     PetscCall(ISDestroy(&faceIS));
2953   }
2954   PetscCall(DMPlexLabelComplete(dm, bdlabel));
2955   PetscCall(DMPlexLabelComplete(dm, edgelabel));
2956   PetscFunctionReturn(PETSC_SUCCESS);
2957 }
2958 
2959 /*@
2960   DMPlexCreateHexCylinderMesh - Creates a mesh on the tensor product of the unit interval with the circle (cylinder) using hexahedra.
2961 
2962   Collective
2963 
2964   Input Parameters:
2965 + comm      - The communicator for the `DM` object
2966 . periodicZ - The boundary type for the Z direction
2967 - Nr        - The number of refinements to carry out
2968 
2969   Output Parameter:
2970 . dm - The `DM` object
2971 
2972   Level: beginner
2973 
2974   Note:
2975   Here is the output numbering looking from the bottom of the cylinder\:
2976 .vb
2977        17-----14
2978         |     |
2979         |  2  |
2980         |     |
2981  17-----8-----7-----14
2982   |     |     |     |
2983   |  3  |  0  |  1  |
2984   |     |     |     |
2985  19-----5-----6-----13
2986         |     |
2987         |  4  |
2988         |     |
2989        19-----13
2990 
2991  and up through the top
2992 
2993        18-----16
2994         |     |
2995         |  2  |
2996         |     |
2997  18----10----11-----16
2998   |     |     |     |
2999   |  3  |  0  |  1  |
3000   |     |     |     |
3001  20-----9----12-----15
3002         |     |
3003         |  4  |
3004         |     |
3005        20-----15
3006 .ve
3007 
3008 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
3009 @*/
3010 PetscErrorCode DMPlexCreateHexCylinderMesh(MPI_Comm comm, DMBoundaryType periodicZ, PetscInt Nr, DM *dm)
3011 {
3012   PetscFunctionBegin;
3013   PetscAssertPointer(dm, 4);
3014   PetscCall(DMCreate(comm, dm));
3015   PetscCall(DMSetType(*dm, DMPLEX));
3016   PetscCall(DMPlexCreateHexCylinderMesh_Internal(*dm, periodicZ, Nr));
3017   PetscFunctionReturn(PETSC_SUCCESS);
3018 }
3019 
3020 static PetscErrorCode DMPlexCreateWedgeCylinderMesh_Internal(DM dm, PetscInt n, PetscBool interpolate)
3021 {
3022   const PetscInt dim = 3;
3023   PetscInt       numCells, numVertices, v;
3024   PetscMPIInt    rank;
3025 
3026   PetscFunctionBegin;
3027   PetscCheck(n >= 0, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of wedges %" PetscInt_FMT " cannot be negative", n);
3028   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
3029   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
3030   PetscCall(DMSetDimension(dm, dim));
3031   /* Must create the celltype label here so that we do not automatically try to compute the types */
3032   PetscCall(DMCreateLabel(dm, "celltype"));
3033   /* Create topology */
3034   {
3035     PetscInt cone[6], c;
3036 
3037     numCells    = rank == 0 ? n : 0;
3038     numVertices = rank == 0 ? 2 * (n + 1) : 0;
3039     PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
3040     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 6));
3041     PetscCall(DMSetUp(dm));
3042     for (c = 0; c < numCells; c++) {
3043       cone[0] = c + n * 1;
3044       cone[1] = (c + 1) % n + n * 1;
3045       cone[2] = 0 + 3 * n;
3046       cone[3] = c + n * 2;
3047       cone[4] = (c + 1) % n + n * 2;
3048       cone[5] = 1 + 3 * n;
3049       PetscCall(DMPlexSetCone(dm, c, cone));
3050       PetscCall(DMPlexSetCellType(dm, c, DM_POLYTOPE_TRI_PRISM_TENSOR));
3051     }
3052     PetscCall(DMPlexSymmetrize(dm));
3053     PetscCall(DMPlexStratify(dm));
3054   }
3055   for (v = numCells; v < numCells + numVertices; ++v) PetscCall(DMPlexSetCellType(dm, v, DM_POLYTOPE_POINT));
3056   /* Create cylinder geometry */
3057   {
3058     Vec          coordinates;
3059     PetscSection coordSection;
3060     PetscScalar *coords;
3061     PetscInt     coordSize, c;
3062 
3063     /* Build coordinates */
3064     PetscCall(DMGetCoordinateSection(dm, &coordSection));
3065     PetscCall(PetscSectionSetNumFields(coordSection, 1));
3066     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
3067     PetscCall(PetscSectionSetChart(coordSection, numCells, numCells + numVertices));
3068     for (v = numCells; v < numCells + numVertices; ++v) {
3069       PetscCall(PetscSectionSetDof(coordSection, v, dim));
3070       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
3071     }
3072     PetscCall(PetscSectionSetUp(coordSection));
3073     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
3074     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
3075     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
3076     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
3077     PetscCall(VecSetBlockSize(coordinates, dim));
3078     PetscCall(VecSetType(coordinates, VECSTANDARD));
3079     PetscCall(VecGetArray(coordinates, &coords));
3080     for (c = 0; c < numCells; c++) {
3081       coords[(c + 0 * n) * dim + 0] = PetscCosReal(2.0 * c * PETSC_PI / n);
3082       coords[(c + 0 * n) * dim + 1] = PetscSinReal(2.0 * c * PETSC_PI / n);
3083       coords[(c + 0 * n) * dim + 2] = 1.0;
3084       coords[(c + 1 * n) * dim + 0] = PetscCosReal(2.0 * c * PETSC_PI / n);
3085       coords[(c + 1 * n) * dim + 1] = PetscSinReal(2.0 * c * PETSC_PI / n);
3086       coords[(c + 1 * n) * dim + 2] = 0.0;
3087     }
3088     if (rank == 0) {
3089       coords[(2 * n + 0) * dim + 0] = 0.0;
3090       coords[(2 * n + 0) * dim + 1] = 0.0;
3091       coords[(2 * n + 0) * dim + 2] = 1.0;
3092       coords[(2 * n + 1) * dim + 0] = 0.0;
3093       coords[(2 * n + 1) * dim + 1] = 0.0;
3094       coords[(2 * n + 1) * dim + 2] = 0.0;
3095     }
3096     PetscCall(VecRestoreArray(coordinates, &coords));
3097     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
3098     PetscCall(VecDestroy(&coordinates));
3099   }
3100   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
3101   /* Interpolate */
3102   if (interpolate) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
3103   PetscFunctionReturn(PETSC_SUCCESS);
3104 }
3105 
3106 /*@
3107   DMPlexCreateWedgeCylinderMesh - Creates a mesh on the tensor product of the unit interval with the circle (cylinder) using wedges.
3108 
3109   Collective
3110 
3111   Input Parameters:
3112 + comm        - The communicator for the `DM` object
3113 . n           - The number of wedges around the origin
3114 - interpolate - Create edges and faces
3115 
3116   Output Parameter:
3117 . dm - The `DM` object
3118 
3119   Level: beginner
3120 
3121 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateHexCylinderMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
3122 @*/
3123 PetscErrorCode DMPlexCreateWedgeCylinderMesh(MPI_Comm comm, PetscInt n, PetscBool interpolate, DM *dm)
3124 {
3125   PetscFunctionBegin;
3126   PetscAssertPointer(dm, 4);
3127   PetscCall(DMCreate(comm, dm));
3128   PetscCall(DMSetType(*dm, DMPLEX));
3129   PetscCall(DMPlexCreateWedgeCylinderMesh_Internal(*dm, n, interpolate));
3130   PetscFunctionReturn(PETSC_SUCCESS);
3131 }
3132 
3133 static inline PetscReal DiffNormReal(PetscInt dim, const PetscReal x[], const PetscReal y[])
3134 {
3135   PetscReal prod = 0.0;
3136   PetscInt  i;
3137   for (i = 0; i < dim; ++i) prod += PetscSqr(x[i] - y[i]);
3138   return PetscSqrtReal(prod);
3139 }
3140 
3141 static inline PetscReal DotReal(PetscInt dim, const PetscReal x[], const PetscReal y[])
3142 {
3143   PetscReal prod = 0.0;
3144   PetscInt  i;
3145   for (i = 0; i < dim; ++i) prod += x[i] * y[i];
3146   return prod;
3147 }
3148 
3149 /* The first constant is the sphere radius */
3150 static void snapToSphere(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
3151 {
3152   PetscReal r     = PetscRealPart(constants[0]);
3153   PetscReal norm2 = 0.0, fac;
3154   PetscInt  n     = uOff[1] - uOff[0], d;
3155 
3156   for (d = 0; d < n; ++d) norm2 += PetscSqr(PetscRealPart(u[d]));
3157   fac = r / PetscSqrtReal(norm2);
3158   for (d = 0; d < n; ++d) f0[d] = u[d] * fac;
3159 }
3160 
3161 static PetscErrorCode DMPlexCreateSphereMesh_Internal(DM dm, PetscInt dim, PetscBool simplex, PetscReal R)
3162 {
3163   const PetscInt embedDim = dim + 1;
3164   PetscSection   coordSection;
3165   Vec            coordinates;
3166   PetscScalar   *coords;
3167   PetscReal     *coordsIn;
3168   PetscInt       numCells, numEdges, numVerts = 0, firstVertex = 0, v, firstEdge, coordSize, d, e;
3169   PetscMPIInt    rank;
3170 
3171   PetscFunctionBegin;
3172   PetscValidLogicalCollectiveBool(dm, simplex, 3);
3173   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
3174   PetscCall(DMSetDimension(dm, dim));
3175   PetscCall(DMSetCoordinateDim(dm, dim + 1));
3176   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
3177   switch (dim) {
3178   case 1:
3179     numCells = 16;
3180     numVerts = numCells;
3181 
3182     // Build Topology
3183     PetscCall(DMPlexSetChart(dm, 0, numCells + numVerts));
3184     for (PetscInt c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, embedDim));
3185     PetscCall(DMSetUp(dm));
3186     for (PetscInt c = 0; c < numCells; ++c) {
3187       PetscInt cone[2];
3188 
3189       cone[0] = c + numCells;
3190       cone[1] = (c + 1) % numVerts + numCells;
3191       PetscCall(DMPlexSetCone(dm, c, cone));
3192     }
3193     PetscCall(DMPlexSymmetrize(dm));
3194     PetscCall(DMPlexStratify(dm));
3195     PetscCall(PetscMalloc1(numVerts * embedDim, &coordsIn));
3196     for (PetscInt v = 0; v < numVerts; ++v) {
3197       const PetscReal rad = 2. * PETSC_PI * v / numVerts;
3198 
3199       coordsIn[v * embedDim + 0] = PetscCosReal(rad);
3200       coordsIn[v * embedDim + 1] = PetscSinReal(rad);
3201     }
3202     break;
3203   case 2:
3204     if (simplex) {
3205       const PetscReal radius    = PetscSqrtReal(1 + PETSC_PHI * PETSC_PHI) / (1.0 + PETSC_PHI);
3206       const PetscReal edgeLen   = 2.0 / (1.0 + PETSC_PHI) * (R / radius);
3207       const PetscInt  degree    = 5;
3208       PetscReal       vertex[3] = {0.0, 1.0 / (1.0 + PETSC_PHI), PETSC_PHI / (1.0 + PETSC_PHI)};
3209       PetscInt        s[3]      = {1, 1, 1};
3210       PetscInt        cone[3];
3211       PetscInt       *graph;
3212 
3213       vertex[0] *= R / radius;
3214       vertex[1] *= R / radius;
3215       vertex[2] *= R / radius;
3216       numCells    = rank == 0 ? 20 : 0;
3217       numVerts    = rank == 0 ? 12 : 0;
3218       firstVertex = numCells;
3219       /* Use icosahedron, which for a R-sphere has coordinates which are all cyclic permutations of
3220 
3221            (0, \pm 1/\phi+1, \pm \phi/\phi+1)
3222 
3223          where \phi^2 - \phi - 1 = 0, meaning \phi is the golden ratio \frac{1 + \sqrt{5}}{2}. The edge
3224          length is then given by 2/(1+\phi) = 2 * 0.38197 = 0.76393.
3225       */
3226       /* Construct vertices */
3227       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
3228       if (rank == 0) {
3229         for (PetscInt p = 0, i = 0; p < embedDim; ++p) {
3230           for (s[1] = -1; s[1] < 2; s[1] += 2) {
3231             for (s[2] = -1; s[2] < 2; s[2] += 2) {
3232               for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[(d + p) % embedDim] * vertex[(d + p) % embedDim];
3233               ++i;
3234             }
3235           }
3236         }
3237       }
3238       /* Construct graph */
3239       PetscCall(PetscCalloc1(numVerts * numVerts, &graph));
3240       for (PetscInt i = 0; i < numVerts; ++i) {
3241         PetscInt k = 0;
3242         for (PetscInt j = 0; j < numVerts; ++j) {
3243           if (PetscAbsReal(DiffNormReal(embedDim, &coordsIn[i * embedDim], &coordsIn[j * embedDim]) - edgeLen) < PETSC_SMALL) {
3244             graph[i * numVerts + j] = 1;
3245             ++k;
3246           }
3247         }
3248         PetscCheck(k == degree, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid icosahedron, vertex %" PetscInt_FMT " degree %" PetscInt_FMT " != %" PetscInt_FMT, i, k, degree);
3249       }
3250       /* Build Topology */
3251       PetscCall(DMPlexSetChart(dm, 0, numCells + numVerts));
3252       for (PetscInt c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, embedDim));
3253       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
3254       /* Cells */
3255       for (PetscInt i = 0, c = 0; i < numVerts; ++i) {
3256         for (PetscInt j = 0; j < i; ++j) {
3257           for (PetscInt k = 0; k < j; ++k) {
3258             if (graph[i * numVerts + j] && graph[j * numVerts + k] && graph[k * numVerts + i]) {
3259               cone[0] = firstVertex + i;
3260               cone[1] = firstVertex + j;
3261               cone[2] = firstVertex + k;
3262               /* Check orientation */
3263               {
3264                 const PetscInt epsilon[3][3][3] = {
3265                   {{0, 0, 0},  {0, 0, 1},  {0, -1, 0}},
3266                   {{0, 0, -1}, {0, 0, 0},  {1, 0, 0} },
3267                   {{0, 1, 0},  {-1, 0, 0}, {0, 0, 0} }
3268                 };
3269                 PetscReal normal[3];
3270                 PetscInt  e, f;
3271 
3272                 for (d = 0; d < embedDim; ++d) {
3273                   normal[d] = 0.0;
3274                   for (e = 0; e < embedDim; ++e) {
3275                     for (f = 0; f < embedDim; ++f) normal[d] += epsilon[d][e][f] * (coordsIn[j * embedDim + e] - coordsIn[i * embedDim + e]) * (coordsIn[k * embedDim + f] - coordsIn[i * embedDim + f]);
3276                   }
3277                 }
3278                 if (DotReal(embedDim, normal, &coordsIn[i * embedDim]) < 0) {
3279                   PetscInt tmp = cone[1];
3280                   cone[1]      = cone[2];
3281                   cone[2]      = tmp;
3282                 }
3283               }
3284               PetscCall(DMPlexSetCone(dm, c++, cone));
3285             }
3286           }
3287         }
3288       }
3289       PetscCall(DMPlexSymmetrize(dm));
3290       PetscCall(DMPlexStratify(dm));
3291       PetscCall(PetscFree(graph));
3292     } else {
3293       /*
3294         12-21--13
3295          |     |
3296         25  4  24
3297          |     |
3298   12-25--9-16--8-24--13
3299    |     |     |     |
3300   23  5 17  0 15  3  22
3301    |     |     |     |
3302   10-20--6-14--7-19--11
3303          |     |
3304         20  1  19
3305          |     |
3306         10-18--11
3307          |     |
3308         23  2  22
3309          |     |
3310         12-21--13
3311        */
3312       PetscInt cone[4], ornt[4];
3313 
3314       numCells    = rank == 0 ? 6 : 0;
3315       numEdges    = rank == 0 ? 12 : 0;
3316       numVerts    = rank == 0 ? 8 : 0;
3317       firstVertex = numCells;
3318       firstEdge   = numCells + numVerts;
3319       /* Build Topology */
3320       PetscCall(DMPlexSetChart(dm, 0, numCells + numEdges + numVerts));
3321       for (PetscInt c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 4));
3322       for (e = firstEdge; e < firstEdge + numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
3323       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
3324       if (rank == 0) {
3325         /* Cell 0 */
3326         cone[0] = 14;
3327         cone[1] = 15;
3328         cone[2] = 16;
3329         cone[3] = 17;
3330         PetscCall(DMPlexSetCone(dm, 0, cone));
3331         ornt[0] = 0;
3332         ornt[1] = 0;
3333         ornt[2] = 0;
3334         ornt[3] = 0;
3335         PetscCall(DMPlexSetConeOrientation(dm, 0, ornt));
3336         /* Cell 1 */
3337         cone[0] = 18;
3338         cone[1] = 19;
3339         cone[2] = 14;
3340         cone[3] = 20;
3341         PetscCall(DMPlexSetCone(dm, 1, cone));
3342         ornt[0] = 0;
3343         ornt[1] = 0;
3344         ornt[2] = -1;
3345         ornt[3] = 0;
3346         PetscCall(DMPlexSetConeOrientation(dm, 1, ornt));
3347         /* Cell 2 */
3348         cone[0] = 21;
3349         cone[1] = 22;
3350         cone[2] = 18;
3351         cone[3] = 23;
3352         PetscCall(DMPlexSetCone(dm, 2, cone));
3353         ornt[0] = 0;
3354         ornt[1] = 0;
3355         ornt[2] = -1;
3356         ornt[3] = 0;
3357         PetscCall(DMPlexSetConeOrientation(dm, 2, ornt));
3358         /* Cell 3 */
3359         cone[0] = 19;
3360         cone[1] = 22;
3361         cone[2] = 24;
3362         cone[3] = 15;
3363         PetscCall(DMPlexSetCone(dm, 3, cone));
3364         ornt[0] = -1;
3365         ornt[1] = -1;
3366         ornt[2] = 0;
3367         ornt[3] = -1;
3368         PetscCall(DMPlexSetConeOrientation(dm, 3, ornt));
3369         /* Cell 4 */
3370         cone[0] = 16;
3371         cone[1] = 24;
3372         cone[2] = 21;
3373         cone[3] = 25;
3374         PetscCall(DMPlexSetCone(dm, 4, cone));
3375         ornt[0] = -1;
3376         ornt[1] = -1;
3377         ornt[2] = -1;
3378         ornt[3] = 0;
3379         PetscCall(DMPlexSetConeOrientation(dm, 4, ornt));
3380         /* Cell 5 */
3381         cone[0] = 20;
3382         cone[1] = 17;
3383         cone[2] = 25;
3384         cone[3] = 23;
3385         PetscCall(DMPlexSetCone(dm, 5, cone));
3386         ornt[0] = -1;
3387         ornt[1] = -1;
3388         ornt[2] = -1;
3389         ornt[3] = -1;
3390         PetscCall(DMPlexSetConeOrientation(dm, 5, ornt));
3391         /* Edges */
3392         cone[0] = 6;
3393         cone[1] = 7;
3394         PetscCall(DMPlexSetCone(dm, 14, cone));
3395         cone[0] = 7;
3396         cone[1] = 8;
3397         PetscCall(DMPlexSetCone(dm, 15, cone));
3398         cone[0] = 8;
3399         cone[1] = 9;
3400         PetscCall(DMPlexSetCone(dm, 16, cone));
3401         cone[0] = 9;
3402         cone[1] = 6;
3403         PetscCall(DMPlexSetCone(dm, 17, cone));
3404         cone[0] = 10;
3405         cone[1] = 11;
3406         PetscCall(DMPlexSetCone(dm, 18, cone));
3407         cone[0] = 11;
3408         cone[1] = 7;
3409         PetscCall(DMPlexSetCone(dm, 19, cone));
3410         cone[0] = 6;
3411         cone[1] = 10;
3412         PetscCall(DMPlexSetCone(dm, 20, cone));
3413         cone[0] = 12;
3414         cone[1] = 13;
3415         PetscCall(DMPlexSetCone(dm, 21, cone));
3416         cone[0] = 13;
3417         cone[1] = 11;
3418         PetscCall(DMPlexSetCone(dm, 22, cone));
3419         cone[0] = 10;
3420         cone[1] = 12;
3421         PetscCall(DMPlexSetCone(dm, 23, cone));
3422         cone[0] = 13;
3423         cone[1] = 8;
3424         PetscCall(DMPlexSetCone(dm, 24, cone));
3425         cone[0] = 12;
3426         cone[1] = 9;
3427         PetscCall(DMPlexSetCone(dm, 25, cone));
3428       }
3429       PetscCall(DMPlexSymmetrize(dm));
3430       PetscCall(DMPlexStratify(dm));
3431       /* Build coordinates */
3432       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
3433       if (rank == 0) {
3434         coordsIn[0 * embedDim + 0] = -R;
3435         coordsIn[0 * embedDim + 1] = R;
3436         coordsIn[0 * embedDim + 2] = -R;
3437         coordsIn[1 * embedDim + 0] = R;
3438         coordsIn[1 * embedDim + 1] = R;
3439         coordsIn[1 * embedDim + 2] = -R;
3440         coordsIn[2 * embedDim + 0] = R;
3441         coordsIn[2 * embedDim + 1] = -R;
3442         coordsIn[2 * embedDim + 2] = -R;
3443         coordsIn[3 * embedDim + 0] = -R;
3444         coordsIn[3 * embedDim + 1] = -R;
3445         coordsIn[3 * embedDim + 2] = -R;
3446         coordsIn[4 * embedDim + 0] = -R;
3447         coordsIn[4 * embedDim + 1] = R;
3448         coordsIn[4 * embedDim + 2] = R;
3449         coordsIn[5 * embedDim + 0] = R;
3450         coordsIn[5 * embedDim + 1] = R;
3451         coordsIn[5 * embedDim + 2] = R;
3452         coordsIn[6 * embedDim + 0] = -R;
3453         coordsIn[6 * embedDim + 1] = -R;
3454         coordsIn[6 * embedDim + 2] = R;
3455         coordsIn[7 * embedDim + 0] = R;
3456         coordsIn[7 * embedDim + 1] = -R;
3457         coordsIn[7 * embedDim + 2] = R;
3458       }
3459     }
3460     break;
3461   case 3:
3462     if (simplex) {
3463       const PetscReal edgeLen         = 1.0 / PETSC_PHI;
3464       PetscReal       vertexA[4]      = {0.5, 0.5, 0.5, 0.5};
3465       PetscReal       vertexB[4]      = {1.0, 0.0, 0.0, 0.0};
3466       PetscReal       vertexC[4]      = {0.5, 0.5 * PETSC_PHI, 0.5 / PETSC_PHI, 0.0};
3467       const PetscInt  degree          = 12;
3468       PetscInt        s[4]            = {1, 1, 1};
3469       PetscInt        evenPerm[12][4] = {
3470         {0, 1, 2, 3},
3471         {0, 2, 3, 1},
3472         {0, 3, 1, 2},
3473         {1, 0, 3, 2},
3474         {1, 2, 0, 3},
3475         {1, 3, 2, 0},
3476         {2, 0, 1, 3},
3477         {2, 1, 3, 0},
3478         {2, 3, 0, 1},
3479         {3, 0, 2, 1},
3480         {3, 1, 0, 2},
3481         {3, 2, 1, 0}
3482       };
3483       PetscInt  cone[4];
3484       PetscInt *graph, p, i, j, k, l;
3485 
3486       vertexA[0] *= R;
3487       vertexA[1] *= R;
3488       vertexA[2] *= R;
3489       vertexA[3] *= R;
3490       vertexB[0] *= R;
3491       vertexB[1] *= R;
3492       vertexB[2] *= R;
3493       vertexB[3] *= R;
3494       vertexC[0] *= R;
3495       vertexC[1] *= R;
3496       vertexC[2] *= R;
3497       vertexC[3] *= R;
3498       numCells    = rank == 0 ? 600 : 0;
3499       numVerts    = rank == 0 ? 120 : 0;
3500       firstVertex = numCells;
3501       /* Use the 600-cell, which for a unit sphere has coordinates which are
3502 
3503            1/2 (\pm 1, \pm 1,    \pm 1, \pm 1)                          16
3504                (\pm 1,    0,       0,      0)  all cyclic permutations   8
3505            1/2 (\pm 1, \pm phi, \pm 1/phi, 0)  all even permutations    96
3506 
3507          where \phi^2 - \phi - 1 = 0, meaning \phi is the golden ratio \frac{1 + \sqrt{5}}{2}. The edge
3508          length is then given by 1/\phi = 0.61803.
3509 
3510          http://buzzard.pugetsound.edu/sage-practice/ch03s03.html
3511          http://mathworld.wolfram.com/600-Cell.html
3512       */
3513       /* Construct vertices */
3514       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
3515       i = 0;
3516       if (rank == 0) {
3517         for (s[0] = -1; s[0] < 2; s[0] += 2) {
3518           for (s[1] = -1; s[1] < 2; s[1] += 2) {
3519             for (s[2] = -1; s[2] < 2; s[2] += 2) {
3520               for (s[3] = -1; s[3] < 2; s[3] += 2) {
3521                 for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[d] * vertexA[d];
3522                 ++i;
3523               }
3524             }
3525           }
3526         }
3527         for (p = 0; p < embedDim; ++p) {
3528           s[1] = s[2] = s[3] = 1;
3529           for (s[0] = -1; s[0] < 2; s[0] += 2) {
3530             for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[(d + p) % embedDim] * vertexB[(d + p) % embedDim];
3531             ++i;
3532           }
3533         }
3534         for (p = 0; p < 12; ++p) {
3535           s[3] = 1;
3536           for (s[0] = -1; s[0] < 2; s[0] += 2) {
3537             for (s[1] = -1; s[1] < 2; s[1] += 2) {
3538               for (s[2] = -1; s[2] < 2; s[2] += 2) {
3539                 for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[evenPerm[p][d]] * vertexC[evenPerm[p][d]];
3540                 ++i;
3541               }
3542             }
3543           }
3544         }
3545       }
3546       PetscCheck(i == numVerts, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid 600-cell, vertices %" PetscInt_FMT " != %" PetscInt_FMT, i, numVerts);
3547       /* Construct graph */
3548       PetscCall(PetscCalloc1(numVerts * numVerts, &graph));
3549       for (i = 0; i < numVerts; ++i) {
3550         for (j = 0, k = 0; j < numVerts; ++j) {
3551           if (PetscAbsReal(DiffNormReal(embedDim, &coordsIn[i * embedDim], &coordsIn[j * embedDim]) - edgeLen) < PETSC_SMALL) {
3552             graph[i * numVerts + j] = 1;
3553             ++k;
3554           }
3555         }
3556         PetscCheck(k == degree, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid 600-cell, vertex %" PetscInt_FMT " degree %" PetscInt_FMT " != %" PetscInt_FMT, i, k, degree);
3557       }
3558       /* Build Topology */
3559       PetscCall(DMPlexSetChart(dm, 0, numCells + numVerts));
3560       for (PetscInt c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, embedDim));
3561       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
3562       /* Cells */
3563       if (rank == 0) {
3564         for (PetscInt i = 0, c = 0; i < numVerts; ++i) {
3565           for (j = 0; j < i; ++j) {
3566             for (k = 0; k < j; ++k) {
3567               for (l = 0; l < k; ++l) {
3568                 if (graph[i * numVerts + j] && graph[j * numVerts + k] && graph[k * numVerts + i] && graph[l * numVerts + i] && graph[l * numVerts + j] && graph[l * numVerts + k]) {
3569                   cone[0] = firstVertex + i;
3570                   cone[1] = firstVertex + j;
3571                   cone[2] = firstVertex + k;
3572                   cone[3] = firstVertex + l;
3573                   /* Check orientation: https://ef.gy/linear-algebra:normal-vectors-in-higher-dimensional-spaces */
3574                   {
3575                     const PetscInt epsilon[4][4][4][4] = {
3576                       {{{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}},  {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 1}, {0, 0, -1, 0}}, {{0, 0, 0, 0}, {0, 0, 0, -1}, {0, 0, 0, 0}, {0, 1, 0, 0}}, {{0, 0, 0, 0}, {0, 0, 1, 0}, {0, -1, 0, 0}, {0, 0, 0, 0}}},
3577 
3578                       {{{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, -1}, {0, 0, 1, 0}}, {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}},  {{0, 0, 0, 1}, {0, 0, 0, 0}, {0, 0, 0, 0}, {-1, 0, 0, 0}}, {{0, 0, -1, 0}, {0, 0, 0, 0}, {1, 0, 0, 0}, {0, 0, 0, 0}}},
3579 
3580                       {{{0, 0, 0, 0}, {0, 0, 0, 1}, {0, 0, 0, 0}, {0, -1, 0, 0}}, {{0, 0, 0, -1}, {0, 0, 0, 0}, {0, 0, 0, 0}, {1, 0, 0, 0}}, {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}},  {{0, 1, 0, 0}, {-1, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}}},
3581 
3582                       {{{0, 0, 0, 0}, {0, 0, -1, 0}, {0, 1, 0, 0}, {0, 0, 0, 0}}, {{0, 0, 1, 0}, {0, 0, 0, 0}, {-1, 0, 0, 0}, {0, 0, 0, 0}}, {{0, -1, 0, 0}, {1, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}}, {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}} }
3583                     };
3584                     PetscReal normal[4];
3585                     PetscInt  e, f, g;
3586 
3587                     for (d = 0; d < embedDim; ++d) {
3588                       normal[d] = 0.0;
3589                       for (e = 0; e < embedDim; ++e) {
3590                         for (f = 0; f < embedDim; ++f) {
3591                           for (g = 0; g < embedDim; ++g) {
3592                             normal[d] += epsilon[d][e][f][g] * (coordsIn[j * embedDim + e] - coordsIn[i * embedDim + e]) * (coordsIn[k * embedDim + f] - coordsIn[i * embedDim + f]) * (coordsIn[l * embedDim + f] - coordsIn[i * embedDim + f]);
3593                           }
3594                         }
3595                       }
3596                     }
3597                     if (DotReal(embedDim, normal, &coordsIn[i * embedDim]) < 0) {
3598                       PetscInt tmp = cone[1];
3599                       cone[1]      = cone[2];
3600                       cone[2]      = tmp;
3601                     }
3602                   }
3603                   PetscCall(DMPlexSetCone(dm, c++, cone));
3604                 }
3605               }
3606             }
3607           }
3608         }
3609       }
3610       PetscCall(DMPlexSymmetrize(dm));
3611       PetscCall(DMPlexStratify(dm));
3612       PetscCall(PetscFree(graph));
3613     }
3614     break;
3615   default:
3616     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Unsupported dimension for sphere: %" PetscInt_FMT, dim);
3617   }
3618   /* Create coordinates */
3619   PetscCall(DMGetCoordinateSection(dm, &coordSection));
3620   PetscCall(PetscSectionSetNumFields(coordSection, 1));
3621   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, embedDim));
3622   PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numVerts));
3623   for (v = firstVertex; v < firstVertex + numVerts; ++v) {
3624     PetscCall(PetscSectionSetDof(coordSection, v, embedDim));
3625     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, embedDim));
3626   }
3627   PetscCall(PetscSectionSetUp(coordSection));
3628   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
3629   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
3630   PetscCall(VecSetBlockSize(coordinates, embedDim));
3631   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
3632   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
3633   PetscCall(VecSetType(coordinates, VECSTANDARD));
3634   PetscCall(VecGetArray(coordinates, &coords));
3635   for (v = 0; v < numVerts; ++v)
3636     for (d = 0; d < embedDim; ++d) coords[v * embedDim + d] = coordsIn[v * embedDim + d];
3637   PetscCall(VecRestoreArray(coordinates, &coords));
3638   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
3639   PetscCall(VecDestroy(&coordinates));
3640   PetscCall(PetscFree(coordsIn));
3641   {
3642     DM          cdm;
3643     PetscDS     cds;
3644     PetscScalar c = R;
3645 
3646     PetscCall(DMPlexCreateCoordinateSpace(dm, 1, PETSC_TRUE, snapToSphere));
3647     PetscCall(DMGetCoordinateDM(dm, &cdm));
3648     PetscCall(DMGetDS(cdm, &cds));
3649     PetscCall(PetscDSSetConstants(cds, 1, &c));
3650   }
3651   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
3652   /* Wait for coordinate creation before doing in-place modification */
3653   if (simplex) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
3654   PetscFunctionReturn(PETSC_SUCCESS);
3655 }
3656 
3657 typedef void (*TPSEvaluateFunc)(const PetscReal[], PetscReal *, PetscReal[], PetscReal (*)[3]);
3658 
3659 /*
3660  The Schwarz P implicit surface is
3661 
3662      f(x) = cos(x0) + cos(x1) + cos(x2) = 0
3663 */
3664 static void TPSEvaluate_SchwarzP(const PetscReal y[3], PetscReal *f, PetscReal grad[], PetscReal (*hess)[3])
3665 {
3666   PetscReal c[3] = {PetscCosReal(y[0] * PETSC_PI), PetscCosReal(y[1] * PETSC_PI), PetscCosReal(y[2] * PETSC_PI)};
3667   PetscReal g[3] = {-PetscSinReal(y[0] * PETSC_PI), -PetscSinReal(y[1] * PETSC_PI), -PetscSinReal(y[2] * PETSC_PI)};
3668   f[0]           = c[0] + c[1] + c[2];
3669   for (PetscInt i = 0; i < 3; i++) {
3670     grad[i] = PETSC_PI * g[i];
3671     for (PetscInt j = 0; j < 3; j++) hess[i][j] = (i == j) ? -PetscSqr(PETSC_PI) * c[i] : 0.;
3672   }
3673 }
3674 
3675 // u[] is a tentative normal on input. Replace with the implicit function gradient in the same direction
3676 static PetscErrorCode TPSExtrudeNormalFunc_SchwarzP(PetscInt dim, PetscReal time, const PetscReal x[], PetscInt r, PetscScalar u[], void *ctx)
3677 {
3678   for (PetscInt i = 0; i < 3; i++) u[i] = -PETSC_PI * PetscSinReal(x[i] * PETSC_PI);
3679   return PETSC_SUCCESS;
3680 }
3681 
3682 /*
3683  The Gyroid implicit surface is
3684 
3685  f(x,y,z) = sin(pi * x) * cos (pi * (y + 1/2))  + sin(pi * (y + 1/2)) * cos(pi * (z + 1/4)) + sin(pi * (z + 1/4)) * cos(pi * x)
3686 
3687 */
3688 static void TPSEvaluate_Gyroid(const PetscReal y[3], PetscReal *f, PetscReal grad[], PetscReal (*hess)[3])
3689 {
3690   PetscReal s[3] = {PetscSinReal(PETSC_PI * y[0]), PetscSinReal(PETSC_PI * (y[1] + .5)), PetscSinReal(PETSC_PI * (y[2] + .25))};
3691   PetscReal c[3] = {PetscCosReal(PETSC_PI * y[0]), PetscCosReal(PETSC_PI * (y[1] + .5)), PetscCosReal(PETSC_PI * (y[2] + .25))};
3692   f[0]           = s[0] * c[1] + s[1] * c[2] + s[2] * c[0];
3693   grad[0]        = PETSC_PI * (c[0] * c[1] - s[2] * s[0]);
3694   grad[1]        = PETSC_PI * (c[1] * c[2] - s[0] * s[1]);
3695   grad[2]        = PETSC_PI * (c[2] * c[0] - s[1] * s[2]);
3696   hess[0][0]     = -PetscSqr(PETSC_PI) * (s[0] * c[1] + s[2] * c[0]);
3697   hess[0][1]     = -PetscSqr(PETSC_PI) * (c[0] * s[1]);
3698   hess[0][2]     = -PetscSqr(PETSC_PI) * (c[2] * s[0]);
3699   hess[1][0]     = -PetscSqr(PETSC_PI) * (s[1] * c[2] + s[0] * c[1]);
3700   hess[1][1]     = -PetscSqr(PETSC_PI) * (c[1] * s[2]);
3701   hess[2][2]     = -PetscSqr(PETSC_PI) * (c[0] * s[1]);
3702   hess[2][0]     = -PetscSqr(PETSC_PI) * (s[2] * c[0] + s[1] * c[2]);
3703   hess[2][1]     = -PetscSqr(PETSC_PI) * (c[2] * s[0]);
3704   hess[2][2]     = -PetscSqr(PETSC_PI) * (c[1] * s[2]);
3705 }
3706 
3707 // u[] is a tentative normal on input. Replace with the implicit function gradient in the same direction
3708 static PetscErrorCode TPSExtrudeNormalFunc_Gyroid(PetscInt dim, PetscReal time, const PetscReal x[], PetscInt r, PetscScalar u[], void *ctx)
3709 {
3710   PetscReal s[3] = {PetscSinReal(PETSC_PI * x[0]), PetscSinReal(PETSC_PI * (x[1] + .5)), PetscSinReal(PETSC_PI * (x[2] + .25))};
3711   PetscReal c[3] = {PetscCosReal(PETSC_PI * x[0]), PetscCosReal(PETSC_PI * (x[1] + .5)), PetscCosReal(PETSC_PI * (x[2] + .25))};
3712   u[0]           = PETSC_PI * (c[0] * c[1] - s[2] * s[0]);
3713   u[1]           = PETSC_PI * (c[1] * c[2] - s[0] * s[1]);
3714   u[2]           = PETSC_PI * (c[2] * c[0] - s[1] * s[2]);
3715   return PETSC_SUCCESS;
3716 }
3717 
3718 /*
3719    We wish to solve
3720 
3721          min_y || y - x ||^2  subject to f(y) = 0
3722 
3723    Let g(y) = grad(f).  The minimization problem is equivalent to asking to satisfy
3724    f(y) = 0 and (y-x) is parallel to g(y).  We do this by using Householder QR to obtain a basis for the
3725    tangent space and ask for both components in the tangent space to be zero.
3726 
3727    Take g to be a column vector and compute the "full QR" factorization Q R = g,
3728    where Q = I - 2 n n^T is a symmetric orthogonal matrix.
3729    The first column of Q is parallel to g so the remaining two columns span the null space.
3730    Let Qn = Q[:,1:] be those remaining columns.  Then Qn Qn^T is an orthogonal projector into the tangent space.
3731    Since Q is symmetric, this is equivalent to multiplying by Q and taking the last two entries.
3732    In total, we have a system of 3 equations in 3 unknowns:
3733 
3734      f(y) = 0                       1 equation
3735      Qn^T (y - x) = 0               2 equations
3736 
3737    Here, we compute the residual and Jacobian of this system.
3738 */
3739 static void TPSNearestPointResJac(TPSEvaluateFunc feval, const PetscScalar x[], const PetscScalar y[], PetscScalar res[], PetscScalar J[])
3740 {
3741   PetscReal yreal[3] = {PetscRealPart(y[0]), PetscRealPart(y[1]), PetscRealPart(y[2])};
3742   PetscReal d[3]     = {PetscRealPart(y[0] - x[0]), PetscRealPart(y[1] - x[1]), PetscRealPart(y[2] - x[2])};
3743   PetscReal f, grad[3], n[3], norm, norm_y[3], nd, nd_y[3], sign;
3744   PetscReal n_y[3][3] = {
3745     {0, 0, 0},
3746     {0, 0, 0},
3747     {0, 0, 0}
3748   };
3749 
3750   feval(yreal, &f, grad, n_y);
3751 
3752   for (PetscInt i = 0; i < 3; i++) n[i] = grad[i];
3753   norm = PetscSqrtReal(PetscSqr(n[0]) + PetscSqr(n[1]) + PetscSqr(n[2]));
3754   for (PetscInt i = 0; i < 3; i++) norm_y[i] = 1. / norm * n[i] * n_y[i][i];
3755 
3756   // Define the Householder reflector
3757   sign = n[0] >= 0 ? 1. : -1.;
3758   n[0] += norm * sign;
3759   for (PetscInt i = 0; i < 3; i++) n_y[0][i] += norm_y[i] * sign;
3760 
3761   norm      = PetscSqrtReal(PetscSqr(n[0]) + PetscSqr(n[1]) + PetscSqr(n[2]));
3762   norm_y[0] = 1. / norm * (n[0] * n_y[0][0]);
3763   norm_y[1] = 1. / norm * (n[0] * n_y[0][1] + n[1] * n_y[1][1]);
3764   norm_y[2] = 1. / norm * (n[0] * n_y[0][2] + n[2] * n_y[2][2]);
3765 
3766   for (PetscInt i = 0; i < 3; i++) {
3767     n[i] /= norm;
3768     for (PetscInt j = 0; j < 3; j++) {
3769       // note that n[i] is n_old[i]/norm when executing the code below
3770       n_y[i][j] = n_y[i][j] / norm - n[i] / norm * norm_y[j];
3771     }
3772   }
3773 
3774   nd = n[0] * d[0] + n[1] * d[1] + n[2] * d[2];
3775   for (PetscInt i = 0; i < 3; i++) nd_y[i] = n[i] + n_y[0][i] * d[0] + n_y[1][i] * d[1] + n_y[2][i] * d[2];
3776 
3777   res[0] = f;
3778   res[1] = d[1] - 2 * n[1] * nd;
3779   res[2] = d[2] - 2 * n[2] * nd;
3780   // J[j][i] is J_{ij} (column major)
3781   for (PetscInt j = 0; j < 3; j++) {
3782     J[0 + j * 3] = grad[j];
3783     J[1 + j * 3] = (j == 1) * 1. - 2 * (n_y[1][j] * nd + n[1] * nd_y[j]);
3784     J[2 + j * 3] = (j == 2) * 1. - 2 * (n_y[2][j] * nd + n[2] * nd_y[j]);
3785   }
3786 }
3787 
3788 /*
3789    Project x to the nearest point on the implicit surface using Newton's method.
3790 */
3791 static PetscErrorCode TPSNearestPoint(TPSEvaluateFunc feval, PetscScalar x[])
3792 {
3793   PetscScalar y[3] = {x[0], x[1], x[2]}; // Initial guess
3794 
3795   PetscFunctionBegin;
3796   for (PetscInt iter = 0; iter < 10; iter++) {
3797     PetscScalar res[3], J[9];
3798     PetscReal   resnorm;
3799     TPSNearestPointResJac(feval, x, y, res, J);
3800     resnorm = PetscSqrtReal(PetscSqr(PetscRealPart(res[0])) + PetscSqr(PetscRealPart(res[1])) + PetscSqr(PetscRealPart(res[2])));
3801     if (0) { // Turn on this monitor if you need to confirm quadratic convergence
3802       PetscCall(PetscPrintf(PETSC_COMM_SELF, "[%" PetscInt_FMT "] res [%g %g %g]\n", iter, (double)PetscRealPart(res[0]), (double)PetscRealPart(res[1]), (double)PetscRealPart(res[2])));
3803     }
3804     if (resnorm < PETSC_SMALL) break;
3805 
3806     // Take the Newton step
3807     PetscCall(PetscKernel_A_gets_inverse_A_3(J, 0., PETSC_FALSE, NULL));
3808     PetscKernel_v_gets_v_minus_A_times_w_3(y, J, res);
3809   }
3810   for (PetscInt i = 0; i < 3; i++) x[i] = y[i];
3811   PetscFunctionReturn(PETSC_SUCCESS);
3812 }
3813 
3814 const char *const DMPlexTPSTypes[] = {"SCHWARZ_P", "GYROID", "DMPlexTPSType", "DMPLEX_TPS_", NULL};
3815 
3816 static PetscErrorCode DMPlexCreateTPSMesh_Internal(DM dm, DMPlexTPSType tpstype, const PetscInt extent[], const DMBoundaryType periodic[], PetscBool tps_distribute, PetscInt refinements, PetscInt layers, PetscReal thickness)
3817 {
3818   PetscMPIInt rank;
3819   PetscInt    topoDim = 2, spaceDim = 3, numFaces = 0, numVertices = 0, numEdges = 0;
3820   PetscInt (*edges)[2] = NULL, *edgeSets = NULL;
3821   PetscInt           *cells_flat = NULL;
3822   PetscReal          *vtxCoords  = NULL;
3823   TPSEvaluateFunc     evalFunc   = NULL;
3824   PetscSimplePointFn *normalFunc = NULL;
3825   DMLabel             label;
3826 
3827   PetscFunctionBegin;
3828   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
3829   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
3830   PetscCheck((layers != 0) ^ (thickness == 0.), PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_INCOMP, "Layers %" PetscInt_FMT " must be nonzero iff thickness %g is nonzero", layers, (double)thickness);
3831   switch (tpstype) {
3832   case DMPLEX_TPS_SCHWARZ_P:
3833     PetscCheck(!periodic || (periodic[0] == DM_BOUNDARY_NONE && periodic[1] == DM_BOUNDARY_NONE && periodic[2] == DM_BOUNDARY_NONE), PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Schwarz P does not support periodic meshes");
3834     if (rank == 0) {
3835       PetscInt (*cells)[6][4][4] = NULL; // [junction, junction-face, cell, conn]
3836       PetscInt  Njunctions = 0, Ncuts = 0, Npipes[3], vcount;
3837       PetscReal L = 1;
3838 
3839       Npipes[0]   = (extent[0] + 1) * extent[1] * extent[2];
3840       Npipes[1]   = extent[0] * (extent[1] + 1) * extent[2];
3841       Npipes[2]   = extent[0] * extent[1] * (extent[2] + 1);
3842       Njunctions  = extent[0] * extent[1] * extent[2];
3843       Ncuts       = 2 * (extent[0] * extent[1] + extent[1] * extent[2] + extent[2] * extent[0]);
3844       numVertices = 4 * (Npipes[0] + Npipes[1] + Npipes[2]) + 8 * Njunctions;
3845       PetscCall(PetscMalloc1(3 * numVertices, &vtxCoords));
3846       PetscCall(PetscMalloc1(Njunctions, &cells));
3847       PetscCall(PetscMalloc1(Ncuts * 4, &edges));
3848       PetscCall(PetscMalloc1(Ncuts * 4, &edgeSets));
3849       // x-normal pipes
3850       vcount = 0;
3851       for (PetscInt i = 0; i < extent[0] + 1; i++) {
3852         for (PetscInt j = 0; j < extent[1]; j++) {
3853           for (PetscInt k = 0; k < extent[2]; k++) {
3854             for (PetscInt l = 0; l < 4; l++) {
3855               vtxCoords[vcount++] = (2 * i - 1) * L;
3856               vtxCoords[vcount++] = 2 * j * L + PetscCosReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
3857               vtxCoords[vcount++] = 2 * k * L + PetscSinReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
3858             }
3859           }
3860         }
3861       }
3862       // y-normal pipes
3863       for (PetscInt i = 0; i < extent[0]; i++) {
3864         for (PetscInt j = 0; j < extent[1] + 1; j++) {
3865           for (PetscInt k = 0; k < extent[2]; k++) {
3866             for (PetscInt l = 0; l < 4; l++) {
3867               vtxCoords[vcount++] = 2 * i * L + PetscSinReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
3868               vtxCoords[vcount++] = (2 * j - 1) * L;
3869               vtxCoords[vcount++] = 2 * k * L + PetscCosReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
3870             }
3871           }
3872         }
3873       }
3874       // z-normal pipes
3875       for (PetscInt i = 0; i < extent[0]; i++) {
3876         for (PetscInt j = 0; j < extent[1]; j++) {
3877           for (PetscInt k = 0; k < extent[2] + 1; k++) {
3878             for (PetscInt l = 0; l < 4; l++) {
3879               vtxCoords[vcount++] = 2 * i * L + PetscCosReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
3880               vtxCoords[vcount++] = 2 * j * L + PetscSinReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
3881               vtxCoords[vcount++] = (2 * k - 1) * L;
3882             }
3883           }
3884         }
3885       }
3886       // junctions
3887       for (PetscInt i = 0; i < extent[0]; i++) {
3888         for (PetscInt j = 0; j < extent[1]; j++) {
3889           for (PetscInt k = 0; k < extent[2]; k++) {
3890             const PetscInt J = (i * extent[1] + j) * extent[2] + k, Jvoff = (Npipes[0] + Npipes[1] + Npipes[2]) * 4 + J * 8;
3891             PetscCheck(vcount / 3 == Jvoff, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Unexpected vertex count");
3892             for (PetscInt ii = 0; ii < 2; ii++) {
3893               for (PetscInt jj = 0; jj < 2; jj++) {
3894                 for (PetscInt kk = 0; kk < 2; kk++) {
3895                   double Ls           = (1 - sqrt(2) / 4) * L;
3896                   vtxCoords[vcount++] = 2 * i * L + (2 * ii - 1) * Ls;
3897                   vtxCoords[vcount++] = 2 * j * L + (2 * jj - 1) * Ls;
3898                   vtxCoords[vcount++] = 2 * k * L + (2 * kk - 1) * Ls;
3899                 }
3900               }
3901             }
3902             const PetscInt jfaces[3][2][4] = {
3903               {{3, 1, 0, 2}, {7, 5, 4, 6}}, // x-aligned
3904               {{5, 4, 0, 1}, {7, 6, 2, 3}}, // y-aligned
3905               {{6, 2, 0, 4}, {7, 3, 1, 5}}  // z-aligned
3906             };
3907             const PetscInt pipe_lo[3] = {// vertex numbers of pipes
3908                                          ((i * extent[1] + j) * extent[2] + k) * 4, ((i * (extent[1] + 1) + j) * extent[2] + k + Npipes[0]) * 4, ((i * extent[1] + j) * (extent[2] + 1) + k + Npipes[0] + Npipes[1]) * 4};
3909             const PetscInt pipe_hi[3] = {// vertex numbers of pipes
3910                                          (((i + 1) * extent[1] + j) * extent[2] + k) * 4, ((i * (extent[1] + 1) + j + 1) * extent[2] + k + Npipes[0]) * 4, ((i * extent[1] + j) * (extent[2] + 1) + k + 1 + Npipes[0] + Npipes[1]) * 4};
3911             for (PetscInt dir = 0; dir < 3; dir++) { // x,y,z
3912               const PetscInt ijk[3] = {i, j, k};
3913               for (PetscInt l = 0; l < 4; l++) { // rotations
3914                 cells[J][dir * 2 + 0][l][0] = pipe_lo[dir] + l;
3915                 cells[J][dir * 2 + 0][l][1] = Jvoff + jfaces[dir][0][l];
3916                 cells[J][dir * 2 + 0][l][2] = Jvoff + jfaces[dir][0][(l - 1 + 4) % 4];
3917                 cells[J][dir * 2 + 0][l][3] = pipe_lo[dir] + (l - 1 + 4) % 4;
3918                 cells[J][dir * 2 + 1][l][0] = Jvoff + jfaces[dir][1][l];
3919                 cells[J][dir * 2 + 1][l][1] = pipe_hi[dir] + l;
3920                 cells[J][dir * 2 + 1][l][2] = pipe_hi[dir] + (l - 1 + 4) % 4;
3921                 cells[J][dir * 2 + 1][l][3] = Jvoff + jfaces[dir][1][(l - 1 + 4) % 4];
3922                 if (ijk[dir] == 0) {
3923                   edges[numEdges][0] = pipe_lo[dir] + l;
3924                   edges[numEdges][1] = pipe_lo[dir] + (l + 1) % 4;
3925                   edgeSets[numEdges] = dir * 2 + 1;
3926                   numEdges++;
3927                 }
3928                 if (ijk[dir] + 1 == extent[dir]) {
3929                   edges[numEdges][0] = pipe_hi[dir] + l;
3930                   edges[numEdges][1] = pipe_hi[dir] + (l + 1) % 4;
3931                   edgeSets[numEdges] = dir * 2 + 2;
3932                   numEdges++;
3933                 }
3934               }
3935             }
3936           }
3937         }
3938       }
3939       PetscCheck(numEdges == Ncuts * 4, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Edge count %" PetscInt_FMT " incompatible with number of cuts %" PetscInt_FMT, numEdges, Ncuts);
3940       numFaces   = 24 * Njunctions;
3941       cells_flat = cells[0][0][0];
3942     }
3943     evalFunc   = TPSEvaluate_SchwarzP;
3944     normalFunc = TPSExtrudeNormalFunc_SchwarzP;
3945     break;
3946   case DMPLEX_TPS_GYROID:
3947     if (rank == 0) {
3948       // This is a coarse mesh approximation of the gyroid shifted to being the zero of the level set
3949       //
3950       //     sin(pi*x)*cos(pi*(y+1/2)) + sin(pi*(y+1/2))*cos(pi*(z+1/4)) + sin(pi*(z+1/4))*cos(x)
3951       //
3952       // on the cell [0,2]^3.
3953       //
3954       // Think about dividing that cell into four columns, and focus on the column [0,1]x[0,1]x[0,2].
3955       // If you looked at the gyroid in that column at different slices of z you would see that it kind of spins
3956       // like a boomerang:
3957       //
3958       //     z = 0          z = 1/4        z = 1/2        z = 3/4     //
3959       //     -----          -------        -------        -------     //
3960       //                                                              //
3961       //     +       +      +       +      +       +      +   \   +   //
3962       //      \                                   /            \      //
3963       //       \            `-_   _-'            /              }     //
3964       //        *-_            `-'            _-'              /      //
3965       //     +     `-+      +       +      +-'     +      +   /   +   //
3966       //                                                              //
3967       //                                                              //
3968       //     z = 1          z = 5/4        z = 3/2        z = 7/4     //
3969       //     -----          -------        -------        -------     //
3970       //                                                              //
3971       //     +-_     +      +       +      +     _-+      +   /   +   //
3972       //        `-_            _-_            _-`            /        //
3973       //           \        _-'   `-_        /              {         //
3974       //            \                       /                \        //
3975       //     +       +      +       +      +       +      +   \   +   //
3976       //
3977       //
3978       // This course mesh approximates each of these slices by two line segments,
3979       // and then connects the segments in consecutive layers with quadrilateral faces.
3980       // All of the end points of the segments are multiples of 1/4 except for the
3981       // point * in the picture for z = 0 above and the similar points in other layers.
3982       // That point is at (gamma, gamma, 0), where gamma is calculated below.
3983       //
3984       // The column  [1,2]x[1,2]x[0,2] looks the same as this column;
3985       // The columns [1,2]x[0,1]x[0,2] and [0,1]x[1,2]x[0,2] are mirror images.
3986       //
3987       // As for how this method turned into the names given to the vertices:
3988       // that was not systematic, it was just the way it worked out in my handwritten notes.
3989 
3990       PetscInt facesPerBlock = 64;
3991       PetscInt vertsPerBlock = 56;
3992       PetscInt extentPlus[3];
3993       PetscInt numBlocks, numBlocksPlus;
3994       const PetscInt A = 0, B = 1, C = 2, D = 3, E = 4, F = 5, G = 6, H = 7, II = 8, J = 9, K = 10, L = 11, M = 12, N = 13, O = 14, P = 15, Q = 16, R = 17, S = 18, T = 19, U = 20, V = 21, W = 22, X = 23, Y = 24, Z = 25, Ap = 26, Bp = 27, Cp = 28, Dp = 29, Ep = 30, Fp = 31, Gp = 32, Hp = 33, Ip = 34, Jp = 35, Kp = 36, Lp = 37, Mp = 38, Np = 39, Op = 40, Pp = 41, Qp = 42, Rp = 43, Sp = 44, Tp = 45, Up = 46, Vp = 47, Wp = 48, Xp = 49, Yp = 50, Zp = 51, Aq = 52, Bq = 53, Cq = 54, Dq = 55;
3995       const PetscInt pattern[64][4] = {
3996         /* face to vertex within the coarse discretization of a single gyroid block */
3997         /* layer 0 */
3998         {A,           C,           K,           G          },
3999         {C,           B,           II,          K          },
4000         {D,           A,           H,           L          },
4001         {B + 56 * 1,  D,           L,           J          },
4002         {E,           B + 56 * 1,  J,           N          },
4003         {A + 56 * 2,  E,           N,           H + 56 * 2 },
4004         {F,           A + 56 * 2,  G + 56 * 2,  M          },
4005         {B,           F,           M,           II         },
4006         /* layer 1 */
4007         {G,           K,           Q,           O          },
4008         {K,           II,          P,           Q          },
4009         {L,           H,           O + 56 * 1,  R          },
4010         {J,           L,           R,           P          },
4011         {N,           J,           P,           S          },
4012         {H + 56 * 2,  N,           S,           O + 56 * 3 },
4013         {M,           G + 56 * 2,  O + 56 * 2,  T          },
4014         {II,          M,           T,           P          },
4015         /* layer 2 */
4016         {O,           Q,           Y,           U          },
4017         {Q,           P,           W,           Y          },
4018         {R,           O + 56 * 1,  U + 56 * 1,  Ap         },
4019         {P,           R,           Ap,          W          },
4020         {S,           P,           X,           Bp         },
4021         {O + 56 * 3,  S,           Bp,          V + 56 * 1 },
4022         {T,           O + 56 * 2,  V,           Z          },
4023         {P,           T,           Z,           X          },
4024         /* layer 3 */
4025         {U,           Y,           Ep,          Dp         },
4026         {Y,           W,           Cp,          Ep         },
4027         {Ap,          U + 56 * 1,  Dp + 56 * 1, Gp         },
4028         {W,           Ap,          Gp,          Cp         },
4029         {Bp,          X,           Cp + 56 * 2, Fp         },
4030         {V + 56 * 1,  Bp,          Fp,          Dp + 56 * 1},
4031         {Z,           V,           Dp,          Hp         },
4032         {X,           Z,           Hp,          Cp + 56 * 2},
4033         /* layer 4 */
4034         {Dp,          Ep,          Mp,          Kp         },
4035         {Ep,          Cp,          Ip,          Mp         },
4036         {Gp,          Dp + 56 * 1, Lp,          Np         },
4037         {Cp,          Gp,          Np,          Jp         },
4038         {Fp,          Cp + 56 * 2, Jp + 56 * 2, Pp         },
4039         {Dp + 56 * 1, Fp,          Pp,          Lp         },
4040         {Hp,          Dp,          Kp,          Op         },
4041         {Cp + 56 * 2, Hp,          Op,          Ip + 56 * 2},
4042         /* layer 5 */
4043         {Kp,          Mp,          Sp,          Rp         },
4044         {Mp,          Ip,          Qp,          Sp         },
4045         {Np,          Lp,          Rp,          Tp         },
4046         {Jp,          Np,          Tp,          Qp + 56 * 1},
4047         {Pp,          Jp + 56 * 2, Qp + 56 * 3, Up         },
4048         {Lp,          Pp,          Up,          Rp         },
4049         {Op,          Kp,          Rp,          Vp         },
4050         {Ip + 56 * 2, Op,          Vp,          Qp + 56 * 2},
4051         /* layer 6 */
4052         {Rp,          Sp,          Aq,          Yp         },
4053         {Sp,          Qp,          Wp,          Aq         },
4054         {Tp,          Rp,          Yp,          Cq         },
4055         {Qp + 56 * 1, Tp,          Cq,          Wp + 56 * 1},
4056         {Up,          Qp + 56 * 3, Xp + 56 * 1, Dq         },
4057         {Rp,          Up,          Dq,          Zp         },
4058         {Vp,          Rp,          Zp,          Bq         },
4059         {Qp + 56 * 2, Vp,          Bq,          Xp         },
4060         /* layer 7 (the top is the periodic image of the bottom of layer 0) */
4061         {Yp,          Aq,          C + 56 * 4,  A + 56 * 4 },
4062         {Aq,          Wp,          B + 56 * 4,  C + 56 * 4 },
4063         {Cq,          Yp,          A + 56 * 4,  D + 56 * 4 },
4064         {Wp + 56 * 1, Cq,          D + 56 * 4,  B + 56 * 5 },
4065         {Dq,          Xp + 56 * 1, B + 56 * 5,  E + 56 * 4 },
4066         {Zp,          Dq,          E + 56 * 4,  A + 56 * 6 },
4067         {Bq,          Zp,          A + 56 * 6,  F + 56 * 4 },
4068         {Xp,          Bq,          F + 56 * 4,  B + 56 * 4 }
4069       };
4070       const PetscReal gamma                = PetscAcosReal((PetscSqrtReal(3.) - 1.) / PetscSqrtReal(2.)) / PETSC_PI;
4071       const PetscReal patternCoords[56][3] = {
4072         {1.,        0.,        0.  }, /* A  */
4073         {0.,        1.,        0.  }, /* B  */
4074         {gamma,     gamma,     0.  }, /* C  */
4075         {1 + gamma, 1 - gamma, 0.  }, /* D  */
4076         {2 - gamma, 2 - gamma, 0.  }, /* E  */
4077         {1 - gamma, 1 + gamma, 0.  }, /* F  */
4078 
4079         {.5,        0,         .25 }, /* G  */
4080         {1.5,       0.,        .25 }, /* H  */
4081         {.5,        1.,        .25 }, /* II */
4082         {1.5,       1.,        .25 }, /* J  */
4083         {.25,       .5,        .25 }, /* K  */
4084         {1.25,      .5,        .25 }, /* L  */
4085         {.75,       1.5,       .25 }, /* M  */
4086         {1.75,      1.5,       .25 }, /* N  */
4087 
4088         {0.,        0.,        .5  }, /* O  */
4089         {1.,        1.,        .5  }, /* P  */
4090         {gamma,     1 - gamma, .5  }, /* Q  */
4091         {1 + gamma, gamma,     .5  }, /* R  */
4092         {2 - gamma, 1 + gamma, .5  }, /* S  */
4093         {1 - gamma, 2 - gamma, .5  }, /* T  */
4094 
4095         {0.,        .5,        .75 }, /* U  */
4096         {0.,        1.5,       .75 }, /* V  */
4097         {1.,        .5,        .75 }, /* W  */
4098         {1.,        1.5,       .75 }, /* X  */
4099         {.5,        .75,       .75 }, /* Y  */
4100         {.5,        1.75,      .75 }, /* Z  */
4101         {1.5,       .25,       .75 }, /* Ap */
4102         {1.5,       1.25,      .75 }, /* Bp */
4103 
4104         {1.,        0.,        1.  }, /* Cp */
4105         {0.,        1.,        1.  }, /* Dp */
4106         {1 - gamma, 1 - gamma, 1.  }, /* Ep */
4107         {1 + gamma, 1 + gamma, 1.  }, /* Fp */
4108         {2 - gamma, gamma,     1.  }, /* Gp */
4109         {gamma,     2 - gamma, 1.  }, /* Hp */
4110 
4111         {.5,        0.,        1.25}, /* Ip */
4112         {1.5,       0.,        1.25}, /* Jp */
4113         {.5,        1.,        1.25}, /* Kp */
4114         {1.5,       1.,        1.25}, /* Lp */
4115         {.75,       .5,        1.25}, /* Mp */
4116         {1.75,      .5,        1.25}, /* Np */
4117         {.25,       1.5,       1.25}, /* Op */
4118         {1.25,      1.5,       1.25}, /* Pp */
4119 
4120         {0.,        0.,        1.5 }, /* Qp */
4121         {1.,        1.,        1.5 }, /* Rp */
4122         {1 - gamma, gamma,     1.5 }, /* Sp */
4123         {2 - gamma, 1 - gamma, 1.5 }, /* Tp */
4124         {1 + gamma, 2 - gamma, 1.5 }, /* Up */
4125         {gamma,     1 + gamma, 1.5 }, /* Vp */
4126 
4127         {0.,        .5,        1.75}, /* Wp */
4128         {0.,        1.5,       1.75}, /* Xp */
4129         {1.,        .5,        1.75}, /* Yp */
4130         {1.,        1.5,       1.75}, /* Zp */
4131         {.5,        .25,       1.75}, /* Aq */
4132         {.5,        1.25,      1.75}, /* Bq */
4133         {1.5,       .75,       1.75}, /* Cq */
4134         {1.5,       1.75,      1.75}, /* Dq */
4135       };
4136       PetscInt (*cells)[64][4] = NULL;
4137       PetscBool *seen;
4138       PetscInt  *vertToTrueVert;
4139       PetscInt   count;
4140 
4141       for (PetscInt i = 0; i < 3; i++) extentPlus[i] = extent[i] + 1;
4142       numBlocks = 1;
4143       for (PetscInt i = 0; i < 3; i++) numBlocks *= extent[i];
4144       numBlocksPlus = 1;
4145       for (PetscInt i = 0; i < 3; i++) numBlocksPlus *= extentPlus[i];
4146       numFaces = numBlocks * facesPerBlock;
4147       PetscCall(PetscMalloc1(numBlocks, &cells));
4148       PetscCall(PetscCalloc1(numBlocksPlus * vertsPerBlock, &seen));
4149       for (PetscInt k = 0; k < extent[2]; k++) {
4150         for (PetscInt j = 0; j < extent[1]; j++) {
4151           for (PetscInt i = 0; i < extent[0]; i++) {
4152             for (PetscInt f = 0; f < facesPerBlock; f++) {
4153               for (PetscInt v = 0; v < 4; v++) {
4154                 PetscInt vertRaw     = pattern[f][v];
4155                 PetscInt blockidx    = vertRaw / 56;
4156                 PetscInt patternvert = vertRaw % 56;
4157                 PetscInt xplus       = (blockidx & 1);
4158                 PetscInt yplus       = (blockidx & 2) >> 1;
4159                 PetscInt zplus       = (blockidx & 4) >> 2;
4160                 PetscInt zcoord      = (periodic && periodic[2] == DM_BOUNDARY_PERIODIC) ? ((k + zplus) % extent[2]) : (k + zplus);
4161                 PetscInt ycoord      = (periodic && periodic[1] == DM_BOUNDARY_PERIODIC) ? ((j + yplus) % extent[1]) : (j + yplus);
4162                 PetscInt xcoord      = (periodic && periodic[0] == DM_BOUNDARY_PERIODIC) ? ((i + xplus) % extent[0]) : (i + xplus);
4163                 PetscInt vert        = ((zcoord * extentPlus[1] + ycoord) * extentPlus[0] + xcoord) * 56 + patternvert;
4164 
4165                 cells[(k * extent[1] + j) * extent[0] + i][f][v] = vert;
4166                 seen[vert]                                       = PETSC_TRUE;
4167               }
4168             }
4169           }
4170         }
4171       }
4172       for (PetscInt i = 0; i < numBlocksPlus * vertsPerBlock; i++)
4173         if (seen[i]) numVertices++;
4174       count = 0;
4175       PetscCall(PetscMalloc1(numBlocksPlus * vertsPerBlock, &vertToTrueVert));
4176       PetscCall(PetscMalloc1(numVertices * 3, &vtxCoords));
4177       for (PetscInt i = 0; i < numBlocksPlus * vertsPerBlock; i++) vertToTrueVert[i] = -1;
4178       for (PetscInt k = 0; k < extentPlus[2]; k++) {
4179         for (PetscInt j = 0; j < extentPlus[1]; j++) {
4180           for (PetscInt i = 0; i < extentPlus[0]; i++) {
4181             for (PetscInt v = 0; v < vertsPerBlock; v++) {
4182               PetscInt vIdx = ((k * extentPlus[1] + j) * extentPlus[0] + i) * vertsPerBlock + v;
4183 
4184               if (seen[vIdx]) {
4185                 PetscInt thisVert;
4186 
4187                 vertToTrueVert[vIdx] = thisVert = count++;
4188 
4189                 for (PetscInt d = 0; d < 3; d++) vtxCoords[3 * thisVert + d] = patternCoords[v][d];
4190                 vtxCoords[3 * thisVert + 0] += i * 2;
4191                 vtxCoords[3 * thisVert + 1] += j * 2;
4192                 vtxCoords[3 * thisVert + 2] += k * 2;
4193               }
4194             }
4195           }
4196         }
4197       }
4198       for (PetscInt i = 0; i < numBlocks; i++) {
4199         for (PetscInt f = 0; f < facesPerBlock; f++) {
4200           for (PetscInt v = 0; v < 4; v++) cells[i][f][v] = vertToTrueVert[cells[i][f][v]];
4201         }
4202       }
4203       PetscCall(PetscFree(vertToTrueVert));
4204       PetscCall(PetscFree(seen));
4205       cells_flat = cells[0][0];
4206       numEdges   = 0;
4207       for (PetscInt i = 0; i < numFaces; i++) {
4208         for (PetscInt e = 0; e < 4; e++) {
4209           PetscInt         ev[]       = {cells_flat[i * 4 + e], cells_flat[i * 4 + ((e + 1) % 4)]};
4210           const PetscReal *evCoords[] = {&vtxCoords[3 * ev[0]], &vtxCoords[3 * ev[1]]};
4211 
4212           for (PetscInt d = 0; d < 3; d++) {
4213             if (!periodic || periodic[0] != DM_BOUNDARY_PERIODIC) {
4214               if (evCoords[0][d] == 0. && evCoords[1][d] == 0.) numEdges++;
4215               if (evCoords[0][d] == 2. * extent[d] && evCoords[1][d] == 2. * extent[d]) numEdges++;
4216             }
4217           }
4218         }
4219       }
4220       PetscCall(PetscMalloc1(numEdges, &edges));
4221       PetscCall(PetscMalloc1(numEdges, &edgeSets));
4222       for (PetscInt edge = 0, i = 0; i < numFaces; i++) {
4223         for (PetscInt e = 0; e < 4; e++) {
4224           PetscInt         ev[]       = {cells_flat[i * 4 + e], cells_flat[i * 4 + ((e + 1) % 4)]};
4225           const PetscReal *evCoords[] = {&vtxCoords[3 * ev[0]], &vtxCoords[3 * ev[1]]};
4226 
4227           for (PetscInt d = 0; d < 3; d++) {
4228             if (!periodic || periodic[d] != DM_BOUNDARY_PERIODIC) {
4229               if (evCoords[0][d] == 0. && evCoords[1][d] == 0.) {
4230                 edges[edge][0]   = ev[0];
4231                 edges[edge][1]   = ev[1];
4232                 edgeSets[edge++] = 2 * d;
4233               }
4234               if (evCoords[0][d] == 2. * extent[d] && evCoords[1][d] == 2. * extent[d]) {
4235                 edges[edge][0]   = ev[0];
4236                 edges[edge][1]   = ev[1];
4237                 edgeSets[edge++] = 2 * d + 1;
4238               }
4239             }
4240           }
4241         }
4242       }
4243     }
4244     evalFunc   = TPSEvaluate_Gyroid;
4245     normalFunc = TPSExtrudeNormalFunc_Gyroid;
4246     break;
4247   }
4248 
4249   PetscCall(DMSetDimension(dm, topoDim));
4250   if (rank == 0) PetscCall(DMPlexBuildFromCellList(dm, numFaces, numVertices, 4, cells_flat));
4251   else PetscCall(DMPlexBuildFromCellList(dm, 0, 0, 0, NULL));
4252   PetscCall(PetscFree(cells_flat));
4253   {
4254     DM idm;
4255     PetscCall(DMPlexInterpolate(dm, &idm));
4256     PetscCall(DMPlexReplace_Internal(dm, &idm));
4257   }
4258   if (rank == 0) PetscCall(DMPlexBuildCoordinatesFromCellList(dm, spaceDim, vtxCoords));
4259   else PetscCall(DMPlexBuildCoordinatesFromCellList(dm, spaceDim, NULL));
4260   PetscCall(PetscFree(vtxCoords));
4261 
4262   PetscCall(DMCreateLabel(dm, "Face Sets"));
4263   PetscCall(DMGetLabel(dm, "Face Sets", &label));
4264   for (PetscInt e = 0; e < numEdges; e++) {
4265     PetscInt        njoin;
4266     const PetscInt *join, verts[] = {numFaces + edges[e][0], numFaces + edges[e][1]};
4267     PetscCall(DMPlexGetJoin(dm, 2, verts, &njoin, &join));
4268     PetscCheck(njoin == 1, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Expected unique join of vertices %" PetscInt_FMT " and %" PetscInt_FMT, edges[e][0], edges[e][1]);
4269     PetscCall(DMLabelSetValue(label, join[0], edgeSets[e]));
4270     PetscCall(DMPlexRestoreJoin(dm, 2, verts, &njoin, &join));
4271   }
4272   PetscCall(PetscFree(edges));
4273   PetscCall(PetscFree(edgeSets));
4274   if (tps_distribute) {
4275     DM               pdm = NULL;
4276     PetscPartitioner part;
4277 
4278     PetscCall(DMPlexGetPartitioner(dm, &part));
4279     PetscCall(PetscPartitionerSetFromOptions(part));
4280     PetscCall(DMPlexDistribute(dm, 0, NULL, &pdm));
4281     if (pdm) PetscCall(DMPlexReplace_Internal(dm, &pdm));
4282     // Do not auto-distribute again
4283     PetscCall(DMPlexDistributeSetDefault(dm, PETSC_FALSE));
4284   }
4285 
4286   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
4287   for (PetscInt refine = 0; refine < refinements; refine++) {
4288     PetscInt     m;
4289     DM           dmf, cdm, cdmf;
4290     Vec          X;
4291     PetscScalar *x;
4292 
4293     PetscCall(DMRefine(dm, MPI_COMM_NULL, &dmf));
4294     PetscCall(DMGetCoordinateDM(dm, &cdm));
4295     PetscCall(DMGetCoordinateDM(dmf, &cdmf));
4296     PetscCall(DMCopyDisc(cdm, cdmf));
4297     PetscCall(DMPlexReplace_Internal(dm, &dmf));
4298 
4299     PetscCall(DMGetCoordinatesLocal(dm, &X));
4300     PetscCall(VecGetLocalSize(X, &m));
4301     PetscCall(VecGetArray(X, &x));
4302     for (PetscInt i = 0; i < m; i += 3) PetscCall(TPSNearestPoint(evalFunc, &x[i]));
4303     PetscCall(VecRestoreArray(X, &x));
4304   }
4305 
4306   // Face Sets has already been propagated to new vertices during refinement; this propagates to the initial vertices.
4307   PetscCall(DMGetLabel(dm, "Face Sets", &label));
4308   PetscCall(DMPlexLabelComplete(dm, label));
4309 
4310   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
4311 
4312   if (thickness > 0) {
4313     DM              edm, cdm, ecdm;
4314     DMPlexTransform tr;
4315     const char     *prefix;
4316     PetscOptions    options;
4317     // Code from DMPlexExtrude
4318     PetscCall(DMPlexTransformCreate(PetscObjectComm((PetscObject)dm), &tr));
4319     PetscCall(DMPlexTransformSetDM(tr, dm));
4320     PetscCall(DMPlexTransformSetType(tr, DMPLEXEXTRUDETYPE));
4321     PetscCall(PetscObjectGetOptionsPrefix((PetscObject)dm, &prefix));
4322     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)tr, prefix));
4323     PetscCall(PetscObjectGetOptions((PetscObject)dm, &options));
4324     PetscCall(PetscObjectSetOptions((PetscObject)tr, options));
4325     PetscCall(DMPlexTransformExtrudeSetLayers(tr, layers));
4326     PetscCall(DMPlexTransformExtrudeSetThickness(tr, thickness));
4327     PetscCall(DMPlexTransformExtrudeSetTensor(tr, PETSC_FALSE));
4328     PetscCall(DMPlexTransformExtrudeSetSymmetric(tr, PETSC_TRUE));
4329     PetscCall(DMPlexTransformExtrudeSetNormalFunction(tr, normalFunc));
4330     PetscCall(DMPlexTransformSetFromOptions(tr));
4331     PetscCall(PetscObjectSetOptions((PetscObject)tr, NULL));
4332     PetscCall(DMPlexTransformSetUp(tr));
4333     PetscCall(PetscObjectViewFromOptions((PetscObject)tr, NULL, "-dm_plex_tps_transform_view"));
4334     PetscCall(DMPlexTransformApply(tr, dm, &edm));
4335     PetscCall(DMCopyDisc(dm, edm));
4336     PetscCall(DMGetCoordinateDM(dm, &cdm));
4337     PetscCall(DMGetCoordinateDM(edm, &ecdm));
4338     PetscCall(DMCopyDisc(cdm, ecdm));
4339     PetscCall(DMPlexTransformCreateDiscLabels(tr, edm));
4340     PetscCall(DMPlexTransformDestroy(&tr));
4341     PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, edm));
4342     PetscCall(DMPlexReplace_Internal(dm, &edm));
4343   }
4344   PetscFunctionReturn(PETSC_SUCCESS);
4345 }
4346 
4347 /*@
4348   DMPlexCreateTPSMesh - Create a distributed, interpolated mesh of a triply-periodic surface
4349 
4350   Collective
4351 
4352   Input Parameters:
4353 + comm           - The communicator for the `DM` object
4354 . tpstype        - Type of triply-periodic surface
4355 . extent         - Array of length 3 containing number of periods in each direction
4356 . periodic       - array of length 3 with periodicity, or `NULL` for non-periodic
4357 . tps_distribute - Distribute 2D manifold mesh prior to refinement and extrusion (more scalable)
4358 . refinements    - Number of factor-of-2 refinements of 2D manifold mesh
4359 . layers         - Number of cell layers extruded in normal direction
4360 - thickness      - Thickness in normal direction
4361 
4362   Output Parameter:
4363 . dm - The `DM` object
4364 
4365   Level: beginner
4366 
4367   Notes:
4368   This meshes the surface of the Schwarz P or Gyroid surfaces.  Schwarz P is the simplest member of the triply-periodic minimal surfaces.
4369   <https://en.wikipedia.org/wiki/Schwarz_minimal_surface#Schwarz_P_(%22Primitive%22)> and can be cut with "clean" boundaries.
4370   The Gyroid <https://en.wikipedia.org/wiki/Gyroid> is another triply-periodic minimal surface with applications in additive manufacturing; it is much more difficult to "cut" since there are no planes of symmetry.
4371   Our implementation creates a very coarse mesh of the surface and refines (by 4-way splitting) as many times as requested.
4372   On each refinement, all vertices are projected to their nearest point on the surface.
4373   This projection could readily be extended to related surfaces.
4374 
4375   See {cite}`maskery2018insights`
4376 
4377   The face (edge) sets for the Schwarz P surface are numbered $1(-x), 2(+x), 3(-y), 4(+y), 5(-z), 6(+z)$.
4378   When the mesh is refined, "Face Sets" contain the new vertices (created during refinement).
4379   Use `DMPlexLabelComplete()` to propagate to coarse-level vertices.
4380 
4381   Developer Notes:
4382   The Gyroid mesh does not currently mark boundary sets.
4383 
4384 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateSphereMesh()`, `DMSetType()`, `DMCreate()`
4385 @*/
4386 PetscErrorCode DMPlexCreateTPSMesh(MPI_Comm comm, DMPlexTPSType tpstype, const PetscInt extent[], const DMBoundaryType periodic[], PetscBool tps_distribute, PetscInt refinements, PetscInt layers, PetscReal thickness, DM *dm)
4387 {
4388   PetscFunctionBegin;
4389   PetscCall(DMCreate(comm, dm));
4390   PetscCall(DMSetType(*dm, DMPLEX));
4391   PetscCall(DMPlexCreateTPSMesh_Internal(*dm, tpstype, extent, periodic, tps_distribute, refinements, layers, thickness));
4392   PetscFunctionReturn(PETSC_SUCCESS);
4393 }
4394 
4395 /*@
4396   DMPlexCreateSphereMesh - Creates a mesh on the d-dimensional sphere, S^d.
4397 
4398   Collective
4399 
4400   Input Parameters:
4401 + comm    - The communicator for the `DM` object
4402 . dim     - The dimension
4403 . simplex - Use simplices, or tensor product cells
4404 - R       - The radius
4405 
4406   Output Parameter:
4407 . dm - The `DM` object
4408 
4409   Level: beginner
4410 
4411 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateBallMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
4412 @*/
4413 PetscErrorCode DMPlexCreateSphereMesh(MPI_Comm comm, PetscInt dim, PetscBool simplex, PetscReal R, DM *dm)
4414 {
4415   PetscFunctionBegin;
4416   PetscAssertPointer(dm, 5);
4417   PetscCall(DMCreate(comm, dm));
4418   PetscCall(DMSetType(*dm, DMPLEX));
4419   PetscCall(DMPlexCreateSphereMesh_Internal(*dm, dim, simplex, R));
4420   PetscFunctionReturn(PETSC_SUCCESS);
4421 }
4422 
4423 static PetscErrorCode DMPlexCreateBallMesh_Internal(DM dm, PetscInt dim, PetscReal R)
4424 {
4425   DM          sdm, vol;
4426   DMLabel     bdlabel;
4427   const char *prefix;
4428 
4429   PetscFunctionBegin;
4430   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &sdm));
4431   PetscCall(DMSetType(sdm, DMPLEX));
4432   PetscCall(DMGetOptionsPrefix(dm, &prefix));
4433   PetscCall(DMSetOptionsPrefix(sdm, prefix));
4434   PetscCall(DMAppendOptionsPrefix(sdm, "bd_"));
4435   PetscCall(DMPlexDistributeSetDefault(sdm, PETSC_FALSE));
4436   PetscCall(DMPlexCreateSphereMesh_Internal(sdm, dim - 1, PETSC_TRUE, R));
4437   PetscCall(DMSetFromOptions(sdm));
4438   PetscCall(DMViewFromOptions(sdm, NULL, "-dm_view"));
4439   PetscCall(DMPlexGenerate(sdm, NULL, PETSC_TRUE, &vol));
4440   PetscCall(DMDestroy(&sdm));
4441   PetscCall(DMPlexReplace_Internal(dm, &vol));
4442   PetscCall(DMCreateLabel(dm, "marker"));
4443   PetscCall(DMGetLabel(dm, "marker", &bdlabel));
4444   PetscCall(DMPlexMarkBoundaryFaces(dm, PETSC_DETERMINE, bdlabel));
4445   PetscCall(DMPlexLabelComplete(dm, bdlabel));
4446   PetscFunctionReturn(PETSC_SUCCESS);
4447 }
4448 
4449 /*@
4450   DMPlexCreateBallMesh - Creates a simplex mesh on the d-dimensional ball, B^d.
4451 
4452   Collective
4453 
4454   Input Parameters:
4455 + comm - The communicator for the `DM` object
4456 . dim  - The dimension
4457 - R    - The radius
4458 
4459   Output Parameter:
4460 . dm - The `DM` object
4461 
4462   Options Database Key:
4463 . bd_dm_refine - This will refine the surface mesh preserving the sphere geometry
4464 
4465   Level: beginner
4466 
4467 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateSphereMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
4468 @*/
4469 PetscErrorCode DMPlexCreateBallMesh(MPI_Comm comm, PetscInt dim, PetscReal R, DM *dm)
4470 {
4471   PetscFunctionBegin;
4472   PetscCall(DMCreate(comm, dm));
4473   PetscCall(DMSetType(*dm, DMPLEX));
4474   PetscCall(DMPlexCreateBallMesh_Internal(*dm, dim, R));
4475   PetscFunctionReturn(PETSC_SUCCESS);
4476 }
4477 
4478 static PetscErrorCode DMPlexCreateReferenceCell_Internal(DM rdm, DMPolytopeType ct)
4479 {
4480   PetscFunctionBegin;
4481   switch (ct) {
4482   case DM_POLYTOPE_POINT: {
4483     PetscInt    numPoints[1]        = {1};
4484     PetscInt    coneSize[1]         = {0};
4485     PetscInt    cones[1]            = {0};
4486     PetscInt    coneOrientations[1] = {0};
4487     PetscScalar vertexCoords[1]     = {0.0};
4488 
4489     PetscCall(DMSetDimension(rdm, 0));
4490     PetscCall(DMPlexCreateFromDAG(rdm, 0, numPoints, coneSize, cones, coneOrientations, vertexCoords));
4491   } break;
4492   case DM_POLYTOPE_SEGMENT: {
4493     PetscInt    numPoints[2]        = {2, 1};
4494     PetscInt    coneSize[3]         = {2, 0, 0};
4495     PetscInt    cones[2]            = {1, 2};
4496     PetscInt    coneOrientations[2] = {0, 0};
4497     PetscScalar vertexCoords[2]     = {-1.0, 1.0};
4498 
4499     PetscCall(DMSetDimension(rdm, 1));
4500     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
4501   } break;
4502   case DM_POLYTOPE_POINT_PRISM_TENSOR: {
4503     PetscInt    numPoints[2]        = {2, 1};
4504     PetscInt    coneSize[3]         = {2, 0, 0};
4505     PetscInt    cones[2]            = {1, 2};
4506     PetscInt    coneOrientations[2] = {0, 0};
4507     PetscScalar vertexCoords[2]     = {-1.0, 1.0};
4508 
4509     PetscCall(DMSetDimension(rdm, 1));
4510     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
4511   } break;
4512   case DM_POLYTOPE_TRIANGLE: {
4513     PetscInt    numPoints[2]        = {3, 1};
4514     PetscInt    coneSize[4]         = {3, 0, 0, 0};
4515     PetscInt    cones[3]            = {1, 2, 3};
4516     PetscInt    coneOrientations[3] = {0, 0, 0};
4517     PetscScalar vertexCoords[6]     = {-1.0, -1.0, 1.0, -1.0, -1.0, 1.0};
4518 
4519     PetscCall(DMSetDimension(rdm, 2));
4520     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
4521   } break;
4522   case DM_POLYTOPE_QUADRILATERAL: {
4523     PetscInt    numPoints[2]        = {4, 1};
4524     PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
4525     PetscInt    cones[4]            = {1, 2, 3, 4};
4526     PetscInt    coneOrientations[4] = {0, 0, 0, 0};
4527     PetscScalar vertexCoords[8]     = {-1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0};
4528 
4529     PetscCall(DMSetDimension(rdm, 2));
4530     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
4531   } break;
4532   case DM_POLYTOPE_SEG_PRISM_TENSOR: {
4533     PetscInt    numPoints[2]        = {4, 1};
4534     PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
4535     PetscInt    cones[4]            = {1, 2, 3, 4};
4536     PetscInt    coneOrientations[4] = {0, 0, 0, 0};
4537     PetscScalar vertexCoords[8]     = {-1.0, -1.0, 1.0, -1.0, -1.0, 1.0, 1.0, 1.0};
4538 
4539     PetscCall(DMSetDimension(rdm, 2));
4540     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
4541   } break;
4542   case DM_POLYTOPE_TETRAHEDRON: {
4543     PetscInt    numPoints[2]        = {4, 1};
4544     PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
4545     PetscInt    cones[4]            = {1, 2, 3, 4};
4546     PetscInt    coneOrientations[4] = {0, 0, 0, 0};
4547     PetscScalar vertexCoords[12]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0};
4548 
4549     PetscCall(DMSetDimension(rdm, 3));
4550     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
4551   } break;
4552   case DM_POLYTOPE_HEXAHEDRON: {
4553     PetscInt    numPoints[2]        = {8, 1};
4554     PetscInt    coneSize[9]         = {8, 0, 0, 0, 0, 0, 0, 0, 0};
4555     PetscInt    cones[8]            = {1, 2, 3, 4, 5, 6, 7, 8};
4556     PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
4557     PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, 1.0, 1.0, 1.0, -1.0, 1.0, 1.0};
4558 
4559     PetscCall(DMSetDimension(rdm, 3));
4560     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
4561   } break;
4562   case DM_POLYTOPE_TRI_PRISM: {
4563     PetscInt    numPoints[2]        = {6, 1};
4564     PetscInt    coneSize[7]         = {6, 0, 0, 0, 0, 0, 0};
4565     PetscInt    cones[6]            = {1, 2, 3, 4, 5, 6};
4566     PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
4567     PetscScalar vertexCoords[18]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, 1.0, 1.0};
4568 
4569     PetscCall(DMSetDimension(rdm, 3));
4570     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
4571   } break;
4572   case DM_POLYTOPE_TRI_PRISM_TENSOR: {
4573     PetscInt    numPoints[2]        = {6, 1};
4574     PetscInt    coneSize[7]         = {6, 0, 0, 0, 0, 0, 0};
4575     PetscInt    cones[6]            = {1, 2, 3, 4, 5, 6};
4576     PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
4577     PetscScalar vertexCoords[18]    = {-1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, 1.0, 1.0};
4578 
4579     PetscCall(DMSetDimension(rdm, 3));
4580     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
4581   } break;
4582   case DM_POLYTOPE_QUAD_PRISM_TENSOR: {
4583     PetscInt    numPoints[2]        = {8, 1};
4584     PetscInt    coneSize[9]         = {8, 0, 0, 0, 0, 0, 0, 0, 0};
4585     PetscInt    cones[8]            = {1, 2, 3, 4, 5, 6, 7, 8};
4586     PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
4587     PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0, 1.0, -1.0, -1.0, 1.0, 1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, 1.0, 1.0, 1.0, -1.0, 1.0, 1.0};
4588 
4589     PetscCall(DMSetDimension(rdm, 3));
4590     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
4591   } break;
4592   case DM_POLYTOPE_PYRAMID: {
4593     PetscInt    numPoints[2]        = {5, 1};
4594     PetscInt    coneSize[6]         = {5, 0, 0, 0, 0, 0};
4595     PetscInt    cones[5]            = {1, 2, 3, 4, 5};
4596     PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
4597     PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, -1.0, 0.0, 0.0, 1.0};
4598 
4599     PetscCall(DMSetDimension(rdm, 3));
4600     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
4601   } break;
4602   default:
4603     SETERRQ(PetscObjectComm((PetscObject)rdm), PETSC_ERR_ARG_WRONG, "Cannot create reference cell for cell type %s", DMPolytopeTypes[ct]);
4604   }
4605   {
4606     PetscInt Nv, v;
4607 
4608     /* Must create the celltype label here so that we do not automatically try to compute the types */
4609     PetscCall(DMCreateLabel(rdm, "celltype"));
4610     PetscCall(DMPlexSetCellType(rdm, 0, ct));
4611     PetscCall(DMPlexGetChart(rdm, NULL, &Nv));
4612     for (v = 1; v < Nv; ++v) PetscCall(DMPlexSetCellType(rdm, v, DM_POLYTOPE_POINT));
4613   }
4614   PetscCall(DMPlexInterpolateInPlace_Internal(rdm));
4615   PetscCall(PetscObjectSetName((PetscObject)rdm, DMPolytopeTypes[ct]));
4616   PetscFunctionReturn(PETSC_SUCCESS);
4617 }
4618 
4619 /*@
4620   DMPlexCreateReferenceCell - Create a `DMPLEX` with the appropriate FEM reference cell
4621 
4622   Collective
4623 
4624   Input Parameters:
4625 + comm - The communicator
4626 - ct   - The cell type of the reference cell
4627 
4628   Output Parameter:
4629 . refdm - The reference cell
4630 
4631   Level: intermediate
4632 
4633 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateBoxMesh()`
4634 @*/
4635 PetscErrorCode DMPlexCreateReferenceCell(MPI_Comm comm, DMPolytopeType ct, DM *refdm)
4636 {
4637   PetscFunctionBegin;
4638   PetscCall(DMCreate(comm, refdm));
4639   PetscCall(DMSetType(*refdm, DMPLEX));
4640   PetscCall(DMPlexCreateReferenceCell_Internal(*refdm, ct));
4641   PetscFunctionReturn(PETSC_SUCCESS);
4642 }
4643 
4644 static PetscErrorCode DMPlexCreateBoundaryLabel_Private(DM dm, const char name[])
4645 {
4646   DM        plex;
4647   DMLabel   label;
4648   PetscBool hasLabel;
4649 
4650   PetscFunctionBegin;
4651   PetscCall(DMHasLabel(dm, name, &hasLabel));
4652   if (hasLabel) PetscFunctionReturn(PETSC_SUCCESS);
4653   PetscCall(DMCreateLabel(dm, name));
4654   PetscCall(DMGetLabel(dm, name, &label));
4655   PetscCall(DMConvert(dm, DMPLEX, &plex));
4656   PetscCall(DMPlexMarkBoundaryFaces(plex, 1, label));
4657   PetscCall(DMPlexLabelComplete(plex, label));
4658   PetscCall(DMDestroy(&plex));
4659   PetscFunctionReturn(PETSC_SUCCESS);
4660 }
4661 
4662 /*
4663   We use the last coordinate as the radius, the inner radius is lower[dim-1] and the outer radius is upper[dim-1]. Then we map the first coordinate around the circle.
4664 
4665     (x, y) -> (r, theta) = (x[1], (x[0] - lower[0]) * 2\pi/(upper[0] - lower[0]))
4666 */
4667 static void boxToAnnulus(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
4668 {
4669   const PetscReal low = PetscRealPart(constants[0]);
4670   const PetscReal upp = PetscRealPart(constants[1]);
4671   const PetscReal r   = PetscRealPart(u[1]);
4672   const PetscReal th  = 2. * PETSC_PI * (PetscRealPart(u[0]) - low) / (upp - low);
4673 
4674   f0[0] = r * PetscCosReal(th);
4675   f0[1] = r * PetscSinReal(th);
4676 }
4677 
4678 // Insert vertices and their joins, marked by depth
4679 static PetscErrorCode ProcessCohesiveLabel_Vertices(DM dm, DMLabel label, DMLabel vlabel, PetscInt val, PetscInt n, const PetscInt vertices[])
4680 {
4681   PetscFunctionBegin;
4682   PetscCall(DMPlexMarkSubmesh_Interpolated(dm, vlabel, val, PETSC_FALSE, PETSC_FALSE, label, NULL));
4683   PetscFunctionReturn(PETSC_SUCCESS);
4684 }
4685 
4686 // Insert faces and their closures, marked by depth
4687 static PetscErrorCode ProcessCohesiveLabel_Faces(DM dm, DMLabel label, PetscInt n, const PetscInt faces[])
4688 {
4689   PetscFunctionBegin;
4690   for (PetscInt p = 0; p < n; ++p) {
4691     const PetscInt point   = faces[p];
4692     PetscInt      *closure = NULL;
4693     PetscInt       clSize, pdepth;
4694 
4695     PetscCall(DMPlexGetPointDepth(dm, point, &pdepth));
4696     PetscCall(DMLabelSetValue(label, point, pdepth));
4697     PetscCall(DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &clSize, &closure));
4698     for (PetscInt cl = 0; cl < clSize * 2; cl += 2) {
4699       PetscCall(DMPlexGetPointDepth(dm, closure[cl], &pdepth));
4700       PetscCall(DMLabelSetValue(label, closure[cl], pdepth));
4701     }
4702     PetscCall(DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &clSize, &closure));
4703   }
4704   PetscFunctionReturn(PETSC_SUCCESS);
4705 }
4706 
4707 PETSC_EXTERN PetscErrorCode PetscOptionsFindPairPrefix_Private(PetscOptions, const char pre[], const char name[], const char *option[], const char *value[], PetscBool *flg);
4708 
4709 const char *const DMPlexShapes[] = {"box", "box_surface", "ball", "sphere", "cylinder", "schwarz_p", "gyroid", "doublet", "annulus", "hypercubic", "zbox", "unknown", "DMPlexShape", "DM_SHAPE_", NULL};
4710 
4711 static PetscErrorCode DMPlexCreateFromOptions_Internal(PetscOptionItems PetscOptionsObject, PetscBool *useCoordSpace, DM dm)
4712 {
4713   DMPlexShape    shape   = DM_SHAPE_BOX;
4714   DMPolytopeType cell    = DM_POLYTOPE_TRIANGLE;
4715   PetscInt       dim     = 2;
4716   PetscBool      simplex = PETSC_TRUE, interpolate = PETSC_TRUE, orient = PETSC_FALSE, adjCone = PETSC_FALSE, adjClosure = PETSC_TRUE, refDomain = PETSC_FALSE;
4717   PetscBool      flg, flg2, fflg, strflg, bdfflg, nameflg;
4718   MPI_Comm       comm;
4719   char           filename[PETSC_MAX_PATH_LEN]   = "<unspecified>";
4720   char           bdFilename[PETSC_MAX_PATH_LEN] = "<unspecified>";
4721   char           plexname[PETSC_MAX_PATH_LEN]   = "";
4722   const char    *option;
4723 
4724   PetscFunctionBegin;
4725   PetscCall(PetscLogEventBegin(DMPLEX_CreateFromOptions, dm, 0, 0, 0));
4726   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
4727   /* TODO Turn this into a registration interface */
4728   PetscCall(PetscOptionsString("-dm_plex_filename", "File containing a mesh", "DMPlexCreateFromFile", filename, filename, sizeof(filename), &fflg));
4729   PetscCall(PetscOptionsString("-dm_plex_file_contents", "Contents of a file format in a string", "DMPlexCreateFromFile", filename, filename, sizeof(filename), &strflg));
4730   PetscCall(PetscOptionsString("-dm_plex_boundary_filename", "File containing a mesh boundary", "DMPlexCreateFromFile", bdFilename, bdFilename, sizeof(bdFilename), &bdfflg));
4731   PetscCall(PetscOptionsString("-dm_plex_name", "Name of the mesh in the file", "DMPlexCreateFromFile", plexname, plexname, sizeof(plexname), &nameflg));
4732   PetscCall(PetscOptionsEnum("-dm_plex_cell", "Cell shape", "", DMPolytopeTypes, (PetscEnum)cell, (PetscEnum *)&cell, NULL));
4733   PetscCall(PetscOptionsBool("-dm_plex_reference_cell_domain", "Use a reference cell domain", "", refDomain, &refDomain, NULL));
4734   PetscCall(PetscOptionsEnum("-dm_plex_shape", "Shape for built-in mesh", "", DMPlexShapes, (PetscEnum)shape, (PetscEnum *)&shape, &flg));
4735   PetscCall(PetscOptionsBoundedInt("-dm_plex_dim", "Topological dimension of the mesh", "DMGetDimension", dim, &dim, &flg, 0));
4736   PetscCall(PetscOptionsBool("-dm_plex_simplex", "Mesh cell shape", "", simplex, &simplex, &flg));
4737   PetscCall(PetscOptionsBool("-dm_plex_interpolate", "Flag to create edges and faces automatically", "", interpolate, &interpolate, &flg));
4738   PetscCall(PetscOptionsBool("-dm_plex_orient", "Orient the constructed mesh", "DMPlexOrient", orient, &orient, &flg));
4739   PetscCall(PetscOptionsBool("-dm_plex_adj_cone", "Set adjacency direction", "DMSetBasicAdjacency", adjCone, &adjCone, &flg));
4740   PetscCall(PetscOptionsBool("-dm_plex_adj_closure", "Set adjacency size", "DMSetBasicAdjacency", adjClosure, &adjClosure, &flg2));
4741   if (flg || flg2) PetscCall(DMSetBasicAdjacency(dm, adjCone, adjClosure));
4742   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_adj", "Debug output level all adjacency computations", "", 0, &((DM_Plex *)dm->data)->printAdj, NULL, 0));
4743 
4744   switch (cell) {
4745   case DM_POLYTOPE_POINT:
4746   case DM_POLYTOPE_SEGMENT:
4747   case DM_POLYTOPE_POINT_PRISM_TENSOR:
4748   case DM_POLYTOPE_TRIANGLE:
4749   case DM_POLYTOPE_QUADRILATERAL:
4750   case DM_POLYTOPE_TETRAHEDRON:
4751   case DM_POLYTOPE_HEXAHEDRON:
4752     *useCoordSpace = PETSC_TRUE;
4753     break;
4754   default:
4755     *useCoordSpace = PETSC_FALSE;
4756     break;
4757   }
4758 
4759   if (fflg) {
4760     DM          dmnew;
4761     const char *name;
4762 
4763     PetscCall(PetscObjectGetName((PetscObject)dm, &name));
4764     PetscCall(DMPlexCreateFromFile(PetscObjectComm((PetscObject)dm), filename, nameflg ? plexname : name, interpolate, &dmnew));
4765     PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
4766     PetscCall(DMPlexReplace_Internal(dm, &dmnew));
4767   } else if (refDomain) {
4768     PetscCall(DMPlexCreateReferenceCell_Internal(dm, cell));
4769   } else if (bdfflg) {
4770     DM          bdm, dmnew;
4771     const char *name;
4772 
4773     PetscCall(PetscObjectGetName((PetscObject)dm, &name));
4774     PetscCall(DMPlexCreateFromFile(PetscObjectComm((PetscObject)dm), bdFilename, nameflg ? plexname : name, interpolate, &bdm));
4775     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)bdm, "bd_"));
4776     PetscCall(DMSetFromOptions(bdm));
4777     PetscCall(DMPlexGenerate(bdm, NULL, interpolate, &dmnew));
4778     PetscCall(DMDestroy(&bdm));
4779     PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
4780     PetscCall(DMPlexReplace_Internal(dm, &dmnew));
4781   } else if (strflg) {
4782     DM          dmnew;
4783     PetscViewer viewer;
4784     const char *contents;
4785     char       *strname;
4786     char        tmpdir[PETSC_MAX_PATH_LEN];
4787     char        tmpfilename[PETSC_MAX_PATH_LEN];
4788     char        name[PETSC_MAX_PATH_LEN];
4789     MPI_Comm    comm;
4790     PetscMPIInt rank;
4791 
4792     PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
4793     PetscCallMPI(MPI_Comm_rank(comm, &rank));
4794     PetscCall(PetscStrchr(filename, ':', &strname));
4795     PetscCheck(strname, comm, PETSC_ERR_ARG_WRONG, "File contents must have the form \"ext:string_name\", not %s", filename);
4796     strname[0] = '\0';
4797     ++strname;
4798     PetscCall(PetscDLSym(NULL, strname, (void **)&contents));
4799     PetscCheck(contents, comm, PETSC_ERR_ARG_WRONG, "Could not locate mesh string %s", strname);
4800     PetscCall(PetscGetTmp(comm, tmpdir, PETSC_MAX_PATH_LEN));
4801     PetscCall(PetscStrlcat(tmpdir, "/meshXXXXXX", PETSC_MAX_PATH_LEN));
4802     PetscCall(PetscMkdtemp(tmpdir));
4803     PetscCall(PetscSNPrintf(tmpfilename, PETSC_MAX_PATH_LEN, "%s/mesh.%s", tmpdir, filename));
4804     PetscCall(PetscViewerASCIIOpen(comm, tmpfilename, &viewer));
4805     PetscCall(PetscViewerASCIIPrintf(viewer, "%s\n", contents));
4806     PetscCall(PetscViewerDestroy(&viewer));
4807     PetscCall(DMPlexCreateFromFile(PetscObjectComm((PetscObject)dm), tmpfilename, plexname, interpolate, &dmnew));
4808     PetscCall(PetscRMTree(tmpdir));
4809     PetscCall(PetscSNPrintf(name, PETSC_MAX_PATH_LEN, "%s Mesh", strname));
4810     PetscCall(PetscObjectSetName((PetscObject)dm, name));
4811     PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
4812     PetscCall(DMPlexReplace_Internal(dm, &dmnew));
4813   } else {
4814     PetscCall(PetscObjectSetName((PetscObject)dm, DMPlexShapes[shape]));
4815     switch (shape) {
4816     case DM_SHAPE_BOX:
4817     case DM_SHAPE_ZBOX:
4818     case DM_SHAPE_ANNULUS: {
4819       PetscInt       faces[3]  = {0, 0, 0};
4820       PetscReal      lower[3]  = {0, 0, 0};
4821       PetscReal      upper[3]  = {1, 1, 1};
4822       DMBoundaryType bdt[3]    = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
4823       PetscBool      isAnnular = shape == DM_SHAPE_ANNULUS ? PETSC_TRUE : PETSC_FALSE;
4824       PetscInt       i, n;
4825 
4826       n = dim;
4827       for (i = 0; i < dim; ++i) faces[i] = (dim == 1 ? 1 : 4 - dim);
4828       PetscCall(PetscOptionsIntArray("-dm_plex_box_faces", "Number of faces along each dimension", "", faces, &n, &flg));
4829       n = 3;
4830       PetscCall(PetscOptionsRealArray("-dm_plex_box_lower", "Lower left corner of box", "", lower, &n, &flg));
4831       PetscCheck(!flg || !(n != dim), comm, PETSC_ERR_ARG_SIZ, "Lower box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
4832       n = 3;
4833       PetscCall(PetscOptionsRealArray("-dm_plex_box_upper", "Upper right corner of box", "", upper, &n, &flg));
4834       PetscCheck(!flg || !(n != dim), comm, PETSC_ERR_ARG_SIZ, "Upper box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
4835       n = 3;
4836       PetscCall(PetscOptionsEnumArray("-dm_plex_box_bd", "Boundary type for each dimension", "", DMBoundaryTypes, (PetscEnum *)bdt, &n, &flg));
4837       PetscCheck(!flg || !(n != dim), comm, PETSC_ERR_ARG_SIZ, "Box boundary types had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
4838 
4839       PetscCheck(!isAnnular || dim == 2, comm, PETSC_ERR_ARG_OUTOFRANGE, "Only two dimensional annuli have been implemented");
4840       if (isAnnular)
4841         for (i = 0; i < dim - 1; ++i) bdt[i] = DM_BOUNDARY_PERIODIC;
4842 
4843       switch (cell) {
4844       case DM_POLYTOPE_TRI_PRISM_TENSOR:
4845         PetscCall(DMPlexCreateWedgeBoxMesh_Internal(dm, faces, lower, upper, bdt));
4846         if (!interpolate) {
4847           DM udm;
4848 
4849           PetscCall(DMPlexUninterpolate(dm, &udm));
4850           PetscCall(DMPlexReplace_Internal(dm, &udm));
4851         }
4852         break;
4853       default:
4854         PetscCall(DMPlexCreateBoxMesh_Internal(dm, shape, dim, simplex, faces, lower, upper, bdt, interpolate));
4855         break;
4856       }
4857       if (isAnnular) {
4858         DM          cdm;
4859         PetscDS     cds;
4860         PetscScalar bounds[2] = {lower[0], upper[0]};
4861 
4862         // Fix coordinates for annular region
4863         PetscCall(DMSetPeriodicity(dm, NULL, NULL, NULL));
4864         PetscCall(DMSetCellCoordinatesLocal(dm, NULL));
4865         PetscCall(DMSetCellCoordinates(dm, NULL));
4866         PetscCall(DMPlexCreateCoordinateSpace(dm, 1, PETSC_TRUE, NULL));
4867         PetscCall(DMGetCoordinateDM(dm, &cdm));
4868         PetscCall(DMGetDS(cdm, &cds));
4869         PetscCall(PetscDSSetConstants(cds, 2, bounds));
4870         PetscCall(DMPlexRemapGeometry(dm, 0.0, boxToAnnulus));
4871       }
4872     } break;
4873     case DM_SHAPE_BOX_SURFACE: {
4874       PetscInt  faces[3] = {0, 0, 0};
4875       PetscReal lower[3] = {0, 0, 0};
4876       PetscReal upper[3] = {1, 1, 1};
4877       PetscInt  i, n;
4878 
4879       n = dim + 1;
4880       for (i = 0; i < dim + 1; ++i) faces[i] = (dim + 1 == 1 ? 1 : 4 - (dim + 1));
4881       PetscCall(PetscOptionsIntArray("-dm_plex_box_faces", "Number of faces along each dimension", "", faces, &n, &flg));
4882       n = 3;
4883       PetscCall(PetscOptionsRealArray("-dm_plex_box_lower", "Lower left corner of box", "", lower, &n, &flg));
4884       PetscCheck(!flg || !(n != dim + 1), comm, PETSC_ERR_ARG_SIZ, "Lower box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim + 1);
4885       n = 3;
4886       PetscCall(PetscOptionsRealArray("-dm_plex_box_upper", "Upper right corner of box", "", upper, &n, &flg));
4887       PetscCheck(!flg || !(n != dim + 1), comm, PETSC_ERR_ARG_SIZ, "Upper box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim + 1);
4888       PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(dm, dim + 1, faces, lower, upper, interpolate));
4889     } break;
4890     case DM_SHAPE_SPHERE: {
4891       PetscReal R = 1.0;
4892 
4893       PetscCall(PetscOptionsReal("-dm_plex_sphere_radius", "Radius of the sphere", "", R, &R, &flg));
4894       PetscCall(DMPlexCreateSphereMesh_Internal(dm, dim, simplex, R));
4895     } break;
4896     case DM_SHAPE_BALL: {
4897       PetscReal R = 1.0;
4898 
4899       PetscCall(PetscOptionsReal("-dm_plex_ball_radius", "Radius of the ball", "", R, &R, &flg));
4900       PetscCall(DMPlexCreateBallMesh_Internal(dm, dim, R));
4901     } break;
4902     case DM_SHAPE_CYLINDER: {
4903       DMBoundaryType bdt = DM_BOUNDARY_NONE;
4904       PetscInt       Nw  = 6;
4905       PetscInt       Nr  = 0;
4906 
4907       PetscCall(PetscOptionsEnum("-dm_plex_cylinder_bd", "Boundary type in the z direction", "", DMBoundaryTypes, (PetscEnum)bdt, (PetscEnum *)&bdt, NULL));
4908       PetscCall(PetscOptionsInt("-dm_plex_cylinder_num_wedges", "Number of wedges around the cylinder", "", Nw, &Nw, NULL));
4909       PetscCall(PetscOptionsInt("-dm_plex_cylinder_num_refine", "Number of refinements before projection", "", Nr, &Nr, NULL));
4910       switch (cell) {
4911       case DM_POLYTOPE_TRI_PRISM_TENSOR:
4912         PetscCall(DMPlexCreateWedgeCylinderMesh_Internal(dm, Nw, interpolate));
4913         break;
4914       default:
4915         PetscCall(DMPlexCreateHexCylinderMesh_Internal(dm, bdt, Nr));
4916         break;
4917       }
4918     } break;
4919     case DM_SHAPE_SCHWARZ_P: // fallthrough
4920     case DM_SHAPE_GYROID: {
4921       PetscInt       extent[3] = {1, 1, 1}, refine = 0, layers = 0, three;
4922       PetscReal      thickness   = 0.;
4923       DMBoundaryType periodic[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
4924       DMPlexTPSType  tps_type    = shape == DM_SHAPE_SCHWARZ_P ? DMPLEX_TPS_SCHWARZ_P : DMPLEX_TPS_GYROID;
4925       PetscBool      tps_distribute;
4926       PetscCall(PetscOptionsIntArray("-dm_plex_tps_extent", "Number of replicas for each of three dimensions", NULL, extent, (three = 3, &three), NULL));
4927       PetscCall(PetscOptionsInt("-dm_plex_tps_refine", "Number of refinements", NULL, refine, &refine, NULL));
4928       PetscCall(PetscOptionsEnumArray("-dm_plex_tps_periodic", "Periodicity in each of three dimensions", NULL, DMBoundaryTypes, (PetscEnum *)periodic, (three = 3, &three), NULL));
4929       PetscCall(PetscOptionsInt("-dm_plex_tps_layers", "Number of layers in volumetric extrusion (or zero to not extrude)", NULL, layers, &layers, NULL));
4930       PetscCall(PetscOptionsReal("-dm_plex_tps_thickness", "Thickness of volumetric extrusion", NULL, thickness, &thickness, NULL));
4931       PetscCall(DMPlexDistributeGetDefault(dm, &tps_distribute));
4932       PetscCall(PetscOptionsBool("-dm_plex_tps_distribute", "Distribute the 2D mesh prior to refinement and extrusion", NULL, tps_distribute, &tps_distribute, NULL));
4933       PetscCall(DMPlexCreateTPSMesh_Internal(dm, tps_type, extent, periodic, tps_distribute, refine, layers, thickness));
4934     } break;
4935     case DM_SHAPE_DOUBLET: {
4936       DM        dmnew;
4937       PetscReal rl = 0.0;
4938 
4939       PetscCall(PetscOptionsReal("-dm_plex_doublet_refinementlimit", "Refinement limit", NULL, rl, &rl, NULL));
4940       PetscCall(DMPlexCreateDoublet(PetscObjectComm((PetscObject)dm), dim, simplex, interpolate, rl, &dmnew));
4941       PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
4942       PetscCall(DMPlexReplace_Internal(dm, &dmnew));
4943     } break;
4944     case DM_SHAPE_HYPERCUBIC: {
4945       PetscInt       *edges, overlap = 1;
4946       PetscReal      *lower, *upper;
4947       DMBoundaryType *bdt;
4948       PetscInt        n, d;
4949 
4950       *useCoordSpace = PETSC_FALSE;
4951       PetscCall(PetscMalloc4(dim, &edges, dim, &lower, dim, &upper, dim, &bdt));
4952       for (d = 0; d < dim; ++d) {
4953         edges[d] = 1;
4954         lower[d] = 0.;
4955         upper[d] = 1.;
4956         bdt[d]   = DM_BOUNDARY_PERIODIC;
4957       }
4958       n = dim;
4959       PetscCall(PetscOptionsIntArray("-dm_plex_box_faces", "Number of faces along each dimension", "", edges, &n, &flg));
4960       n = dim;
4961       PetscCall(PetscOptionsRealArray("-dm_plex_box_lower", "Lower left corner of box", "", lower, &n, &flg));
4962       PetscCheck(!flg || n == dim, comm, PETSC_ERR_ARG_SIZ, "Lower box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
4963       n = dim;
4964       PetscCall(PetscOptionsRealArray("-dm_plex_box_upper", "Upper right corner of box", "", upper, &n, &flg));
4965       PetscCheck(!flg || n == dim, comm, PETSC_ERR_ARG_SIZ, "Upper box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
4966       n = dim;
4967       PetscCall(PetscOptionsEnumArray("-dm_plex_box_bd", "Boundary type for each dimension", "", DMBoundaryTypes, (PetscEnum *)bdt, &n, &flg));
4968       PetscCheck(!flg || n == dim, comm, PETSC_ERR_ARG_SIZ, "Box boundary types had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
4969       PetscCall(PetscOptionsBoundedInt("-dm_distribute_overlap", "The size of the overlap halo", "DMPlexDistribute", overlap, &overlap, NULL, 0));
4970       PetscCall(DMPlexCreateHypercubicMesh_Internal(dm, dim, lower, upper, edges, overlap, bdt));
4971       PetscCall(PetscFree4(edges, lower, upper, bdt));
4972     } break;
4973     default:
4974       SETERRQ(comm, PETSC_ERR_SUP, "Domain shape %s is unsupported", DMPlexShapes[shape]);
4975     }
4976   }
4977   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
4978   if (!((PetscObject)dm)->name && nameflg) PetscCall(PetscObjectSetName((PetscObject)dm, plexname));
4979   if (orient) PetscCall(DMPlexOrient(dm));
4980   // Allow label creation
4981   PetscCall(PetscOptionsFindPairPrefix_Private(NULL, ((PetscObject)dm)->prefix, "-dm_plex_label_", &option, NULL, &flg));
4982   if (flg) {
4983     DMLabel     label;
4984     PetscInt    points[1024], n = 1024;
4985     char        fulloption[PETSC_MAX_PATH_LEN];
4986     const char *name = &option[14];
4987 
4988     PetscCall(DMCreateLabel(dm, name));
4989     PetscCall(DMGetLabel(dm, name, &label));
4990     fulloption[0] = '-';
4991     fulloption[1] = 0;
4992     PetscCall(PetscStrlcat(fulloption, option, PETSC_MAX_PATH_LEN));
4993     PetscCall(PetscOptionsGetIntArray(NULL, ((PetscObject)dm)->prefix, fulloption, points, &n, NULL));
4994     for (PetscInt p = 0; p < n; ++p) PetscCall(DMLabelSetValue(label, points[p], 1));
4995   }
4996   // Allow cohesive label creation
4997   //   Faces are input, completed, and all points are marked with their depth
4998   PetscCall(PetscOptionsFindPairPrefix_Private(NULL, ((PetscObject)dm)->prefix, "-dm_plex_cohesive_label_", &option, NULL, &flg));
4999   if (flg) {
5000     DMLabel   label;
5001     PetscInt  points[1024], n, pStart, pEnd, Nl = 1;
5002     PetscBool noCreate = PETSC_FALSE;
5003     char      fulloption[PETSC_MAX_PATH_LEN];
5004     char      name[PETSC_MAX_PATH_LEN];
5005     size_t    len;
5006 
5007     PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
5008     PetscCall(PetscStrncpy(name, &option[23], PETSC_MAX_PATH_LEN));
5009     PetscCall(PetscStrlen(name, &len));
5010     if (name[len - 1] == '0') Nl = 10;
5011     for (PetscInt l = 0; l < Nl; ++l) {
5012       if (l > 0) name[len - 1] = (char)('0' + l);
5013       fulloption[0] = 0;
5014       PetscCall(PetscStrlcat(fulloption, "-dm_plex_cohesive_label_", 32));
5015       PetscCall(PetscStrlcat(fulloption, name, PETSC_MAX_PATH_LEN - 32));
5016       n = 1024;
5017       PetscCall(PetscOptionsGetIntArray(NULL, ((PetscObject)dm)->prefix, fulloption, points, &n, &flg));
5018       if (!flg) break;
5019       PetscCall(DMHasLabel(dm, name, &noCreate));
5020       if (noCreate) {
5021         DMLabel         inlabel;
5022         IS              pointIS;
5023         const PetscInt *lpoints;
5024         PetscInt        pdep, ln, inval = points[0];
5025         char            newname[PETSC_MAX_PATH_LEN];
5026 
5027         PetscCheck(n == 1, comm, PETSC_ERR_ARG_WRONG, "Must specify a label value with this option");
5028         PetscCall(DMGetLabel(dm, name, &inlabel));
5029         PetscCall(DMLabelGetStratumIS(inlabel, inval, &pointIS));
5030         PetscCall(ISGetLocalSize(pointIS, &ln));
5031         PetscCall(ISGetIndices(pointIS, &lpoints));
5032         PetscCall(DMPlexGetPointDepth(dm, lpoints[0], &pdep));
5033         PetscCall(PetscSNPrintf(newname, PETSC_MAX_PATH_LEN, "%s%" PetscInt_FMT, name, points[0]));
5034         PetscCall(DMCreateLabel(dm, newname));
5035         PetscCall(DMGetLabel(dm, newname, &label));
5036         if (!pdep) PetscCall(ProcessCohesiveLabel_Vertices(dm, label, inlabel, inval, ln, lpoints));
5037         else PetscCall(ProcessCohesiveLabel_Faces(dm, label, ln, lpoints));
5038         PetscCall(ISRestoreIndices(pointIS, &lpoints));
5039         PetscCall(ISDestroy(&pointIS));
5040       } else {
5041         PetscCall(DMCreateLabel(dm, name));
5042         PetscCall(DMGetLabel(dm, name, &label));
5043         if (pStart >= pEnd) n = 0;
5044         PetscCall(ProcessCohesiveLabel_Faces(dm, label, n, points));
5045       }
5046       PetscCall(DMPlexOrientLabel(dm, label));
5047       PetscCall(DMPlexLabelCohesiveComplete(dm, label, NULL, 1, PETSC_FALSE, PETSC_FALSE, NULL));
5048     }
5049   }
5050   PetscCall(DMViewFromOptions(dm, NULL, "-created_dm_view"));
5051   PetscCall(PetscLogEventEnd(DMPLEX_CreateFromOptions, dm, 0, 0, 0));
5052   PetscFunctionReturn(PETSC_SUCCESS);
5053 }
5054 
5055 PetscErrorCode DMSetFromOptions_NonRefinement_Plex(DM dm, PetscOptionItems PetscOptionsObject)
5056 {
5057   DM_Plex  *mesh = (DM_Plex *)dm->data;
5058   PetscBool flg, flg2;
5059   char      bdLabel[PETSC_MAX_PATH_LEN];
5060   char      method[PETSC_MAX_PATH_LEN];
5061 
5062   PetscFunctionBegin;
5063   /* Handle viewing */
5064   PetscCall(PetscOptionsBool("-dm_plex_print_set_values", "Output all set values info", "DMPlexMatSetClosure", PETSC_FALSE, &mesh->printSetValues, NULL));
5065   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_fem", "Debug output level for all fem computations", "DMPlexSNESComputeResidualFEM", 0, &mesh->printFEM, NULL, 0));
5066   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_fvm", "Debug output level for all fvm computations", "DMPlexSNESComputeResidualFVM", 0, &mesh->printFVM, NULL, 0));
5067   PetscCall(PetscOptionsReal("-dm_plex_print_tol", "Tolerance for FEM output", "DMPlexSNESComputeResidualFEM", mesh->printTol, &mesh->printTol, NULL));
5068   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_l2", "Debug output level all L2 diff computations", "DMComputeL2Diff", 0, &mesh->printL2, NULL, 0));
5069   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_locate", "Debug output level all point location computations", "DMLocatePoints", 0, &mesh->printLocate, NULL, 0));
5070   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_project", "Debug output level all projection computations", "DMPlexProject", 0, &mesh->printProject, NULL, 0));
5071   PetscCall(DMMonitorSetFromOptions(dm, "-dm_plex_monitor_throughput", "Monitor the simulation throughput", "DMPlexMonitorThroughput", DMPlexMonitorThroughput, NULL, &flg));
5072   if (flg) PetscCall(PetscLogDefaultBegin());
5073   // Interpolation
5074   PetscCall(PetscOptionsBool("-dm_plex_interpolate_prefer_tensor", "When different orderings exist, prefer the tensor order", "DMPlexSetInterpolationPreferTensor", mesh->interpolatePreferTensor, &mesh->interpolatePreferTensor, NULL));
5075   /* Labeling */
5076   PetscCall(PetscOptionsString("-dm_plex_boundary_label", "Label to mark the mesh boundary", "", bdLabel, bdLabel, sizeof(bdLabel), &flg));
5077   if (flg) PetscCall(DMPlexCreateBoundaryLabel_Private(dm, bdLabel));
5078   /* Point Location */
5079   PetscCall(PetscOptionsBool("-dm_plex_hash_location", "Use grid hashing for point location", "DMInterpolate", PETSC_FALSE, &mesh->useHashLocation, NULL));
5080   /* Partitioning and distribution */
5081   PetscCall(PetscOptionsBool("-dm_plex_partition_balance", "Attempt to evenly divide points on partition boundary between processes", "DMPlexSetPartitionBalance", PETSC_FALSE, &mesh->partitionBalance, NULL));
5082   /* Reordering */
5083   PetscCall(PetscOptionsBool("-dm_reorder_section", "Compute point permutation for local section", "DMReorderSectionSetDefault", PETSC_FALSE, &flg2, &flg));
5084   if (flg) PetscCall(DMReorderSectionSetDefault(dm, flg2 ? DM_REORDER_DEFAULT_TRUE : DM_REORDER_DEFAULT_FALSE));
5085   PetscCall(PetscOptionsString("-dm_reorder_section_type", "Reordering method for local section", "DMReorderSectionSetType", method, method, PETSC_MAX_PATH_LEN, &flg));
5086   if (flg) PetscCall(DMReorderSectionSetType(dm, method));
5087   /* Generation and remeshing */
5088   PetscCall(PetscOptionsBool("-dm_plex_remesh_bd", "Allow changes to the boundary on remeshing", "DMAdapt", PETSC_FALSE, &mesh->remeshBd, NULL));
5089   /* Projection behavior */
5090   PetscCall(PetscOptionsBoundedInt("-dm_plex_max_projection_height", "Maximum mesh point height used to project locally", "DMPlexSetMaxProjectionHeight", 0, &mesh->maxProjectionHeight, NULL, 0));
5091   PetscCall(PetscOptionsBool("-dm_plex_regular_refinement", "Use special nested projection algorithm for regular refinement", "DMPlexSetRegularRefinement", mesh->regularRefinement, &mesh->regularRefinement, NULL));
5092   /* Checking structure */
5093   {
5094     PetscBool all = PETSC_FALSE;
5095 
5096     PetscCall(PetscOptionsBool("-dm_plex_check_all", "Perform all basic checks", "DMPlexCheck", PETSC_FALSE, &all, NULL));
5097     if (all) {
5098       PetscCall(DMPlexCheck(dm));
5099     } else {
5100       PetscCall(PetscOptionsBool("-dm_plex_check_symmetry", "Check that the adjacency information in the mesh is symmetric", "DMPlexCheckSymmetry", PETSC_FALSE, &flg, &flg2));
5101       if (flg && flg2) PetscCall(DMPlexCheckSymmetry(dm));
5102       PetscCall(PetscOptionsBool("-dm_plex_check_skeleton", "Check that each cell has the correct number of vertices (only for homogeneous simplex or tensor meshes)", "DMPlexCheckSkeleton", PETSC_FALSE, &flg, &flg2));
5103       if (flg && flg2) PetscCall(DMPlexCheckSkeleton(dm, 0));
5104       PetscCall(PetscOptionsBool("-dm_plex_check_faces", "Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type", "DMPlexCheckFaces", PETSC_FALSE, &flg, &flg2));
5105       if (flg && flg2) PetscCall(DMPlexCheckFaces(dm, 0));
5106       PetscCall(PetscOptionsBool("-dm_plex_check_geometry", "Check that cells have positive volume", "DMPlexCheckGeometry", PETSC_FALSE, &flg, &flg2));
5107       if (flg && flg2) PetscCall(DMPlexCheckGeometry(dm));
5108       PetscCall(PetscOptionsBool("-dm_plex_check_pointsf", "Check some necessary conditions for PointSF", "DMPlexCheckPointSF", PETSC_FALSE, &flg, &flg2));
5109       if (flg && flg2) PetscCall(DMPlexCheckPointSF(dm, NULL, PETSC_FALSE));
5110       PetscCall(PetscOptionsBool("-dm_plex_check_interface_cones", "Check points on inter-partition interfaces have conforming order of cone points", "DMPlexCheckInterfaceCones", PETSC_FALSE, &flg, &flg2));
5111       if (flg && flg2) PetscCall(DMPlexCheckInterfaceCones(dm));
5112     }
5113     PetscCall(PetscOptionsBool("-dm_plex_check_cell_shape", "Check cell shape", "DMPlexCheckCellShape", PETSC_FALSE, &flg, &flg2));
5114     if (flg && flg2) PetscCall(DMPlexCheckCellShape(dm, PETSC_TRUE, PETSC_DETERMINE));
5115   }
5116   {
5117     PetscReal scale = 1.0;
5118 
5119     PetscCall(PetscOptionsReal("-dm_plex_scale", "Scale factor for mesh coordinates", "DMPlexScale", scale, &scale, &flg));
5120     if (flg) {
5121       Vec coordinates, coordinatesLocal;
5122 
5123       PetscCall(DMGetCoordinates(dm, &coordinates));
5124       PetscCall(DMGetCoordinatesLocal(dm, &coordinatesLocal));
5125       PetscCall(VecScale(coordinates, scale));
5126       PetscCall(VecScale(coordinatesLocal, scale));
5127     }
5128   }
5129   PetscCall(PetscPartitionerSetFromOptions(mesh->partitioner));
5130   PetscFunctionReturn(PETSC_SUCCESS);
5131 }
5132 
5133 PetscErrorCode DMSetFromOptions_Overlap_Plex(DM dm, PetscOptionItems PetscOptionsObject, PetscInt *overlap)
5134 {
5135   PetscInt  numOvLabels = 16, numOvExLabels = 16;
5136   char     *ovLabelNames[16], *ovExLabelNames[16];
5137   PetscInt  numOvValues = 16, numOvExValues = 16, l;
5138   PetscBool flg;
5139 
5140   PetscFunctionBegin;
5141   PetscCall(PetscOptionsBoundedInt("-dm_distribute_overlap", "The size of the overlap halo", "DMPlexDistribute", *overlap, overlap, NULL, 0));
5142   PetscCall(PetscOptionsStringArray("-dm_distribute_overlap_labels", "List of overlap label names", "DMPlexDistribute", ovLabelNames, &numOvLabels, &flg));
5143   if (!flg) numOvLabels = 0;
5144   if (numOvLabels) {
5145     ((DM_Plex *)dm->data)->numOvLabels = numOvLabels;
5146     for (l = 0; l < numOvLabels; ++l) {
5147       PetscCall(DMGetLabel(dm, ovLabelNames[l], &((DM_Plex *)dm->data)->ovLabels[l]));
5148       PetscCheck(((DM_Plex *)dm->data)->ovLabels[l], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid label name %s", ovLabelNames[l]);
5149       PetscCall(PetscFree(ovLabelNames[l]));
5150     }
5151     PetscCall(PetscOptionsIntArray("-dm_distribute_overlap_values", "List of overlap label values", "DMPlexDistribute", ((DM_Plex *)dm->data)->ovValues, &numOvValues, &flg));
5152     if (!flg) numOvValues = 0;
5153     PetscCheck(numOvLabels == numOvValues, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "The number of labels %" PetscInt_FMT " must match the number of values %" PetscInt_FMT, numOvLabels, numOvValues);
5154 
5155     PetscCall(PetscOptionsStringArray("-dm_distribute_overlap_exclude_labels", "List of overlap exclude label names", "DMPlexDistribute", ovExLabelNames, &numOvExLabels, &flg));
5156     if (!flg) numOvExLabels = 0;
5157     ((DM_Plex *)dm->data)->numOvExLabels = numOvExLabels;
5158     for (l = 0; l < numOvExLabels; ++l) {
5159       PetscCall(DMGetLabel(dm, ovExLabelNames[l], &((DM_Plex *)dm->data)->ovExLabels[l]));
5160       PetscCheck(((DM_Plex *)dm->data)->ovExLabels[l], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid label name %s", ovExLabelNames[l]);
5161       PetscCall(PetscFree(ovExLabelNames[l]));
5162     }
5163     PetscCall(PetscOptionsIntArray("-dm_distribute_overlap_exclude_values", "List of overlap exclude label values", "DMPlexDistribute", ((DM_Plex *)dm->data)->ovExValues, &numOvExValues, &flg));
5164     if (!flg) numOvExValues = 0;
5165     PetscCheck(numOvExLabels == numOvExValues, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "The number of exclude labels %" PetscInt_FMT " must match the number of values %" PetscInt_FMT, numOvExLabels, numOvExValues);
5166   }
5167   PetscFunctionReturn(PETSC_SUCCESS);
5168 }
5169 
5170 static PetscErrorCode DMSetFromOptions_Plex(DM dm, PetscOptionItems PetscOptionsObject)
5171 {
5172   PetscFunctionList    ordlist;
5173   char                 oname[256];
5174   char                 sublabelname[PETSC_MAX_PATH_LEN] = "";
5175   DMReorderDefaultFlag reorder;
5176   PetscReal            volume    = -1.0;
5177   PetscInt             prerefine = 0, refine = 0, r, coarsen = 0, overlap = 0, extLayers = 0, dim;
5178   PetscBool            uniformOrig = PETSC_FALSE, uniform = PETSC_TRUE, distribute, saveSF = PETSC_FALSE, interpolate = PETSC_TRUE, coordSpace = PETSC_FALSE, remap = PETSC_TRUE, ghostCells = PETSC_FALSE, isHierarchy, flg;
5179 
5180   PetscFunctionBegin;
5181   PetscOptionsHeadBegin(PetscOptionsObject, "DMPlex Options");
5182   if (dm->cloneOpts) goto non_refine;
5183   /* Handle automatic creation */
5184   PetscCall(DMGetDimension(dm, &dim));
5185   if (dim < 0) PetscCall(DMPlexCreateFromOptions_Internal(PetscOptionsObject, &coordSpace, dm));
5186   PetscCall(DMGetDimension(dm, &dim));
5187   /* Handle interpolation before distribution */
5188   PetscCall(PetscOptionsBool("-dm_plex_interpolate_pre", "Flag to interpolate mesh before distribution", "", interpolate, &interpolate, &flg));
5189   if (flg) {
5190     DMPlexInterpolatedFlag interpolated;
5191 
5192     PetscCall(DMPlexIsInterpolated(dm, &interpolated));
5193     if (interpolated == DMPLEX_INTERPOLATED_FULL && !interpolate) {
5194       DM udm;
5195 
5196       PetscCall(DMPlexUninterpolate(dm, &udm));
5197       PetscCall(DMPlexReplace_Internal(dm, &udm));
5198     } else if (interpolated != DMPLEX_INTERPOLATED_FULL && interpolate) {
5199       DM idm;
5200 
5201       PetscCall(DMPlexInterpolate(dm, &idm));
5202       PetscCall(DMPlexReplace_Internal(dm, &idm));
5203     }
5204   }
5205   // Handle submesh selection before distribution
5206   PetscCall(PetscOptionsString("-dm_plex_submesh", "Label to use for submesh selection", "", sublabelname, sublabelname, PETSC_MAX_PATH_LEN, &flg));
5207   if (flg) {
5208     DM              subdm;
5209     DMLabel         label;
5210     IS              valueIS, pointIS;
5211     const PetscInt *values, *points;
5212     PetscBool       markedFaces = PETSC_FALSE;
5213     PetscInt        Nv, value, Np;
5214 
5215     PetscCall(DMGetLabel(dm, sublabelname, &label));
5216     PetscCall(DMLabelGetNumValues(label, &Nv));
5217     PetscCheck(Nv == 1, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Only a single label value is currently supported for submesh selection, not %" PetscInt_FMT, Nv);
5218     PetscCall(DMLabelGetValueIS(label, &valueIS));
5219     PetscCall(ISGetIndices(valueIS, &values));
5220     value = values[0];
5221     PetscCall(ISRestoreIndices(valueIS, &values));
5222     PetscCall(ISDestroy(&valueIS));
5223     PetscCall(DMLabelGetStratumSize(label, value, &Np));
5224     PetscCall(DMLabelGetStratumIS(label, value, &pointIS));
5225     PetscCall(ISGetIndices(pointIS, &points));
5226     for (PetscInt p = 0; p < Np; ++p) {
5227       PetscInt pdepth;
5228 
5229       PetscCall(DMPlexGetPointDepth(dm, points[p], &pdepth));
5230       if (pdepth) {
5231         markedFaces = PETSC_TRUE;
5232         break;
5233       }
5234     }
5235     PetscCall(ISRestoreIndices(pointIS, &points));
5236     PetscCall(ISDestroy(&pointIS));
5237     PetscCall(DMPlexCreateSubmesh(dm, label, value, markedFaces, &subdm));
5238     PetscCall(DMPlexReplace_Internal(dm, &subdm));
5239     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
5240   }
5241   /* Handle DMPlex refinement before distribution */
5242   PetscCall(DMPlexGetRefinementUniform(dm, &uniformOrig));
5243   PetscCall(PetscOptionsBoundedInt("-dm_refine_pre", "The number of refinements before distribution", "DMCreate", prerefine, &prerefine, NULL, 0));
5244   PetscCall(PetscOptionsBool("-dm_refine_remap_pre", "Flag to control coordinate remapping", "DMCreate", remap, &remap, NULL));
5245   PetscCall(PetscOptionsBool("-dm_refine_uniform_pre", "Flag for uniform refinement before distribution", "DMCreate", uniform, &uniform, &flg));
5246   if (flg) PetscCall(DMPlexSetRefinementUniform(dm, uniform));
5247   PetscCall(PetscOptionsReal("-dm_refine_volume_limit_pre", "The maximum cell volume after refinement before distribution", "DMCreate", volume, &volume, &flg));
5248   if (flg) {
5249     PetscCall(DMPlexSetRefinementUniform(dm, PETSC_FALSE));
5250     PetscCall(DMPlexSetRefinementLimit(dm, volume));
5251     prerefine = PetscMax(prerefine, 1);
5252   }
5253   if (prerefine) PetscCall(DMLocalizeCoordinates(dm));
5254   for (r = 0; r < prerefine; ++r) {
5255     DM             rdm;
5256     PetscPointFunc coordFunc = ((DM_Plex *)dm->data)->coordFunc;
5257 
5258     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
5259     PetscCall(DMRefine(dm, PetscObjectComm((PetscObject)dm), &rdm));
5260     PetscCall(DMPlexReplace_Internal(dm, &rdm));
5261     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
5262     if (coordFunc && remap) {
5263       PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
5264       ((DM_Plex *)dm->data)->coordFunc = coordFunc;
5265     }
5266   }
5267   PetscCall(DMPlexSetRefinementUniform(dm, uniformOrig));
5268   /* Handle DMPlex extrusion before distribution */
5269   PetscCall(PetscOptionsBoundedInt("-dm_extrude", "The number of layers to extrude", "", extLayers, &extLayers, NULL, 0));
5270   if (extLayers) {
5271     DM edm;
5272 
5273     PetscCall(DMExtrude(dm, extLayers, &edm));
5274     PetscCall(DMPlexReplace_Internal(dm, &edm));
5275     ((DM_Plex *)dm->data)->coordFunc = NULL;
5276     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
5277     extLayers = 0;
5278     PetscCall(DMGetDimension(dm, &dim));
5279   }
5280   /* Handle DMPlex reordering before distribution */
5281   PetscCall(DMPlexReorderGetDefault(dm, &reorder));
5282   PetscCall(MatGetOrderingList(&ordlist));
5283   PetscCall(PetscStrncpy(oname, MATORDERINGNATURAL, sizeof(oname)));
5284   PetscCall(PetscOptionsFList("-dm_plex_reorder", "Set mesh reordering type", "DMPlexGetOrdering", ordlist, MATORDERINGNATURAL, oname, sizeof(oname), &flg));
5285   if (reorder == DM_REORDER_DEFAULT_TRUE || flg) {
5286     DM pdm;
5287     IS perm;
5288 
5289     PetscCall(DMPlexGetOrdering(dm, oname, NULL, &perm));
5290     PetscCall(DMPlexPermute(dm, perm, &pdm));
5291     PetscCall(ISDestroy(&perm));
5292     PetscCall(DMPlexReplace_Internal(dm, &pdm));
5293     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
5294   }
5295   /* Handle DMPlex distribution */
5296   PetscCall(DMPlexDistributeGetDefault(dm, &distribute));
5297   PetscCall(PetscOptionsBool("-dm_distribute", "Flag to redistribute a mesh among processes", "DMPlexDistribute", distribute, &distribute, NULL));
5298   PetscCall(PetscOptionsBool("-dm_distribute_save_sf", "Flag to save the migration SF", "DMPlexSetMigrationSF", saveSF, &saveSF, NULL));
5299   PetscCall(DMSetFromOptions_Overlap_Plex(dm, PetscOptionsObject, &overlap));
5300   if (distribute) {
5301     DM               pdm = NULL;
5302     PetscPartitioner part;
5303     PetscSF          sfMigration;
5304     PetscBool        use_partition_balance;
5305 
5306     PetscCall(DMPlexGetPartitioner(dm, &part));
5307     PetscCall(PetscPartitionerSetFromOptions(part));
5308     PetscCall(DMPlexGetPartitionBalance(dm, &use_partition_balance));
5309     PetscCall(PetscOptionsBool("-dm_plex_partition_balance", "Attempt to evenly divide points on partition boundary between processes", "DMPlexSetPartitionBalance", use_partition_balance, &use_partition_balance, &flg));
5310     if (flg) PetscCall(DMPlexSetPartitionBalance(dm, use_partition_balance));
5311     PetscCall(DMPlexDistribute(dm, overlap, &sfMigration, &pdm));
5312     if (pdm) {
5313       // Delete the local section to force the existing one to be rebuilt with the distributed DM
5314       PetscCall(DMSetLocalSection(dm, pdm->localSection));
5315       PetscCall(DMPlexReplace_Internal(dm, &pdm));
5316     }
5317     if (saveSF) PetscCall(DMPlexSetMigrationSF(dm, sfMigration));
5318     PetscCall(PetscSFDestroy(&sfMigration));
5319   }
5320 
5321   {
5322     PetscBool useBoxLabel = PETSC_FALSE;
5323     PetscCall(PetscOptionsBool("-dm_plex_box_label", "Create 'Face Sets' assuming boundary faces align with cartesian directions", "DMCreate", useBoxLabel, &useBoxLabel, NULL));
5324     if (useBoxLabel) {
5325       PetscInt       n      = 3;
5326       DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
5327 
5328       PetscCall(PetscOptionsEnumArray("-dm_plex_box_label_bd", "Boundary type for each dimension when using -dm_plex_box_label", "", DMBoundaryTypes, (PetscEnum *)bdt, &n, &flg));
5329       PetscCheck(!flg || !(n != dim), PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_SIZ, "Box boundary types had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
5330       PetscCall(DMPlexSetBoxLabel_Internal(dm, bdt));
5331     }
5332   }
5333   /* Must check CEED options before creating function space for coordinates */
5334   {
5335     PetscBool useCeed = PETSC_FALSE, flg;
5336 
5337     PetscCall(PetscOptionsBool("-dm_plex_use_ceed", "Use LibCEED as the FEM backend", "DMPlexSetUseCeed", useCeed, &useCeed, &flg));
5338     if (flg) PetscCall(DMPlexSetUseCeed(dm, useCeed));
5339   }
5340   /* Create coordinate space */
5341   PetscCall(PetscOptionsBool("-dm_coord_space", "Use an FEM space for coordinates", "", coordSpace, &coordSpace, NULL));
5342   if (coordSpace) {
5343     DM_Plex  *mesh   = (DM_Plex *)dm->data;
5344     PetscInt  degree = 1, deg;
5345     PetscInt  height = 0;
5346     DM        cdm;
5347     PetscBool localize = PETSC_TRUE, sparseLocalize = PETSC_TRUE;
5348 
5349     PetscCall(PetscOptionsInt("-dm_coord_petscspace_degree", "FEM degree for coordinate space", "", degree, &degree, NULL));
5350     PetscCall(DMGetCoordinateDegree_Internal(dm, &deg));
5351     if (coordSpace && deg <= 1) PetscCall(DMPlexCreateCoordinateSpace(dm, degree, PETSC_TRUE, mesh->coordFunc));
5352     PetscCall(DMGetCoordinateDM(dm, &cdm));
5353     if (!coordSpace) {
5354       PetscDS      cds;
5355       PetscObject  obj;
5356       PetscClassId id;
5357 
5358       PetscCall(DMGetDS(cdm, &cds));
5359       PetscCall(PetscDSGetDiscretization(cds, 0, &obj));
5360       PetscCall(PetscObjectGetClassId(obj, &id));
5361       if (id == PETSCFE_CLASSID) {
5362         PetscContainer dummy;
5363 
5364         PetscCall(PetscContainerCreate(PETSC_COMM_SELF, &dummy));
5365         PetscCall(PetscObjectSetName((PetscObject)dummy, "coordinates"));
5366         PetscCall(DMSetField(cdm, 0, NULL, (PetscObject)dummy));
5367         PetscCall(PetscContainerDestroy(&dummy));
5368         PetscCall(DMClearDS(cdm));
5369       }
5370       mesh->coordFunc = NULL;
5371     }
5372     PetscCall(PetscOptionsBool("-dm_localize", "Localize mesh coordinates", "", localize, &localize, NULL));
5373     PetscCall(PetscOptionsBool("-dm_sparse_localize", "Localize only necessary cells", "DMSetSparseLocalize", sparseLocalize, &sparseLocalize, &flg));
5374     if (flg) PetscCall(DMSetSparseLocalize(dm, sparseLocalize));
5375     PetscCall(PetscOptionsInt("-dm_localize_height", "Localize edges and faces in addition to cells", "", height, &height, &flg));
5376     if (flg) PetscCall(DMPlexSetMaxProjectionHeight(cdm, height));
5377     if (localize) PetscCall(DMLocalizeCoordinates(dm));
5378   }
5379   /* Handle DMPlex refinement */
5380   remap = PETSC_TRUE;
5381   PetscCall(PetscOptionsBoundedInt("-dm_refine", "The number of uniform refinements", "DMCreate", refine, &refine, NULL, 0));
5382   PetscCall(PetscOptionsBool("-dm_refine_remap", "Flag to control coordinate remapping", "DMCreate", remap, &remap, NULL));
5383   PetscCall(PetscOptionsBoundedInt("-dm_refine_hierarchy", "The number of uniform refinements", "DMCreate", refine, &refine, &isHierarchy, 0));
5384   if (refine) PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
5385   if (refine && isHierarchy) {
5386     DM *dms, coarseDM;
5387 
5388     PetscCall(DMGetCoarseDM(dm, &coarseDM));
5389     PetscCall(PetscObjectReference((PetscObject)coarseDM));
5390     PetscCall(PetscMalloc1(refine, &dms));
5391     PetscCall(DMRefineHierarchy(dm, refine, dms));
5392     /* Total hack since we do not pass in a pointer */
5393     PetscCall(DMPlexSwap_Static(dm, dms[refine - 1]));
5394     if (refine == 1) {
5395       PetscCall(DMSetCoarseDM(dm, dms[0]));
5396       PetscCall(DMPlexSetRegularRefinement(dm, PETSC_TRUE));
5397     } else {
5398       PetscCall(DMSetCoarseDM(dm, dms[refine - 2]));
5399       PetscCall(DMPlexSetRegularRefinement(dm, PETSC_TRUE));
5400       PetscCall(DMSetCoarseDM(dms[0], dms[refine - 1]));
5401       PetscCall(DMPlexSetRegularRefinement(dms[0], PETSC_TRUE));
5402     }
5403     PetscCall(DMSetCoarseDM(dms[refine - 1], coarseDM));
5404     PetscCall(PetscObjectDereference((PetscObject)coarseDM));
5405     /* Free DMs */
5406     for (r = 0; r < refine; ++r) {
5407       PetscCall(DMSetFromOptions_NonRefinement_Plex(dms[r], PetscOptionsObject));
5408       PetscCall(DMDestroy(&dms[r]));
5409     }
5410     PetscCall(PetscFree(dms));
5411   } else {
5412     for (r = 0; r < refine; ++r) {
5413       DM             rdm;
5414       PetscPointFunc coordFunc = ((DM_Plex *)dm->data)->coordFunc;
5415 
5416       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
5417       PetscCall(DMRefine(dm, PetscObjectComm((PetscObject)dm), &rdm));
5418       /* Total hack since we do not pass in a pointer */
5419       PetscCall(DMPlexReplace_Internal(dm, &rdm));
5420       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
5421       if (coordFunc && remap) {
5422         PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
5423         ((DM_Plex *)dm->data)->coordFunc = coordFunc;
5424       }
5425     }
5426   }
5427   /* Handle DMPlex coarsening */
5428   PetscCall(PetscOptionsBoundedInt("-dm_coarsen", "Coarsen the mesh", "DMCreate", coarsen, &coarsen, NULL, 0));
5429   PetscCall(PetscOptionsBoundedInt("-dm_coarsen_hierarchy", "The number of coarsenings", "DMCreate", coarsen, &coarsen, &isHierarchy, 0));
5430   if (coarsen && isHierarchy) {
5431     DM *dms;
5432 
5433     PetscCall(PetscMalloc1(coarsen, &dms));
5434     PetscCall(DMCoarsenHierarchy(dm, coarsen, dms));
5435     /* Free DMs */
5436     for (r = 0; r < coarsen; ++r) {
5437       PetscCall(DMSetFromOptions_NonRefinement_Plex(dms[r], PetscOptionsObject));
5438       PetscCall(DMDestroy(&dms[r]));
5439     }
5440     PetscCall(PetscFree(dms));
5441   } else {
5442     for (r = 0; r < coarsen; ++r) {
5443       DM             cdm;
5444       PetscPointFunc coordFunc = ((DM_Plex *)dm->data)->coordFunc;
5445 
5446       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
5447       PetscCall(DMCoarsen(dm, PetscObjectComm((PetscObject)dm), &cdm));
5448       /* Total hack since we do not pass in a pointer */
5449       PetscCall(DMPlexReplace_Internal(dm, &cdm));
5450       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
5451       if (coordFunc) {
5452         PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
5453         ((DM_Plex *)dm->data)->coordFunc = coordFunc;
5454       }
5455     }
5456   }
5457   // Handle coordinate remapping
5458   remap = PETSC_FALSE;
5459   PetscCall(PetscOptionsBool("-dm_coord_remap", "Flag to control coordinate remapping", "", remap, &remap, NULL));
5460   if (remap) {
5461     DMPlexCoordMap map     = DM_COORD_MAP_NONE;
5462     PetscPointFunc mapFunc = NULL;
5463     PetscScalar    params[16];
5464     PetscInt       Np = PETSC_STATIC_ARRAY_LENGTH(params), cdim;
5465     MPI_Comm       comm;
5466 
5467     PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
5468     PetscCall(DMGetCoordinateDim(dm, &cdim));
5469     PetscCall(PetscOptionsScalarArray("-dm_coord_map_params", "Parameters for the coordinate remapping", "", params, &Np, &flg));
5470     if (!flg) Np = 0;
5471     // TODO Allow user to pass a map function by name
5472     PetscCall(PetscOptionsEnum("-dm_coord_map", "Coordinate mapping for built-in mesh", "", DMPlexCoordMaps, (PetscEnum)map, (PetscEnum *)&map, &flg));
5473     if (flg) {
5474       switch (map) {
5475       case DM_COORD_MAP_NONE:
5476         mapFunc = coordMap_identity;
5477         break;
5478       case DM_COORD_MAP_SHEAR:
5479         mapFunc = coordMap_shear;
5480         if (!Np) {
5481           Np        = cdim + 1;
5482           params[0] = 0;
5483           for (PetscInt d = 1; d <= cdim; ++d) params[d] = 1.0;
5484         }
5485         PetscCheck(Np == cdim + 1, comm, PETSC_ERR_ARG_WRONG, "The shear coordinate map must have cdim + 1 = %" PetscInt_FMT " parameters, not %" PetscInt_FMT, cdim + 1, Np);
5486         break;
5487       case DM_COORD_MAP_FLARE:
5488         mapFunc = coordMap_flare;
5489         if (!Np) {
5490           Np        = cdim + 1;
5491           params[0] = 0;
5492           for (PetscInt d = 1; d <= cdim; ++d) params[d] = 1.0;
5493         }
5494         PetscCheck(Np == cdim + 1, comm, PETSC_ERR_ARG_WRONG, "The flare coordinate map must have cdim + 1 = %" PetscInt_FMT " parameters, not %" PetscInt_FMT, cdim + 1, Np);
5495         break;
5496       case DM_COORD_MAP_ANNULUS:
5497         mapFunc = coordMap_annulus;
5498         if (!Np) {
5499           Np        = 2;
5500           params[0] = 1.;
5501           params[1] = 2.;
5502         }
5503         PetscCheck(Np == 2, comm, PETSC_ERR_ARG_WRONG, "The annulus coordinate map must have 2 parameters, not %" PetscInt_FMT, Np);
5504         break;
5505       case DM_COORD_MAP_SHELL:
5506         mapFunc = coordMap_shell;
5507         if (!Np) {
5508           Np        = 2;
5509           params[0] = 1.;
5510           params[1] = 2.;
5511         }
5512         PetscCheck(Np == 2, comm, PETSC_ERR_ARG_WRONG, "The spherical shell coordinate map must have 2 parameters, not %" PetscInt_FMT, Np);
5513         break;
5514       case DM_COORD_MAP_SINUSOID:
5515         mapFunc = coordMap_sinusoid;
5516         if (!Np) {
5517           Np        = 3;
5518           params[0] = 1.;
5519           params[1] = 1.;
5520           params[2] = 1.;
5521         }
5522         PetscCheck(Np == 3, comm, PETSC_ERR_ARG_WRONG, "The sinusoidal coordinate map must have 3 parameters, not %" PetscInt_FMT, Np);
5523         break;
5524       default:
5525         mapFunc = coordMap_identity;
5526       }
5527     }
5528     if (Np) {
5529       DM      cdm;
5530       PetscDS cds;
5531 
5532       PetscCall(DMGetCoordinateDM(dm, &cdm));
5533       PetscCall(DMGetDS(cdm, &cds));
5534       PetscCall(PetscDSSetConstants(cds, Np, params));
5535     }
5536     PetscCall(DMPlexRemapGeometry(dm, 0.0, mapFunc));
5537   }
5538   /* Handle ghost cells */
5539   PetscCall(PetscOptionsBool("-dm_plex_create_fv_ghost_cells", "Flag to create finite volume ghost cells on the boundary", "DMCreate", ghostCells, &ghostCells, NULL));
5540   if (ghostCells) {
5541     DM   gdm;
5542     char lname[PETSC_MAX_PATH_LEN];
5543 
5544     lname[0] = '\0';
5545     PetscCall(PetscOptionsString("-dm_plex_fv_ghost_cells_label", "Label name for ghost cells boundary", "DMCreate", lname, lname, sizeof(lname), &flg));
5546     PetscCall(DMPlexConstructGhostCells(dm, flg ? lname : NULL, NULL, &gdm));
5547     PetscCall(DMPlexReplace_Internal(dm, &gdm));
5548   }
5549   /* Handle 1D order */
5550   if (reorder != DM_REORDER_DEFAULT_FALSE && dim == 1) {
5551     DM           cdm, rdm;
5552     PetscDS      cds;
5553     PetscObject  obj;
5554     PetscClassId id = PETSC_OBJECT_CLASSID;
5555     IS           perm;
5556     PetscInt     Nf;
5557     PetscBool    distributed;
5558 
5559     PetscCall(DMPlexIsDistributed(dm, &distributed));
5560     PetscCall(DMGetCoordinateDM(dm, &cdm));
5561     PetscCall(DMGetDS(cdm, &cds));
5562     PetscCall(PetscDSGetNumFields(cds, &Nf));
5563     if (Nf) {
5564       PetscCall(PetscDSGetDiscretization(cds, 0, &obj));
5565       PetscCall(PetscObjectGetClassId(obj, &id));
5566     }
5567     if (!distributed && id != PETSCFE_CLASSID) {
5568       PetscCall(DMPlexGetOrdering1D(dm, &perm));
5569       PetscCall(DMPlexPermute(dm, perm, &rdm));
5570       PetscCall(DMPlexReplace_Internal(dm, &rdm));
5571       PetscCall(ISDestroy(&perm));
5572     }
5573   }
5574 /* Handle */
5575 non_refine:
5576   PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
5577   char    *phases[16];
5578   PetscInt Nphases = 16;
5579   PetscCall(PetscOptionsStringArray("-dm_plex_option_phases", "Option phase prefixes", "DMSetFromOptions", phases, &Nphases, &flg));
5580   PetscOptionsHeadEnd();
5581 
5582   // Phases
5583   if (flg) {
5584     DM          cdm;
5585     char       *oldPrefix, *oldCoordPrefix;
5586     const char *tmp;
5587 
5588     PetscCall(DMGetCoordinateDM(dm, &cdm));
5589     PetscCall(PetscObjectGetOptionsPrefix((PetscObject)dm, &tmp));
5590     PetscCall(PetscStrallocpy(tmp, &oldPrefix));
5591     PetscCall(PetscObjectGetOptionsPrefix((PetscObject)cdm, &tmp));
5592     PetscCall(PetscStrallocpy(tmp, &oldCoordPrefix));
5593     for (PetscInt ph = 0; ph < Nphases; ++ph) {
5594       PetscCall(PetscObjectAppendOptionsPrefix((PetscObject)dm, phases[ph]));
5595       PetscCall(PetscObjectAppendOptionsPrefix((PetscObject)cdm, phases[ph]));
5596       PetscCall(PetscInfo(dm, "Options phase %s for DM %s\n", phases[ph], dm->hdr.name));
5597       PetscCall(DMSetFromOptions(dm));
5598       PetscCall(PetscObjectSetOptionsPrefix((PetscObject)dm, oldPrefix));
5599       PetscCall(DMGetCoordinateDM(dm, &cdm));
5600       PetscCall(PetscObjectSetOptionsPrefix((PetscObject)cdm, oldCoordPrefix));
5601       PetscCall(PetscFree(phases[ph]));
5602     }
5603     PetscCall(PetscFree(oldPrefix));
5604     PetscCall(PetscFree(oldCoordPrefix));
5605   }
5606   PetscFunctionReturn(PETSC_SUCCESS);
5607 }
5608 
5609 static PetscErrorCode DMCreateGlobalVector_Plex(DM dm, Vec *vec)
5610 {
5611   PetscFunctionBegin;
5612   PetscCall(DMCreateGlobalVector_Section_Private(dm, vec));
5613   /* PetscCall(VecSetOperation(*vec, VECOP_DUPLICATE, (void(*)(void)) VecDuplicate_MPI_DM)); */
5614   PetscCall(VecSetOperation(*vec, VECOP_VIEW, (void (*)(void))VecView_Plex));
5615   PetscCall(VecSetOperation(*vec, VECOP_VIEWNATIVE, (void (*)(void))VecView_Plex_Native));
5616   PetscCall(VecSetOperation(*vec, VECOP_LOAD, (void (*)(void))VecLoad_Plex));
5617   PetscCall(VecSetOperation(*vec, VECOP_LOADNATIVE, (void (*)(void))VecLoad_Plex_Native));
5618   PetscFunctionReturn(PETSC_SUCCESS);
5619 }
5620 
5621 static PetscErrorCode DMCreateLocalVector_Plex(DM dm, Vec *vec)
5622 {
5623   PetscFunctionBegin;
5624   PetscCall(DMCreateLocalVector_Section_Private(dm, vec));
5625   PetscCall(VecSetOperation(*vec, VECOP_VIEW, (void (*)(void))VecView_Plex_Local));
5626   PetscCall(VecSetOperation(*vec, VECOP_LOAD, (void (*)(void))VecLoad_Plex_Local));
5627   PetscFunctionReturn(PETSC_SUCCESS);
5628 }
5629 
5630 static PetscErrorCode DMGetDimPoints_Plex(DM dm, PetscInt dim, PetscInt *pStart, PetscInt *pEnd)
5631 {
5632   PetscInt depth, d;
5633 
5634   PetscFunctionBegin;
5635   PetscCall(DMPlexGetDepth(dm, &depth));
5636   if (depth == 1) {
5637     PetscCall(DMGetDimension(dm, &d));
5638     if (dim == 0) PetscCall(DMPlexGetDepthStratum(dm, dim, pStart, pEnd));
5639     else if (dim == d) PetscCall(DMPlexGetDepthStratum(dm, 1, pStart, pEnd));
5640     else {
5641       *pStart = 0;
5642       *pEnd   = 0;
5643     }
5644   } else {
5645     PetscCall(DMPlexGetDepthStratum(dm, dim, pStart, pEnd));
5646   }
5647   PetscFunctionReturn(PETSC_SUCCESS);
5648 }
5649 
5650 static PetscErrorCode DMGetNeighbors_Plex(DM dm, PetscInt *nranks, const PetscMPIInt *ranks[])
5651 {
5652   PetscSF            sf;
5653   PetscMPIInt        niranks, njranks;
5654   PetscInt           n;
5655   const PetscMPIInt *iranks, *jranks;
5656   DM_Plex           *data = (DM_Plex *)dm->data;
5657 
5658   PetscFunctionBegin;
5659   PetscCall(DMGetPointSF(dm, &sf));
5660   if (!data->neighbors) {
5661     PetscCall(PetscSFSetUp(sf));
5662     PetscCall(PetscSFGetRootRanks(sf, &njranks, &jranks, NULL, NULL, NULL));
5663     PetscCall(PetscSFGetLeafRanks(sf, &niranks, &iranks, NULL, NULL));
5664     PetscCall(PetscMalloc1(njranks + niranks + 1, &data->neighbors));
5665     PetscCall(PetscArraycpy(data->neighbors + 1, jranks, njranks));
5666     PetscCall(PetscArraycpy(data->neighbors + njranks + 1, iranks, niranks));
5667     n = njranks + niranks;
5668     PetscCall(PetscSortRemoveDupsMPIInt(&n, data->neighbors + 1));
5669     /* The following cast should never fail: can't have more neighbors than PETSC_MPI_INT_MAX */
5670     PetscCall(PetscMPIIntCast(n, data->neighbors));
5671   }
5672   if (nranks) *nranks = data->neighbors[0];
5673   if (ranks) {
5674     if (data->neighbors[0]) *ranks = data->neighbors + 1;
5675     else *ranks = NULL;
5676   }
5677   PetscFunctionReturn(PETSC_SUCCESS);
5678 }
5679 
5680 PETSC_INTERN PetscErrorCode DMInterpolateSolution_Plex(DM, DM, Mat, Vec, Vec);
5681 
5682 static PetscErrorCode DMInitialize_Plex(DM dm)
5683 {
5684   PetscFunctionBegin;
5685   dm->ops->view                      = DMView_Plex;
5686   dm->ops->load                      = DMLoad_Plex;
5687   dm->ops->setfromoptions            = DMSetFromOptions_Plex;
5688   dm->ops->clone                     = DMClone_Plex;
5689   dm->ops->setup                     = DMSetUp_Plex;
5690   dm->ops->createlocalsection        = DMCreateLocalSection_Plex;
5691   dm->ops->createsectionpermutation  = DMCreateSectionPermutation_Plex;
5692   dm->ops->createdefaultconstraints  = DMCreateDefaultConstraints_Plex;
5693   dm->ops->createglobalvector        = DMCreateGlobalVector_Plex;
5694   dm->ops->createlocalvector         = DMCreateLocalVector_Plex;
5695   dm->ops->getlocaltoglobalmapping   = NULL;
5696   dm->ops->createfieldis             = NULL;
5697   dm->ops->createcoordinatedm        = DMCreateCoordinateDM_Plex;
5698   dm->ops->createcoordinatefield     = DMCreateCoordinateField_Plex;
5699   dm->ops->getcoloring               = NULL;
5700   dm->ops->creatematrix              = DMCreateMatrix_Plex;
5701   dm->ops->createinterpolation       = DMCreateInterpolation_Plex;
5702   dm->ops->createmassmatrix          = DMCreateMassMatrix_Plex;
5703   dm->ops->createmassmatrixlumped    = DMCreateMassMatrixLumped_Plex;
5704   dm->ops->createinjection           = DMCreateInjection_Plex;
5705   dm->ops->refine                    = DMRefine_Plex;
5706   dm->ops->coarsen                   = DMCoarsen_Plex;
5707   dm->ops->refinehierarchy           = DMRefineHierarchy_Plex;
5708   dm->ops->coarsenhierarchy          = DMCoarsenHierarchy_Plex;
5709   dm->ops->extrude                   = DMExtrude_Plex;
5710   dm->ops->globaltolocalbegin        = NULL;
5711   dm->ops->globaltolocalend          = NULL;
5712   dm->ops->localtoglobalbegin        = NULL;
5713   dm->ops->localtoglobalend          = NULL;
5714   dm->ops->destroy                   = DMDestroy_Plex;
5715   dm->ops->createsubdm               = DMCreateSubDM_Plex;
5716   dm->ops->createsuperdm             = DMCreateSuperDM_Plex;
5717   dm->ops->getdimpoints              = DMGetDimPoints_Plex;
5718   dm->ops->locatepoints              = DMLocatePoints_Plex;
5719   dm->ops->projectfunctionlocal      = DMProjectFunctionLocal_Plex;
5720   dm->ops->projectfunctionlabellocal = DMProjectFunctionLabelLocal_Plex;
5721   dm->ops->projectfieldlocal         = DMProjectFieldLocal_Plex;
5722   dm->ops->projectfieldlabellocal    = DMProjectFieldLabelLocal_Plex;
5723   dm->ops->projectbdfieldlabellocal  = DMProjectBdFieldLabelLocal_Plex;
5724   dm->ops->computel2diff             = DMComputeL2Diff_Plex;
5725   dm->ops->computel2gradientdiff     = DMComputeL2GradientDiff_Plex;
5726   dm->ops->computel2fielddiff        = DMComputeL2FieldDiff_Plex;
5727   dm->ops->getneighbors              = DMGetNeighbors_Plex;
5728   dm->ops->getlocalboundingbox       = DMGetLocalBoundingBox_Coordinates;
5729   dm->ops->createdomaindecomposition = DMCreateDomainDecomposition_Plex;
5730   dm->ops->createddscatters          = DMCreateDomainDecompositionScatters_Plex;
5731   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexInsertBoundaryValues_C", DMPlexInsertBoundaryValues_Plex));
5732   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexInsertTimeDerivativeBoundaryValues_C", DMPlexInsertTimeDerivativeBoundaryValues_Plex));
5733   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMSetUpGLVisViewer_C", DMSetUpGLVisViewer_Plex));
5734   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMCreateNeumannOverlap_C", DMCreateNeumannOverlap_Plex));
5735   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexDistributeGetDefault_C", DMPlexDistributeGetDefault_Plex));
5736   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexDistributeSetDefault_C", DMPlexDistributeSetDefault_Plex));
5737   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexReorderGetDefault_C", DMPlexReorderGetDefault_Plex));
5738   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexReorderSetDefault_C", DMPlexReorderSetDefault_Plex));
5739   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMReorderSectionGetDefault_C", DMReorderSectionGetDefault_Plex));
5740   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMReorderSectionSetDefault_C", DMReorderSectionSetDefault_Plex));
5741   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMReorderSectionGetType_C", DMReorderSectionGetType_Plex));
5742   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMReorderSectionSetType_C", DMReorderSectionSetType_Plex));
5743   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMInterpolateSolution_C", DMInterpolateSolution_Plex));
5744   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexGetOverlap_C", DMPlexGetOverlap_Plex));
5745   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexSetOverlap_C", DMPlexSetOverlap_Plex));
5746   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexGetUseCeed_C", DMPlexGetUseCeed_Plex));
5747   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexSetUseCeed_C", DMPlexSetUseCeed_Plex));
5748   PetscFunctionReturn(PETSC_SUCCESS);
5749 }
5750 
5751 PETSC_INTERN PetscErrorCode DMClone_Plex(DM dm, DM *newdm)
5752 {
5753   DM_Plex       *mesh = (DM_Plex *)dm->data;
5754   const PetscSF *face_sfs;
5755   PetscInt       num_face_sfs;
5756 
5757   PetscFunctionBegin;
5758   mesh->refct++;
5759   (*newdm)->data = mesh;
5760   PetscCall(DMPlexGetIsoperiodicFaceSF(dm, &num_face_sfs, &face_sfs));
5761   PetscCall(DMPlexSetIsoperiodicFaceSF(*newdm, num_face_sfs, (PetscSF *)face_sfs));
5762   PetscCall(PetscObjectChangeTypeName((PetscObject)*newdm, DMPLEX));
5763   PetscCall(DMInitialize_Plex(*newdm));
5764   PetscFunctionReturn(PETSC_SUCCESS);
5765 }
5766 
5767 /*MC
5768   DMPLEX = "plex" - A `DM` object that encapsulates an unstructured mesh (or grid), or CW Complex {cite}`cwcomplex`,
5769            which can be expressed using a Hasse Diagram {cite}`hassediagram`.
5770            In the local representation, `Vec`s contain all unknowns in the interior and shared boundary. This is
5771            specified by a `PetscSection` object. Ownership in the global representation is determined by
5772            ownership of the underlying `DMPLEX` points. This is specified by another `PetscSection` object.
5773 
5774   Options Database Keys:
5775 + -dm_refine_pre                     - Refine mesh before distribution
5776 + -dm_refine_uniform_pre             - Choose uniform or generator-based refinement
5777 + -dm_refine_volume_limit_pre        - Cell volume limit after pre-refinement using generator
5778 . -dm_distribute                     - Distribute mesh across processes
5779 . -dm_distribute_overlap             - Number of cells to overlap for distribution
5780 . -dm_refine                         - Refine mesh after distribution
5781 . -dm_localize <bool>                - Whether to localize coordinates for periodic meshes
5782 . -dm_sparse_localize <bool>         - Whether to only localize cells on the periodic boundary
5783 . -dm_plex_hash_location             - Use grid hashing for point location
5784 . -dm_plex_hash_box_faces <n,m,p>    - The number of divisions in each direction of the grid hash
5785 . -dm_plex_partition_balance         - Attempt to evenly divide points on partition boundary between processes
5786 . -dm_plex_remesh_bd                 - Allow changes to the boundary on remeshing
5787 . -dm_plex_max_projection_height     - Maximum mesh point height used to project locally
5788 . -dm_plex_regular_refinement        - Use special nested projection algorithm for regular refinement
5789 . -dm_plex_reorder_section           - Use specialized blocking if available
5790 . -dm_plex_check_all                 - Perform all checks below
5791 . -dm_plex_check_symmetry            - Check that the adjacency information in the mesh is symmetric
5792 . -dm_plex_check_skeleton <celltype> - Check that each cell has the correct number of vertices
5793 . -dm_plex_check_faces <celltype>    - Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type
5794 . -dm_plex_check_geometry            - Check that cells have positive volume
5795 . -dm_view :mesh.tex:ascii_latex     - View the mesh in LaTeX/TikZ
5796 . -dm_plex_view_scale <num>          - Scale the TikZ
5797 . -dm_plex_print_fem <num>           - View FEM assembly information, such as element vectors and matrices
5798 - -dm_plex_print_fvm <num>           - View FVM assembly information, such as flux updates
5799 
5800   Level: intermediate
5801 
5802 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMType`, `DMPlexCreate()`, `DMCreate()`, `DMSetType()`, `PetscSection`
5803 M*/
5804 
5805 PETSC_EXTERN PetscErrorCode DMCreate_Plex(DM dm)
5806 {
5807   DM_Plex *mesh;
5808   PetscInt unit;
5809 
5810   PetscFunctionBegin;
5811   PetscCall(PetscCitationsRegister(PlexCitation, &Plexcite));
5812   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5813   PetscCall(PetscNew(&mesh));
5814   dm->reorderSection = DM_REORDER_DEFAULT_NOTSET;
5815   dm->data           = mesh;
5816 
5817   mesh->refct = 1;
5818   PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->coneSection));
5819   PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->supportSection));
5820   mesh->refinementUniform       = PETSC_TRUE;
5821   mesh->refinementLimit         = -1.0;
5822   mesh->distDefault             = PETSC_TRUE;
5823   mesh->reorderDefault          = DM_REORDER_DEFAULT_NOTSET;
5824   mesh->distributionName        = NULL;
5825   mesh->interpolated            = DMPLEX_INTERPOLATED_INVALID;
5826   mesh->interpolatedCollective  = DMPLEX_INTERPOLATED_INVALID;
5827   mesh->interpolatePreferTensor = PETSC_TRUE;
5828 
5829   PetscCall(PetscPartitionerCreate(PetscObjectComm((PetscObject)dm), &mesh->partitioner));
5830   mesh->remeshBd = PETSC_FALSE;
5831 
5832   for (unit = 0; unit < NUM_PETSC_UNITS; ++unit) mesh->scale[unit] = 1.0;
5833 
5834   mesh->depthState    = -1;
5835   mesh->celltypeState = -1;
5836   mesh->printTol      = 1.0e-10;
5837   mesh->nonempty_comm = MPI_COMM_SELF;
5838 
5839   PetscCall(DMInitialize_Plex(dm));
5840   PetscFunctionReturn(PETSC_SUCCESS);
5841 }
5842 
5843 /*@
5844   DMPlexCreate - Creates a `DMPLEX` object, which encapsulates an unstructured mesh, or CW complex, which can be expressed using a Hasse Diagram.
5845 
5846   Collective
5847 
5848   Input Parameter:
5849 . comm - The communicator for the `DMPLEX` object
5850 
5851   Output Parameter:
5852 . mesh - The `DMPLEX` object
5853 
5854   Level: beginner
5855 
5856 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMType`, `DMCreate()`, `DMSetType()`
5857 @*/
5858 PetscErrorCode DMPlexCreate(MPI_Comm comm, DM *mesh)
5859 {
5860   PetscFunctionBegin;
5861   PetscAssertPointer(mesh, 2);
5862   PetscCall(DMCreate(comm, mesh));
5863   PetscCall(DMSetType(*mesh, DMPLEX));
5864   PetscFunctionReturn(PETSC_SUCCESS);
5865 }
5866 
5867 /*@C
5868   DMPlexBuildFromCellListParallel - Build distributed `DMPLEX` topology from a list of vertices for each cell (common mesh generator output) where all cells have the same celltype
5869 
5870   Collective; No Fortran Support
5871 
5872   Input Parameters:
5873 + dm          - The `DM`
5874 . numCells    - The number of cells owned by this process
5875 . numVertices - The number of vertices to be owned by this process, or `PETSC_DECIDE`
5876 . NVertices   - The global number of vertices, or `PETSC_DETERMINE`
5877 . numCorners  - The number of vertices for each cell
5878 - cells       - An array of numCells*numCorners numbers, the global vertex numbers for each cell
5879 
5880   Output Parameters:
5881 + vertexSF         - (Optional) `PetscSF` describing complete vertex ownership
5882 - verticesAdjSaved - (Optional) vertex adjacency array
5883 
5884   Level: advanced
5885 
5886   Notes:
5887   Two triangles sharing a face
5888 .vb
5889 
5890         2
5891       / | \
5892      /  |  \
5893     /   |   \
5894    0  0 | 1  3
5895     \   |   /
5896      \  |  /
5897       \ | /
5898         1
5899 .ve
5900   would have input
5901 .vb
5902   numCells = 2, numVertices = 4
5903   cells = [0 1 2  1 3 2]
5904 .ve
5905   which would result in the `DMPLEX`
5906 .vb
5907 
5908         4
5909       / | \
5910      /  |  \
5911     /   |   \
5912    2  0 | 1  5
5913     \   |   /
5914      \  |  /
5915       \ | /
5916         3
5917 .ve
5918 
5919   Vertices are implicitly numbered consecutively 0,...,NVertices.
5920   Each rank owns a chunk of numVertices consecutive vertices.
5921   If numVertices is `PETSC_DECIDE`, PETSc will distribute them as evenly as possible using PetscLayout.
5922   If NVertices is `PETSC_DETERMINE` and numVertices is PETSC_DECIDE, NVertices is computed by PETSc as the maximum vertex index in cells + 1.
5923   If only NVertices is `PETSC_DETERMINE`, it is computed as the sum of numVertices over all ranks.
5924 
5925   The cell distribution is arbitrary non-overlapping, independent of the vertex distribution.
5926 
5927 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexBuildFromCellList()`, `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildCoordinatesFromCellListParallel()`,
5928           `PetscSF`
5929 @*/
5930 PetscErrorCode DMPlexBuildFromCellListParallel(DM dm, PetscInt numCells, PetscInt numVertices, PetscInt NVertices, PetscInt numCorners, const PetscInt cells[], PeOp PetscSF *vertexSF, PeOp PetscInt *verticesAdjSaved[])
5931 {
5932   PetscSF     sfPoint;
5933   PetscLayout layout;
5934   PetscInt    numVerticesAdj, *verticesAdj, *cones, c, p;
5935 
5936   PetscFunctionBegin;
5937   PetscValidLogicalCollectiveInt(dm, NVertices, 4);
5938   PetscCall(PetscLogEventBegin(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
5939   /* Get/check global number of vertices */
5940   {
5941     PetscInt       NVerticesInCells, i;
5942     const PetscInt len = numCells * numCorners;
5943 
5944     /* NVerticesInCells = max(cells) + 1 */
5945     NVerticesInCells = PETSC_INT_MIN;
5946     for (i = 0; i < len; i++)
5947       if (cells[i] > NVerticesInCells) NVerticesInCells = cells[i];
5948     ++NVerticesInCells;
5949     PetscCallMPI(MPIU_Allreduce(MPI_IN_PLACE, &NVerticesInCells, 1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject)dm)));
5950 
5951     if (numVertices == PETSC_DECIDE && NVertices == PETSC_DECIDE) NVertices = NVerticesInCells;
5952     else
5953       PetscCheck(NVertices == PETSC_DECIDE || NVertices >= NVerticesInCells, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Specified global number of vertices %" PetscInt_FMT " must be greater than or equal to the number of vertices in cells %" PetscInt_FMT, NVertices, NVerticesInCells);
5954   }
5955   /* Count locally unique vertices */
5956   {
5957     PetscHSetI vhash;
5958     PetscInt   off = 0;
5959 
5960     PetscCall(PetscHSetICreate(&vhash));
5961     for (c = 0; c < numCells; ++c) {
5962       for (p = 0; p < numCorners; ++p) PetscCall(PetscHSetIAdd(vhash, cells[c * numCorners + p]));
5963     }
5964     PetscCall(PetscHSetIGetSize(vhash, &numVerticesAdj));
5965     if (!verticesAdjSaved) PetscCall(PetscMalloc1(numVerticesAdj, &verticesAdj));
5966     else verticesAdj = *verticesAdjSaved;
5967     PetscCall(PetscHSetIGetElems(vhash, &off, verticesAdj));
5968     PetscCall(PetscHSetIDestroy(&vhash));
5969     PetscCheck(off == numVerticesAdj, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid number of local vertices %" PetscInt_FMT " should be %" PetscInt_FMT, off, numVerticesAdj);
5970   }
5971   PetscCall(PetscSortInt(numVerticesAdj, verticesAdj));
5972   /* Create cones */
5973   PetscCall(DMPlexSetChart(dm, 0, numCells + numVerticesAdj));
5974   for (c = 0; c < numCells; ++c) PetscCall(DMPlexSetConeSize(dm, c, numCorners));
5975   PetscCall(DMSetUp(dm));
5976   PetscCall(DMPlexGetCones(dm, &cones));
5977   for (c = 0; c < numCells; ++c) {
5978     for (p = 0; p < numCorners; ++p) {
5979       const PetscInt gv = cells[c * numCorners + p];
5980       PetscInt       lv;
5981 
5982       /* Positions within verticesAdj form 0-based local vertex numbering;
5983          we need to shift it by numCells to get correct DAG points (cells go first) */
5984       PetscCall(PetscFindInt(gv, numVerticesAdj, verticesAdj, &lv));
5985       PetscCheck(lv >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Could not find global vertex %" PetscInt_FMT " in local connectivity", gv);
5986       cones[c * numCorners + p] = lv + numCells;
5987     }
5988   }
5989   /* Build point sf */
5990   PetscCall(PetscLayoutCreate(PetscObjectComm((PetscObject)dm), &layout));
5991   PetscCall(PetscLayoutSetSize(layout, NVertices));
5992   PetscCall(PetscLayoutSetLocalSize(layout, numVertices));
5993   PetscCall(PetscLayoutSetBlockSize(layout, 1));
5994   PetscCall(PetscSFCreateByMatchingIndices(layout, numVerticesAdj, verticesAdj, NULL, numCells, numVerticesAdj, verticesAdj, NULL, numCells, vertexSF, &sfPoint));
5995   PetscCall(PetscLayoutDestroy(&layout));
5996   if (!verticesAdjSaved) PetscCall(PetscFree(verticesAdj));
5997   PetscCall(PetscObjectSetName((PetscObject)sfPoint, "point SF"));
5998   if (dm->sf) {
5999     const char *prefix;
6000 
6001     PetscCall(PetscObjectGetOptionsPrefix((PetscObject)dm->sf, &prefix));
6002     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)sfPoint, prefix));
6003   }
6004   PetscCall(DMSetPointSF(dm, sfPoint));
6005   PetscCall(PetscSFDestroy(&sfPoint));
6006   if (vertexSF) PetscCall(PetscObjectSetName((PetscObject)*vertexSF, "Vertex Ownership SF"));
6007   /* Fill in the rest of the topology structure */
6008   PetscCall(DMPlexSymmetrize(dm));
6009   PetscCall(DMPlexStratify(dm));
6010   PetscCall(PetscLogEventEnd(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
6011   PetscFunctionReturn(PETSC_SUCCESS);
6012 }
6013 
6014 /*@C
6015   DMPlexBuildFromCellSectionParallel - Build distributed `DMPLEX` topology from a list of vertices for each cell (common mesh generator output) allowing multiple celltypes
6016 
6017   Collective; No Fortran Support
6018 
6019   Input Parameters:
6020 + dm          - The `DM`
6021 . numCells    - The number of cells owned by this process
6022 . numVertices - The number of vertices to be owned by this process, or `PETSC_DECIDE`
6023 . NVertices   - The global number of vertices, or `PETSC_DETERMINE`
6024 . cellSection - The `PetscSection` giving the number of vertices for each cell (layout of cells)
6025 - cells       - An array of the global vertex numbers for each cell
6026 
6027   Output Parameters:
6028 + vertexSF         - (Optional) `PetscSF` describing complete vertex ownership
6029 - verticesAdjSaved - (Optional) vertex adjacency array
6030 
6031   Level: advanced
6032 
6033   Notes:
6034   A triangle and quadrilateral sharing a face
6035 .vb
6036         2----------3
6037       / |          |
6038      /  |          |
6039     /   |          |
6040    0  0 |     1    |
6041     \   |          |
6042      \  |          |
6043       \ |          |
6044         1----------4
6045 .ve
6046   would have input
6047 .vb
6048   numCells = 2, numVertices = 5
6049   cells = [0 1 2  1 4 3 2]
6050 .ve
6051   which would result in the `DMPLEX`
6052 .vb
6053         4----------5
6054       / |          |
6055      /  |          |
6056     /   |          |
6057    2  0 |     1    |
6058     \   |          |
6059      \  |          |
6060       \ |          |
6061         3----------6
6062 .ve
6063 
6064   Vertices are implicitly numbered consecutively 0,...,NVertices.
6065   Each rank owns a chunk of numVertices consecutive vertices.
6066   If numVertices is `PETSC_DECIDE`, PETSc will distribute them as evenly as possible using PetscLayout.
6067   If NVertices is `PETSC_DETERMINE` and numVertices is PETSC_DECIDE, NVertices is computed by PETSc as the maximum vertex index in cells + 1.
6068   If only NVertices is `PETSC_DETERMINE`, it is computed as the sum of numVertices over all ranks.
6069 
6070   The cell distribution is arbitrary non-overlapping, independent of the vertex distribution.
6071 
6072 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexBuildFromCellListParallel()`, `DMPlexCreateFromCellSectionParallel()`, `DMPlexBuildCoordinatesFromCellListParallel()`,
6073           `PetscSF`
6074 @*/
6075 PetscErrorCode DMPlexBuildFromCellSectionParallel(DM dm, PetscInt numCells, PetscInt numVertices, PetscInt NVertices, PetscSection cellSection, const PetscInt cells[], PeOp PetscSF *vertexSF, PeOp PetscInt *verticesAdjSaved[])
6076 {
6077   PetscSF     sfPoint;
6078   PetscLayout layout;
6079   PetscInt    numVerticesAdj, *verticesAdj, *cones, cStart, cEnd, len;
6080 
6081   PetscFunctionBegin;
6082   PetscValidLogicalCollectiveInt(dm, NVertices, 4);
6083   PetscCall(PetscLogEventBegin(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
6084   PetscCall(PetscSectionGetChart(cellSection, &cStart, &cEnd));
6085   PetscCall(PetscSectionGetStorageSize(cellSection, &len));
6086   PetscCheck(cStart == 0 && cEnd == numCells, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Section chart [%" PetscInt_FMT ", %" PetscInt_FMT ") should be [0, %" PetscInt_FMT ")", cStart, cEnd, numCells);
6087   /* Get/check global number of vertices */
6088   {
6089     PetscInt NVerticesInCells;
6090 
6091     /* NVerticesInCells = max(cells) + 1 */
6092     NVerticesInCells = PETSC_MIN_INT;
6093     for (PetscInt i = 0; i < len; i++)
6094       if (cells[i] > NVerticesInCells) NVerticesInCells = cells[i];
6095     ++NVerticesInCells;
6096     PetscCallMPI(MPIU_Allreduce(MPI_IN_PLACE, &NVerticesInCells, 1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject)dm)));
6097 
6098     if (numVertices == PETSC_DECIDE && NVertices == PETSC_DECIDE) NVertices = NVerticesInCells;
6099     else
6100       PetscCheck(NVertices == PETSC_DECIDE || NVertices >= NVerticesInCells, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Specified global number of vertices %" PetscInt_FMT " must be greater than or equal to the number of vertices in cells %" PetscInt_FMT, NVertices, NVerticesInCells);
6101   }
6102   /* Count locally unique vertices */
6103   {
6104     PetscHSetI vhash;
6105     PetscInt   off = 0;
6106 
6107     PetscCall(PetscHSetICreate(&vhash));
6108     for (PetscInt i = 0; i < len; i++) PetscCall(PetscHSetIAdd(vhash, cells[i]));
6109     PetscCall(PetscHSetIGetSize(vhash, &numVerticesAdj));
6110     if (!verticesAdjSaved) PetscCall(PetscMalloc1(numVerticesAdj, &verticesAdj));
6111     else verticesAdj = *verticesAdjSaved;
6112     PetscCall(PetscHSetIGetElems(vhash, &off, verticesAdj));
6113     PetscCall(PetscHSetIDestroy(&vhash));
6114     PetscCheck(off == numVerticesAdj, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid number of local vertices %" PetscInt_FMT " should be %" PetscInt_FMT, off, numVerticesAdj);
6115   }
6116   PetscCall(PetscSortInt(numVerticesAdj, verticesAdj));
6117   /* Create cones */
6118   PetscCall(DMPlexSetChart(dm, 0, numCells + numVerticesAdj));
6119   for (PetscInt c = 0; c < numCells; ++c) {
6120     PetscInt dof;
6121 
6122     PetscCall(PetscSectionGetDof(cellSection, c, &dof));
6123     PetscCall(DMPlexSetConeSize(dm, c, dof));
6124   }
6125   PetscCall(DMSetUp(dm));
6126   PetscCall(DMPlexGetCones(dm, &cones));
6127   for (PetscInt c = 0; c < numCells; ++c) {
6128     PetscInt dof, off;
6129 
6130     PetscCall(PetscSectionGetDof(cellSection, c, &dof));
6131     PetscCall(PetscSectionGetOffset(cellSection, c, &off));
6132     for (PetscInt p = off; p < off + dof; ++p) {
6133       const PetscInt gv = cells[p];
6134       PetscInt       lv;
6135 
6136       /* Positions within verticesAdj form 0-based local vertex numbering;
6137          we need to shift it by numCells to get correct DAG points (cells go first) */
6138       PetscCall(PetscFindInt(gv, numVerticesAdj, verticesAdj, &lv));
6139       PetscCheck(lv >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Could not find global vertex %" PetscInt_FMT " in local connectivity", gv);
6140       cones[p] = lv + numCells;
6141     }
6142   }
6143   /* Build point sf */
6144   PetscCall(PetscLayoutCreate(PetscObjectComm((PetscObject)dm), &layout));
6145   PetscCall(PetscLayoutSetSize(layout, NVertices));
6146   PetscCall(PetscLayoutSetLocalSize(layout, numVertices));
6147   PetscCall(PetscLayoutSetBlockSize(layout, 1));
6148   PetscCall(PetscSFCreateByMatchingIndices(layout, numVerticesAdj, verticesAdj, NULL, numCells, numVerticesAdj, verticesAdj, NULL, numCells, vertexSF, &sfPoint));
6149   PetscCall(PetscLayoutDestroy(&layout));
6150   if (!verticesAdjSaved) PetscCall(PetscFree(verticesAdj));
6151   PetscCall(PetscObjectSetName((PetscObject)sfPoint, "point SF"));
6152   if (dm->sf) {
6153     const char *prefix;
6154 
6155     PetscCall(PetscObjectGetOptionsPrefix((PetscObject)dm->sf, &prefix));
6156     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)sfPoint, prefix));
6157   }
6158   PetscCall(DMSetPointSF(dm, sfPoint));
6159   PetscCall(PetscSFDestroy(&sfPoint));
6160   if (vertexSF) PetscCall(PetscObjectSetName((PetscObject)*vertexSF, "Vertex Ownership SF"));
6161   /* Fill in the rest of the topology structure */
6162   PetscCall(DMPlexSymmetrize(dm));
6163   PetscCall(DMPlexStratify(dm));
6164   PetscCall(PetscLogEventEnd(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
6165   PetscFunctionReturn(PETSC_SUCCESS);
6166 }
6167 
6168 /*@
6169   DMPlexBuildCoordinatesFromCellListParallel - Build `DM` coordinates from a list of coordinates for each owned vertex (common mesh generator output)
6170 
6171   Collective; No Fortran Support
6172 
6173   Input Parameters:
6174 + dm           - The `DM`
6175 . spaceDim     - The spatial dimension used for coordinates
6176 . sfVert       - `PetscSF` describing complete vertex ownership
6177 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
6178 
6179   Level: advanced
6180 
6181 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildFromCellListParallel()`
6182 @*/
6183 PetscErrorCode DMPlexBuildCoordinatesFromCellListParallel(DM dm, PetscInt spaceDim, PetscSF sfVert, const PetscReal vertexCoords[])
6184 {
6185   PetscSection coordSection;
6186   Vec          coordinates;
6187   PetscScalar *coords;
6188   PetscInt     numVertices, numVerticesAdj, coordSize, v, vStart, vEnd;
6189   PetscMPIInt  spaceDimi;
6190 
6191   PetscFunctionBegin;
6192   PetscCall(PetscLogEventBegin(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
6193   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
6194   PetscCheck(vStart >= 0 && vEnd >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM is not set up properly. DMPlexBuildFromCellList() should be called first.");
6195   PetscCall(DMSetCoordinateDim(dm, spaceDim));
6196   PetscCall(PetscSFGetGraph(sfVert, &numVertices, &numVerticesAdj, NULL, NULL));
6197   PetscCheck(vEnd - vStart == numVerticesAdj, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Supplied sfVert has wrong number of leaves = %" PetscInt_FMT " != %" PetscInt_FMT " = vEnd - vStart", numVerticesAdj, vEnd - vStart);
6198   PetscCall(DMGetCoordinateSection(dm, &coordSection));
6199   PetscCall(PetscSectionSetNumFields(coordSection, 1));
6200   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, spaceDim));
6201   PetscCall(PetscSectionSetChart(coordSection, vStart, vEnd));
6202   for (v = vStart; v < vEnd; ++v) {
6203     PetscCall(PetscSectionSetDof(coordSection, v, spaceDim));
6204     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, spaceDim));
6205   }
6206   PetscCall(PetscSectionSetUp(coordSection));
6207   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
6208   PetscCall(VecCreate(PetscObjectComm((PetscObject)dm), &coordinates));
6209   PetscCall(VecSetBlockSize(coordinates, spaceDim));
6210   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
6211   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
6212   PetscCall(VecSetType(coordinates, VECSTANDARD));
6213   PetscCall(VecGetArray(coordinates, &coords));
6214   {
6215     MPI_Datatype coordtype;
6216 
6217     /* Need a temp buffer for coords if we have complex/single */
6218     PetscCall(PetscMPIIntCast(spaceDim, &spaceDimi));
6219     PetscCallMPI(MPI_Type_contiguous(spaceDimi, MPIU_SCALAR, &coordtype));
6220     PetscCallMPI(MPI_Type_commit(&coordtype));
6221 #if defined(PETSC_USE_COMPLEX)
6222     {
6223       PetscScalar *svertexCoords;
6224       PetscInt     i;
6225       PetscCall(PetscMalloc1(numVertices * spaceDim, &svertexCoords));
6226       for (i = 0; i < numVertices * spaceDim; i++) svertexCoords[i] = vertexCoords[i];
6227       PetscCall(PetscSFBcastBegin(sfVert, coordtype, svertexCoords, coords, MPI_REPLACE));
6228       PetscCall(PetscSFBcastEnd(sfVert, coordtype, svertexCoords, coords, MPI_REPLACE));
6229       PetscCall(PetscFree(svertexCoords));
6230     }
6231 #else
6232     PetscCall(PetscSFBcastBegin(sfVert, coordtype, vertexCoords, coords, MPI_REPLACE));
6233     PetscCall(PetscSFBcastEnd(sfVert, coordtype, vertexCoords, coords, MPI_REPLACE));
6234 #endif
6235     PetscCallMPI(MPI_Type_free(&coordtype));
6236   }
6237   PetscCall(VecRestoreArray(coordinates, &coords));
6238   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
6239   PetscCall(VecDestroy(&coordinates));
6240   PetscCall(PetscLogEventEnd(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
6241   PetscFunctionReturn(PETSC_SUCCESS);
6242 }
6243 
6244 /*@
6245   DMPlexCreateFromCellListParallelPetsc - Create distributed `DMPLEX` from a list of vertices for each cell (common mesh generator output) where all cells have the same celltype
6246 
6247   Collective
6248 
6249   Input Parameters:
6250 + comm         - The communicator
6251 . dim          - The topological dimension of the mesh
6252 . numCells     - The number of cells owned by this process
6253 . numVertices  - The number of vertices owned by this process, or `PETSC_DECIDE`
6254 . NVertices    - The global number of vertices, or `PETSC_DECIDE`
6255 . numCorners   - The number of vertices for each cell
6256 . interpolate  - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
6257 . cells        - An array of numCells*numCorners numbers, the global vertex numbers for each cell
6258 . spaceDim     - The spatial dimension used for coordinates
6259 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
6260 
6261   Output Parameters:
6262 + dm          - The `DM`
6263 . vertexSF    - (Optional) `PetscSF` describing complete vertex ownership
6264 - verticesAdj - (Optional) vertex adjacency array
6265 
6266   Level: intermediate
6267 
6268   Notes:
6269   This function is just a convenient sequence of `DMCreate()`, `DMSetType()`, `DMSetDimension()`,
6270   `DMPlexBuildFromCellListParallel()`, `DMPlexInterpolate()`, `DMPlexBuildCoordinatesFromCellListParallel()`
6271 
6272   See `DMPlexBuildFromCellListParallel()` for an example and details about the topology-related parameters.
6273 
6274   See `DMPlexBuildCoordinatesFromCellListParallel()` for details about the geometry-related parameters.
6275 
6276 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromCellListPetsc()`, `DMPlexBuildFromCellListParallel()`, `DMPlexBuildCoordinatesFromCellListParallel()`, `DMPlexCreateFromDAG()`, `DMPlexCreate()`
6277 @*/
6278 PetscErrorCode DMPlexCreateFromCellListParallelPetsc(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt NVertices, PetscInt numCorners, PetscBool interpolate, const PetscInt cells[], PetscInt spaceDim, const PetscReal vertexCoords[], PeOp PetscSF *vertexSF, PeOp PetscInt *verticesAdj[], DM *dm)
6279 {
6280   PetscSF sfVert;
6281 
6282   PetscFunctionBegin;
6283   PetscCall(DMCreate(comm, dm));
6284   PetscCall(DMSetType(*dm, DMPLEX));
6285   PetscValidLogicalCollectiveInt(*dm, dim, 2);
6286   PetscValidLogicalCollectiveInt(*dm, spaceDim, 9);
6287   PetscCall(DMSetDimension(*dm, dim));
6288   PetscCall(DMPlexBuildFromCellListParallel(*dm, numCells, numVertices, NVertices, numCorners, cells, &sfVert, verticesAdj));
6289   if (interpolate) {
6290     DM idm;
6291 
6292     PetscCall(DMPlexInterpolate(*dm, &idm));
6293     PetscCall(DMDestroy(dm));
6294     *dm = idm;
6295   }
6296   PetscCall(DMPlexBuildCoordinatesFromCellListParallel(*dm, spaceDim, sfVert, vertexCoords));
6297   if (vertexSF) *vertexSF = sfVert;
6298   else PetscCall(PetscSFDestroy(&sfVert));
6299   PetscFunctionReturn(PETSC_SUCCESS);
6300 }
6301 
6302 /*@
6303   DMPlexCreateFromCellSectionParallel - Create distributed `DMPLEX` from a list of vertices for each cell (common mesh generator output) and supports multiple celltypes
6304 
6305   Collective
6306 
6307   Input Parameters:
6308 + comm         - The communicator
6309 . dim          - The topological dimension of the mesh
6310 . numCells     - The number of cells owned by this process
6311 . numVertices  - The number of vertices owned by this process, or `PETSC_DECIDE`
6312 . NVertices    - The global number of vertices, or `PETSC_DECIDE`
6313 . cellSection  - The `PetscSection` giving the number of vertices for each cell (layout of cells)
6314 . interpolate  - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
6315 . cells        - An array of the global vertex numbers for each cell
6316 . spaceDim     - The spatial dimension used for coordinates
6317 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
6318 
6319   Output Parameters:
6320 + dm          - The `DM`
6321 . vertexSF    - (Optional) `PetscSF` describing complete vertex ownership
6322 - verticesAdj - (Optional) vertex adjacency array
6323 
6324   Level: intermediate
6325 
6326   Notes:
6327   This function is just a convenient sequence of `DMCreate()`, `DMSetType()`, `DMSetDimension()`,
6328   `DMPlexBuildFromCellSectionParallel()`, `DMPlexInterpolate()`, `DMPlexBuildCoordinatesFromCellListParallel()`
6329 
6330   See `DMPlexBuildFromCellSectionParallel()` for an example and details about the topology-related parameters.
6331 
6332   See `DMPlexBuildCoordinatesFromCellListParallel()` for details about the geometry-related parameters.
6333 
6334 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromCellListPetsc()`, `DMPlexBuildFromCellListParallel()`, `DMPlexBuildCoordinatesFromCellListParallel()`, `DMPlexCreateFromDAG()`, `DMPlexCreate()`
6335 @*/
6336 PetscErrorCode DMPlexCreateFromCellSectionParallel(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt NVertices, PetscSection cellSection, PetscBool interpolate, const PetscInt cells[], PetscInt spaceDim, const PetscReal vertexCoords[], PeOp PetscSF *vertexSF, PeOp PetscInt *verticesAdj[], DM *dm)
6337 {
6338   PetscSF sfVert;
6339 
6340   PetscFunctionBegin;
6341   PetscCall(DMCreate(comm, dm));
6342   PetscCall(DMSetType(*dm, DMPLEX));
6343   PetscValidLogicalCollectiveInt(*dm, dim, 2);
6344   PetscValidLogicalCollectiveInt(*dm, spaceDim, 9);
6345   PetscCall(DMSetDimension(*dm, dim));
6346   PetscCall(DMPlexBuildFromCellSectionParallel(*dm, numCells, numVertices, NVertices, cellSection, cells, &sfVert, verticesAdj));
6347   if (interpolate) {
6348     DM idm;
6349 
6350     PetscCall(DMPlexInterpolate(*dm, &idm));
6351     PetscCall(DMDestroy(dm));
6352     *dm = idm;
6353   }
6354   PetscCall(DMPlexBuildCoordinatesFromCellListParallel(*dm, spaceDim, sfVert, vertexCoords));
6355   if (vertexSF) *vertexSF = sfVert;
6356   else PetscCall(PetscSFDestroy(&sfVert));
6357   PetscFunctionReturn(PETSC_SUCCESS);
6358 }
6359 
6360 /*@
6361   DMPlexBuildFromCellList - Build `DMPLEX` topology from a list of vertices for each cell (common mesh generator output)
6362 
6363   Collective; No Fortran Support
6364 
6365   Input Parameters:
6366 + dm          - The `DM`
6367 . numCells    - The number of cells owned by this process
6368 . numVertices - The number of vertices owned by this process, or `PETSC_DETERMINE`
6369 . numCorners  - The number of vertices for each cell
6370 - cells       - An array of `numCells` x `numCorners` numbers, the global vertex numbers for each cell
6371 
6372   Level: advanced
6373 
6374   Notes:
6375   Two triangles sharing a face
6376 .vb
6377 
6378         2
6379       / | \
6380      /  |  \
6381     /   |   \
6382    0  0 | 1  3
6383     \   |   /
6384      \  |  /
6385       \ | /
6386         1
6387 .ve
6388   would have input
6389 .vb
6390   numCells = 2, numVertices = 4
6391   cells = [0 1 2  1 3 2]
6392 .ve
6393   which would result in the `DMPLEX`
6394 .vb
6395 
6396         4
6397       / | \
6398      /  |  \
6399     /   |   \
6400    2  0 | 1  5
6401     \   |   /
6402      \  |  /
6403       \ | /
6404         3
6405 .ve
6406 
6407   If numVertices is `PETSC_DETERMINE`, it is computed by PETSc as the maximum vertex index in cells + 1.
6408 
6409 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexBuildFromCellListParallel()`, `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromCellListPetsc()`
6410 @*/
6411 PetscErrorCode DMPlexBuildFromCellList(DM dm, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, const PetscInt cells[])
6412 {
6413   PetscInt *cones, c, p, dim;
6414 
6415   PetscFunctionBegin;
6416   PetscCall(PetscLogEventBegin(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
6417   PetscCall(DMGetDimension(dm, &dim));
6418   /* Get/check global number of vertices */
6419   {
6420     PetscInt       NVerticesInCells, i;
6421     const PetscInt len = numCells * numCorners;
6422 
6423     /* NVerticesInCells = max(cells) + 1 */
6424     NVerticesInCells = PETSC_INT_MIN;
6425     for (i = 0; i < len; i++)
6426       if (cells[i] > NVerticesInCells) NVerticesInCells = cells[i];
6427     ++NVerticesInCells;
6428 
6429     if (numVertices == PETSC_DECIDE) numVertices = NVerticesInCells;
6430     else
6431       PetscCheck(numVertices >= NVerticesInCells, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Specified number of vertices %" PetscInt_FMT " must be greater than or equal to the number of vertices in cells %" PetscInt_FMT, numVertices, NVerticesInCells);
6432   }
6433   PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
6434   for (c = 0; c < numCells; ++c) PetscCall(DMPlexSetConeSize(dm, c, numCorners));
6435   PetscCall(DMSetUp(dm));
6436   PetscCall(DMPlexGetCones(dm, &cones));
6437   for (c = 0; c < numCells; ++c) {
6438     for (p = 0; p < numCorners; ++p) cones[c * numCorners + p] = cells[c * numCorners + p] + numCells;
6439   }
6440   PetscCall(DMPlexSymmetrize(dm));
6441   PetscCall(DMPlexStratify(dm));
6442   PetscCall(PetscLogEventEnd(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
6443   PetscFunctionReturn(PETSC_SUCCESS);
6444 }
6445 
6446 /*@
6447   DMPlexBuildCoordinatesFromCellList - Build `DM` coordinates from a list of coordinates for each owned vertex (common mesh generator output)
6448 
6449   Collective
6450 
6451   Input Parameters:
6452 + dm           - The `DM`
6453 . spaceDim     - The spatial dimension used for coordinates
6454 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
6455 
6456   Level: advanced
6457 
6458 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexBuildCoordinatesFromCellListParallel()`, `DMPlexCreateFromCellListPetsc()`, `DMPlexBuildFromCellList()`
6459 @*/
6460 PetscErrorCode DMPlexBuildCoordinatesFromCellList(DM dm, PetscInt spaceDim, const PetscReal vertexCoords[])
6461 {
6462   PetscSection coordSection;
6463   Vec          coordinates;
6464   DM           cdm;
6465   PetscScalar *coords;
6466   PetscInt     v, vStart, vEnd, d;
6467 
6468   PetscFunctionBegin;
6469   PetscCall(PetscLogEventBegin(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
6470   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
6471   PetscCheck(vStart >= 0 && vEnd >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM is not set up properly. DMPlexBuildFromCellList() should be called first.");
6472   PetscCall(DMSetCoordinateDim(dm, spaceDim));
6473   PetscCall(DMGetCoordinateSection(dm, &coordSection));
6474   PetscCall(PetscSectionSetNumFields(coordSection, 1));
6475   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, spaceDim));
6476   PetscCall(PetscSectionSetChart(coordSection, vStart, vEnd));
6477   for (v = vStart; v < vEnd; ++v) {
6478     PetscCall(PetscSectionSetDof(coordSection, v, spaceDim));
6479     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, spaceDim));
6480   }
6481   PetscCall(PetscSectionSetUp(coordSection));
6482 
6483   PetscCall(DMGetCoordinateDM(dm, &cdm));
6484   PetscCall(DMCreateLocalVector(cdm, &coordinates));
6485   PetscCall(VecSetBlockSize(coordinates, spaceDim));
6486   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
6487   PetscCall(VecGetArrayWrite(coordinates, &coords));
6488   for (v = 0; v < vEnd - vStart; ++v) {
6489     for (d = 0; d < spaceDim; ++d) coords[v * spaceDim + d] = vertexCoords[v * spaceDim + d];
6490   }
6491   PetscCall(VecRestoreArrayWrite(coordinates, &coords));
6492   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
6493   PetscCall(VecDestroy(&coordinates));
6494   PetscCall(PetscLogEventEnd(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
6495   PetscFunctionReturn(PETSC_SUCCESS);
6496 }
6497 
6498 /*@
6499   DMPlexCreateFromCellListPetsc - Create `DMPLEX` from a list of vertices for each cell (common mesh generator output), but only process 0 takes in the input
6500 
6501   Collective
6502 
6503   Input Parameters:
6504 + comm         - The communicator
6505 . dim          - The topological dimension of the mesh
6506 . numCells     - The number of cells, only on process 0
6507 . numVertices  - The number of vertices owned by this process, or `PETSC_DECIDE`, only on process 0
6508 . numCorners   - The number of vertices for each cell, only on process 0
6509 . interpolate  - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
6510 . cells        - An array of $ numCells \times numCorners$ numbers, the vertices for each cell, only on process 0
6511 . spaceDim     - The spatial dimension used for coordinates
6512 - vertexCoords - An array of $ numVertices \times spaceDim$ numbers, the coordinates of each vertex, only on process 0
6513 
6514   Output Parameter:
6515 . dm - The `DM`, which only has points on process 0
6516 
6517   Level: intermediate
6518 
6519   Notes:
6520   This function is just a convenient sequence of `DMCreate()`, `DMSetType()`, `DMSetDimension()`, `DMPlexBuildFromCellList()`,
6521   `DMPlexInterpolate()`, `DMPlexBuildCoordinatesFromCellList()`
6522 
6523   See `DMPlexBuildFromCellList()` for an example and details about the topology-related parameters.
6524   See `DMPlexBuildCoordinatesFromCellList()` for details about the geometry-related parameters.
6525   See `DMPlexCreateFromCellListParallelPetsc()` for parallel input
6526 
6527 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildFromCellList()`, `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromDAG()`, `DMPlexCreate()`
6528 @*/
6529 PetscErrorCode DMPlexCreateFromCellListPetsc(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, PetscBool interpolate, const PetscInt cells[], PetscInt spaceDim, const PetscReal vertexCoords[], DM *dm)
6530 {
6531   PetscMPIInt rank;
6532 
6533   PetscFunctionBegin;
6534   PetscCheck(dim, comm, PETSC_ERR_ARG_OUTOFRANGE, "This is not appropriate for 0-dimensional meshes. Consider either creating the DM using DMPlexCreateFromDAG(), by hand, or using DMSwarm.");
6535   PetscCallMPI(MPI_Comm_rank(comm, &rank));
6536   PetscCall(DMCreate(comm, dm));
6537   PetscCall(DMSetType(*dm, DMPLEX));
6538   PetscCall(DMSetDimension(*dm, dim));
6539   if (rank == 0) PetscCall(DMPlexBuildFromCellList(*dm, numCells, numVertices, numCorners, cells));
6540   else PetscCall(DMPlexBuildFromCellList(*dm, 0, 0, 0, NULL));
6541   if (interpolate) {
6542     DM idm;
6543 
6544     PetscCall(DMPlexInterpolate(*dm, &idm));
6545     PetscCall(DMDestroy(dm));
6546     *dm = idm;
6547   }
6548   if (rank == 0) PetscCall(DMPlexBuildCoordinatesFromCellList(*dm, spaceDim, vertexCoords));
6549   else PetscCall(DMPlexBuildCoordinatesFromCellList(*dm, spaceDim, NULL));
6550   PetscFunctionReturn(PETSC_SUCCESS);
6551 }
6552 
6553 /*@
6554   DMPlexCreateFromDAG - This takes as input the adjacency-list representation of the Directed Acyclic Graph (Hasse Diagram) encoding a mesh, and produces a `DM`
6555 
6556   Input Parameters:
6557 + dm               - The empty `DM` object, usually from `DMCreate()` and `DMSetDimension()`
6558 . depth            - The depth of the DAG
6559 . numPoints        - Array of size $ depth + 1 $ containing the number of points at each `depth`
6560 . coneSize         - The cone size of each point
6561 . cones            - The concatenation of the cone points for each point, the cone list must be oriented correctly for each point
6562 . coneOrientations - The orientation of each cone point
6563 - vertexCoords     - An array of $ numPoints[0] \times spacedim $ numbers representing the coordinates of each vertex, with `spacedim` the value set via `DMSetCoordinateDim()`
6564 
6565   Output Parameter:
6566 . dm - The `DM`
6567 
6568   Level: advanced
6569 
6570   Note:
6571   Two triangles sharing a face would have input
6572 .vb
6573   depth = 1, numPoints = [4 2], coneSize = [3 3 0 0 0 0]
6574   cones = [2 3 4  3 5 4], coneOrientations = [0 0 0  0 0 0]
6575  vertexCoords = [-1.0 0.0  0.0 -1.0  0.0 1.0  1.0 0.0]
6576 .ve
6577   which would result in the DMPlex
6578 .vb
6579         4
6580       / | \
6581      /  |  \
6582     /   |   \
6583    2  0 | 1  5
6584     \   |   /
6585      \  |  /
6586       \ | /
6587         3
6588 .ve
6589   Notice that all points are numbered consecutively, unlike `DMPlexCreateFromCellListPetsc()`
6590 
6591   Developer Note:
6592   This does not create anything so should not have create in the name.
6593 
6594 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromCellListPetsc()`, `DMPlexCreate()`
6595 @*/
6596 PetscErrorCode DMPlexCreateFromDAG(DM dm, PetscInt depth, const PetscInt numPoints[], const PetscInt coneSize[], const PetscInt cones[], const PetscInt coneOrientations[], const PetscScalar vertexCoords[])
6597 {
6598   Vec          coordinates;
6599   PetscSection coordSection;
6600   PetscScalar *coords;
6601   PetscInt     coordSize, firstVertex = -1, pStart = 0, pEnd = 0, p, v, dim, dimEmbed, d, off;
6602 
6603   PetscFunctionBegin;
6604   PetscCall(DMGetDimension(dm, &dim));
6605   PetscCall(DMGetCoordinateDim(dm, &dimEmbed));
6606   PetscCheck(dimEmbed >= dim, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Embedding dimension %" PetscInt_FMT " cannot be less than intrinsic dimension %" PetscInt_FMT, dimEmbed, dim);
6607   for (d = 0; d <= depth; ++d) pEnd += numPoints[d];
6608   PetscCall(DMPlexSetChart(dm, pStart, pEnd));
6609   for (p = pStart; p < pEnd; ++p) {
6610     PetscCall(DMPlexSetConeSize(dm, p, coneSize[p - pStart]));
6611     if (firstVertex < 0 && !coneSize[p - pStart]) firstVertex = p - pStart;
6612   }
6613   PetscCheck(firstVertex >= 0 || !numPoints[0], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Expected %" PetscInt_FMT " vertices but could not find any", numPoints[0]);
6614   PetscCall(DMSetUp(dm)); /* Allocate space for cones */
6615   for (p = pStart, off = 0; p < pEnd; off += coneSize[p - pStart], ++p) {
6616     PetscCall(DMPlexSetCone(dm, p, &cones[off]));
6617     PetscCall(DMPlexSetConeOrientation(dm, p, &coneOrientations[off]));
6618   }
6619   PetscCall(DMPlexSymmetrize(dm));
6620   PetscCall(DMPlexStratify(dm));
6621   /* Build coordinates */
6622   PetscCall(DMGetCoordinateSection(dm, &coordSection));
6623   PetscCall(PetscSectionSetNumFields(coordSection, 1));
6624   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dimEmbed));
6625   PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numPoints[0]));
6626   for (v = firstVertex; v < firstVertex + numPoints[0]; ++v) {
6627     PetscCall(PetscSectionSetDof(coordSection, v, dimEmbed));
6628     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dimEmbed));
6629   }
6630   PetscCall(PetscSectionSetUp(coordSection));
6631   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
6632   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
6633   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
6634   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
6635   PetscCall(VecSetBlockSize(coordinates, PetscMax(dimEmbed, 1)));
6636   PetscCall(VecSetType(coordinates, VECSTANDARD));
6637   if (vertexCoords) {
6638     PetscCall(VecGetArray(coordinates, &coords));
6639     for (v = 0; v < numPoints[0]; ++v) {
6640       PetscInt off;
6641 
6642       PetscCall(PetscSectionGetOffset(coordSection, v + firstVertex, &off));
6643       for (d = 0; d < dimEmbed; ++d) coords[off + d] = vertexCoords[v * dimEmbed + d];
6644     }
6645   }
6646   PetscCall(VecRestoreArray(coordinates, &coords));
6647   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
6648   PetscCall(VecDestroy(&coordinates));
6649   PetscFunctionReturn(PETSC_SUCCESS);
6650 }
6651 
6652 /*
6653   DMPlexCreateCellVertexFromFile - Create a `DMPLEX` mesh from a simple cell-vertex file.
6654 
6655   Collective
6656 
6657 + comm        - The MPI communicator
6658 . filename    - Name of the .dat file
6659 - interpolate - Create faces and edges in the mesh
6660 
6661   Output Parameter:
6662 . dm  - The `DM` object representing the mesh
6663 
6664   Level: beginner
6665 
6666   Note:
6667   The format is the simplest possible:
6668 .vb
6669   dim Ne Nv Nc Nl
6670   v_1 v_2 ... v_Nc
6671   ...
6672   x y z marker_1 ... marker_Nl
6673 .ve
6674 
6675   Developer Note:
6676   Should use a `PetscViewer` not a filename
6677 
6678 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromFile()`, `DMPlexCreateGmsh()`, `DMPlexCreate()`
6679 */
6680 static PetscErrorCode DMPlexCreateCellVertexFromFile(MPI_Comm comm, const char filename[], PetscBool interpolate, DM *dm)
6681 {
6682   DMLabel      marker;
6683   PetscViewer  viewer;
6684   Vec          coordinates;
6685   PetscSection coordSection;
6686   PetscScalar *coords;
6687   char         line[PETSC_MAX_PATH_LEN];
6688   PetscInt     cdim, coordSize, v, c, d;
6689   PetscMPIInt  rank;
6690   int          snum, dim, Nv, Nc, Ncn, Nl;
6691 
6692   PetscFunctionBegin;
6693   PetscCallMPI(MPI_Comm_rank(comm, &rank));
6694   PetscCall(PetscViewerCreate(comm, &viewer));
6695   PetscCall(PetscViewerSetType(viewer, PETSCVIEWERASCII));
6696   PetscCall(PetscViewerFileSetMode(viewer, FILE_MODE_READ));
6697   PetscCall(PetscViewerFileSetName(viewer, filename));
6698   if (rank == 0) {
6699     PetscCall(PetscViewerRead(viewer, line, 5, NULL, PETSC_STRING));
6700     snum = sscanf(line, "%d %d %d %d %d", &dim, &Nc, &Nv, &Ncn, &Nl);
6701     PetscCheck(snum == 5, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
6702   } else {
6703     Nc = Nv = Ncn = Nl = 0;
6704   }
6705   PetscCallMPI(MPI_Bcast(&dim, 1, MPI_INT, 0, comm));
6706   cdim = dim;
6707   PetscCall(DMCreate(comm, dm));
6708   PetscCall(DMSetType(*dm, DMPLEX));
6709   PetscCall(DMPlexSetChart(*dm, 0, Nc + Nv));
6710   PetscCall(DMSetDimension(*dm, dim));
6711   PetscCall(DMSetCoordinateDim(*dm, cdim));
6712   /* Read topology */
6713   if (rank == 0) {
6714     char     format[PETSC_MAX_PATH_LEN];
6715     PetscInt cone[8];
6716     int      vbuf[8], v;
6717 
6718     for (c = 0; c < Ncn; ++c) {
6719       format[c * 3 + 0] = '%';
6720       format[c * 3 + 1] = 'd';
6721       format[c * 3 + 2] = ' ';
6722     }
6723     format[Ncn * 3 - 1] = '\0';
6724     for (c = 0; c < Nc; ++c) PetscCall(DMPlexSetConeSize(*dm, c, Ncn));
6725     PetscCall(DMSetUp(*dm));
6726     for (c = 0; c < Nc; ++c) {
6727       PetscCall(PetscViewerRead(viewer, line, Ncn, NULL, PETSC_STRING));
6728       switch (Ncn) {
6729       case 2:
6730         snum = sscanf(line, format, &vbuf[0], &vbuf[1]);
6731         break;
6732       case 3:
6733         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2]);
6734         break;
6735       case 4:
6736         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3]);
6737         break;
6738       case 6:
6739         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3], &vbuf[4], &vbuf[5]);
6740         break;
6741       case 8:
6742         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3], &vbuf[4], &vbuf[5], &vbuf[6], &vbuf[7]);
6743         break;
6744       default:
6745         SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "No cell shape with %d vertices", Ncn);
6746       }
6747       PetscCheck(snum == Ncn, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
6748       for (v = 0; v < Ncn; ++v) cone[v] = vbuf[v] + Nc;
6749       /* Hexahedra are inverted */
6750       if (Ncn == 8) {
6751         PetscInt tmp = cone[1];
6752         cone[1]      = cone[3];
6753         cone[3]      = tmp;
6754       }
6755       PetscCall(DMPlexSetCone(*dm, c, cone));
6756     }
6757   }
6758   PetscCall(DMPlexSymmetrize(*dm));
6759   PetscCall(DMPlexStratify(*dm));
6760   /* Read coordinates */
6761   PetscCall(DMGetCoordinateSection(*dm, &coordSection));
6762   PetscCall(PetscSectionSetNumFields(coordSection, 1));
6763   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, cdim));
6764   PetscCall(PetscSectionSetChart(coordSection, Nc, Nc + Nv));
6765   for (v = Nc; v < Nc + Nv; ++v) {
6766     PetscCall(PetscSectionSetDof(coordSection, v, cdim));
6767     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, cdim));
6768   }
6769   PetscCall(PetscSectionSetUp(coordSection));
6770   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
6771   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
6772   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
6773   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
6774   PetscCall(VecSetBlockSize(coordinates, cdim));
6775   PetscCall(VecSetType(coordinates, VECSTANDARD));
6776   PetscCall(VecGetArray(coordinates, &coords));
6777   if (rank == 0) {
6778     char   format[PETSC_MAX_PATH_LEN];
6779     double x[3];
6780     int    l, val[3];
6781 
6782     if (Nl) {
6783       for (l = 0; l < Nl; ++l) {
6784         format[l * 3 + 0] = '%';
6785         format[l * 3 + 1] = 'd';
6786         format[l * 3 + 2] = ' ';
6787       }
6788       format[Nl * 3 - 1] = '\0';
6789       PetscCall(DMCreateLabel(*dm, "marker"));
6790       PetscCall(DMGetLabel(*dm, "marker", &marker));
6791     }
6792     for (v = 0; v < Nv; ++v) {
6793       PetscCall(PetscViewerRead(viewer, line, 3 + Nl, NULL, PETSC_STRING));
6794       snum = sscanf(line, "%lg %lg %lg", &x[0], &x[1], &x[2]);
6795       PetscCheck(snum == 3, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
6796       switch (Nl) {
6797       case 0:
6798         snum = 0;
6799         break;
6800       case 1:
6801         snum = sscanf(line, format, &val[0]);
6802         break;
6803       case 2:
6804         snum = sscanf(line, format, &val[0], &val[1]);
6805         break;
6806       case 3:
6807         snum = sscanf(line, format, &val[0], &val[1], &val[2]);
6808         break;
6809       default:
6810         SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Request support for %d labels", Nl);
6811       }
6812       PetscCheck(snum == Nl, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
6813       for (d = 0; d < cdim; ++d) coords[v * cdim + d] = x[d];
6814       for (l = 0; l < Nl; ++l) PetscCall(DMLabelSetValue(marker, v + Nc, val[l]));
6815     }
6816   }
6817   PetscCall(VecRestoreArray(coordinates, &coords));
6818   PetscCall(DMSetCoordinatesLocal(*dm, coordinates));
6819   PetscCall(VecDestroy(&coordinates));
6820   PetscCall(PetscViewerDestroy(&viewer));
6821   if (interpolate) {
6822     DM      idm;
6823     DMLabel bdlabel;
6824 
6825     PetscCall(DMPlexInterpolate(*dm, &idm));
6826     PetscCall(DMDestroy(dm));
6827     *dm = idm;
6828 
6829     if (!Nl) {
6830       PetscCall(DMCreateLabel(*dm, "marker"));
6831       PetscCall(DMGetLabel(*dm, "marker", &bdlabel));
6832       PetscCall(DMPlexMarkBoundaryFaces(*dm, PETSC_DETERMINE, bdlabel));
6833       PetscCall(DMPlexLabelComplete(*dm, bdlabel));
6834     }
6835   }
6836   PetscFunctionReturn(PETSC_SUCCESS);
6837 }
6838 
6839 /*@
6840   DMPlexCreateFromFile - This takes a filename and produces a `DM`
6841 
6842   Collective
6843 
6844   Input Parameters:
6845 + comm        - The communicator
6846 . filename    - A file name
6847 . plexname    - The object name of the resulting `DM`, also used for intra-datafile lookup by some formats
6848 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
6849 
6850   Output Parameter:
6851 . dm - The `DM`
6852 
6853   Options Database Key:
6854 . -dm_plex_create_from_hdf5_xdmf - use the `PETSC_VIEWER_HDF5_XDMF` format for reading HDF5
6855 
6856   Use `-dm_plex_create_ prefix` to pass options to the internal `PetscViewer`, e.g. `-dm_plex_create_viewer_hdf5_collective`
6857 
6858   Level: beginner
6859 
6860   Notes:
6861   Using `PETSCVIEWERHDF5` type with `PETSC_VIEWER_HDF5_PETSC` format, one can save multiple `DMPLEX`
6862   meshes in a single HDF5 file. This in turn requires one to name the `DMPLEX` object with `PetscObjectSetName()`
6863   before saving it with `DMView()` and before loading it with `DMLoad()` for identification of the mesh object.
6864   The input parameter name is thus used to name the `DMPLEX` object when `DMPlexCreateFromFile()` internally
6865   calls `DMLoad()`. Currently, name is ignored for other viewer types and/or formats.
6866 
6867 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromDAG()`, `DMPlexCreateFromCellListPetsc()`, `DMPlexCreate()`, `PetscObjectSetName()`, `DMView()`, `DMLoad()`
6868 @*/
6869 PetscErrorCode DMPlexCreateFromFile(MPI_Comm comm, const char filename[], const char plexname[], PetscBool interpolate, DM *dm)
6870 {
6871   const char  extGmsh[]      = ".msh";
6872   const char  extGmsh2[]     = ".msh2";
6873   const char  extGmsh4[]     = ".msh4";
6874   const char  extCGNS[]      = ".cgns";
6875   const char  extExodus[]    = ".exo";
6876   const char  extExodus_e[]  = ".e";
6877   const char  extGenesis[]   = ".gen";
6878   const char  extFluent[]    = ".cas";
6879   const char  extHDF5[]      = ".h5";
6880   const char  extXDMFHDF5[]  = ".xdmf.h5";
6881   const char  extPLY[]       = ".ply";
6882   const char  extEGADSlite[] = ".egadslite";
6883   const char  extEGADS[]     = ".egads";
6884   const char  extIGES[]      = ".igs";
6885   const char  extIGES2[]     = ".iges";
6886   const char  extSTEP[]      = ".stp";
6887   const char  extSTEP2[]     = ".step";
6888   const char  extBREP[]      = ".brep";
6889   const char  extCV[]        = ".dat";
6890   size_t      len;
6891   PetscBool   isGmsh, isGmsh2, isGmsh4, isCGNS, isExodus, isGenesis, isFluent, isHDF5, isPLY, isEGADSlite, isEGADS, isIGES, isIGES2, isSTEP, isSTEP2, isBREP, isCV, isXDMFHDF5;
6892   PetscMPIInt rank;
6893 
6894   PetscFunctionBegin;
6895   PetscAssertPointer(filename, 2);
6896   if (plexname) PetscAssertPointer(plexname, 3);
6897   PetscAssertPointer(dm, 5);
6898   PetscCall(DMInitializePackage());
6899   PetscCall(PetscLogEventBegin(DMPLEX_CreateFromFile, 0, 0, 0, 0));
6900   PetscCallMPI(MPI_Comm_rank(comm, &rank));
6901   PetscCall(PetscStrlen(filename, &len));
6902   PetscCheck(len, comm, PETSC_ERR_ARG_WRONG, "Filename must be a valid path");
6903 
6904 #define CheckExtension(extension__, is_extension__) \
6905   do { \
6906     PetscAssert(sizeof(extension__), comm, PETSC_ERR_PLIB, "Zero-size extension: %s", extension__); \
6907     /* don't count the null-terminator at the end */ \
6908     const size_t ext_len = sizeof(extension__) - 1; \
6909     if (len < ext_len) { \
6910       is_extension__ = PETSC_FALSE; \
6911     } else { \
6912       PetscCall(PetscStrncmp(filename + len - ext_len, extension__, ext_len, &is_extension__)); \
6913     } \
6914   } while (0)
6915 
6916   CheckExtension(extGmsh, isGmsh);
6917   CheckExtension(extGmsh2, isGmsh2);
6918   CheckExtension(extGmsh4, isGmsh4);
6919   CheckExtension(extCGNS, isCGNS);
6920   CheckExtension(extExodus, isExodus);
6921   if (!isExodus) CheckExtension(extExodus_e, isExodus);
6922   CheckExtension(extGenesis, isGenesis);
6923   CheckExtension(extFluent, isFluent);
6924   CheckExtension(extHDF5, isHDF5);
6925   CheckExtension(extPLY, isPLY);
6926   CheckExtension(extEGADSlite, isEGADSlite);
6927   CheckExtension(extEGADS, isEGADS);
6928   CheckExtension(extIGES, isIGES);
6929   CheckExtension(extIGES2, isIGES2);
6930   CheckExtension(extSTEP, isSTEP);
6931   CheckExtension(extSTEP2, isSTEP2);
6932   CheckExtension(extBREP, isBREP);
6933   CheckExtension(extCV, isCV);
6934   CheckExtension(extXDMFHDF5, isXDMFHDF5);
6935 
6936 #undef CheckExtension
6937 
6938   if (isGmsh || isGmsh2 || isGmsh4) {
6939     PetscCall(DMPlexCreateGmshFromFile(comm, filename, interpolate, dm));
6940   } else if (isCGNS) {
6941     PetscCall(DMPlexCreateCGNSFromFile(comm, filename, interpolate, dm));
6942   } else if (isExodus || isGenesis) {
6943     PetscCall(DMPlexCreateExodusFromFile(comm, filename, interpolate, dm));
6944   } else if (isFluent) {
6945     PetscCall(DMPlexCreateFluentFromFile(comm, filename, interpolate, dm));
6946   } else if (isHDF5) {
6947     PetscViewer viewer;
6948 
6949     /* PETSC_VIEWER_HDF5_XDMF is used if the filename ends with .xdmf.h5, or if -dm_plex_create_from_hdf5_xdmf option is present */
6950     PetscCall(PetscOptionsGetBool(NULL, NULL, "-dm_plex_create_from_hdf5_xdmf", &isXDMFHDF5, NULL));
6951     PetscCall(PetscViewerCreate(comm, &viewer));
6952     PetscCall(PetscViewerSetType(viewer, PETSCVIEWERHDF5));
6953     PetscCall(PetscViewerSetOptionsPrefix(viewer, "dm_plex_create_"));
6954     PetscCall(PetscViewerSetFromOptions(viewer));
6955     PetscCall(PetscViewerFileSetMode(viewer, FILE_MODE_READ));
6956     PetscCall(PetscViewerFileSetName(viewer, filename));
6957 
6958     PetscCall(DMCreate(comm, dm));
6959     PetscCall(PetscObjectSetName((PetscObject)*dm, plexname));
6960     PetscCall(DMSetType(*dm, DMPLEX));
6961     if (isXDMFHDF5) PetscCall(PetscViewerPushFormat(viewer, PETSC_VIEWER_HDF5_XDMF));
6962     PetscCall(DMLoad(*dm, viewer));
6963     if (isXDMFHDF5) PetscCall(PetscViewerPopFormat(viewer));
6964     PetscCall(PetscViewerDestroy(&viewer));
6965 
6966     if (interpolate) {
6967       DM idm;
6968 
6969       PetscCall(DMPlexInterpolate(*dm, &idm));
6970       PetscCall(DMDestroy(dm));
6971       *dm = idm;
6972     }
6973   } else if (isPLY) {
6974     PetscCall(DMPlexCreatePLYFromFile(comm, filename, interpolate, dm));
6975   } else if (isEGADSlite || isEGADS || isIGES || isIGES2 || isSTEP || isSTEP2 || isBREP) {
6976     PetscCall(DMPlexCreateGeomFromFile(comm, filename, dm, isEGADSlite));
6977 
6978     if (!interpolate) {
6979       DM udm;
6980 
6981       PetscCall(DMPlexUninterpolate(*dm, &udm));
6982       PetscCall(DMDestroy(dm));
6983       *dm = udm;
6984     }
6985   } else if (isCV) {
6986     PetscCall(DMPlexCreateCellVertexFromFile(comm, filename, interpolate, dm));
6987   } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cannot load file %s: unrecognized extension", filename);
6988   PetscCall(PetscStrlen(plexname, &len));
6989   if (len) PetscCall(PetscObjectSetName((PetscObject)*dm, plexname));
6990   PetscCall(PetscLogEventEnd(DMPLEX_CreateFromFile, 0, 0, 0, 0));
6991   PetscFunctionReturn(PETSC_SUCCESS);
6992 }
6993 
6994 /*@
6995   DMPlexCreateEphemeral - This takes a `DMPlexTransform` and a base `DMPlex` and produces an ephemeral `DM`, meaning one that is created on the fly in response to queries.
6996 
6997   Input Parameters:
6998 + tr     - The `DMPlexTransform`
6999 - prefix - An options prefix, or NULL
7000 
7001   Output Parameter:
7002 . dm - The `DM`
7003 
7004   Level: beginner
7005 
7006   Notes:
7007   An emphemeral mesh is one that is not stored concretely, as in the default `DMPLEX` implementation, but rather is produced on the fly in response to queries, using information from the transform and the base mesh.
7008 
7009 .seealso: `DMPlexCreateFromFile`, `DMPlexCreateFromDAG()`, `DMPlexCreateFromCellListPetsc()`, `DMPlexCreate()`
7010 @*/
7011 PetscErrorCode DMPlexCreateEphemeral(DMPlexTransform tr, const char prefix[], DM *dm)
7012 {
7013   DM           bdm, bcdm, cdm;
7014   Vec          coordinates, coordinatesNew;
7015   PetscSection cs;
7016   PetscInt     cdim, Nl;
7017 
7018   PetscFunctionBegin;
7019   PetscCall(DMCreate(PetscObjectComm((PetscObject)tr), dm));
7020   PetscCall(DMSetType(*dm, DMPLEX));
7021   ((DM_Plex *)(*dm)->data)->interpolated = DMPLEX_INTERPOLATED_FULL;
7022   // Handle coordinates
7023   PetscCall(DMPlexTransformGetDM(tr, &bdm));
7024   PetscCall(DMPlexTransformSetDimensions(tr, bdm, *dm));
7025   PetscCall(DMGetCoordinateDim(*dm, &cdim));
7026   PetscCall(DMGetCoordinateDM(bdm, &bcdm));
7027   PetscCall(DMGetCoordinateDM(*dm, &cdm));
7028   PetscCall(DMCopyDisc(bcdm, cdm));
7029   PetscCall(DMGetLocalSection(cdm, &cs));
7030   PetscCall(PetscSectionSetNumFields(cs, 1));
7031   PetscCall(PetscSectionSetFieldComponents(cs, 0, cdim));
7032   PetscCall(DMGetCoordinatesLocal(bdm, &coordinates));
7033   PetscCall(VecDuplicate(coordinates, &coordinatesNew));
7034   PetscCall(VecCopy(coordinates, coordinatesNew));
7035   PetscCall(DMSetCoordinatesLocal(*dm, coordinatesNew));
7036   PetscCall(VecDestroy(&coordinatesNew));
7037 
7038   PetscCall(PetscObjectReference((PetscObject)tr));
7039   PetscCall(DMPlexTransformDestroy(&((DM_Plex *)(*dm)->data)->tr));
7040   ((DM_Plex *)(*dm)->data)->tr = tr;
7041   PetscCall(DMPlexDistributeSetDefault(*dm, PETSC_FALSE));
7042   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)*dm, prefix));
7043   PetscCall(DMSetFromOptions(*dm));
7044 
7045   PetscCall(DMGetNumLabels(bdm, &Nl));
7046   for (PetscInt l = 0; l < Nl; ++l) {
7047     DMLabel     label, labelNew;
7048     const char *lname;
7049     PetscBool   isDepth, isCellType;
7050 
7051     PetscCall(DMGetLabelName(bdm, l, &lname));
7052     PetscCall(PetscStrcmp(lname, "depth", &isDepth));
7053     if (isDepth) continue;
7054     PetscCall(PetscStrcmp(lname, "celltype", &isCellType));
7055     if (isCellType) continue;
7056     PetscCall(DMCreateLabel(*dm, lname));
7057     PetscCall(DMGetLabel(bdm, lname, &label));
7058     PetscCall(DMGetLabel(*dm, lname, &labelNew));
7059     PetscCall(DMLabelSetType(labelNew, DMLABELEPHEMERAL));
7060     PetscCall(DMLabelEphemeralSetLabel(labelNew, label));
7061     PetscCall(DMLabelEphemeralSetTransform(labelNew, tr));
7062     PetscCall(DMLabelSetUp(labelNew));
7063   }
7064   PetscFunctionReturn(PETSC_SUCCESS);
7065 }
7066