xref: /petsc/src/dm/impls/plex/plexcreate.c (revision 0baf8eba40dbc839082666f9f7396a225d6f663c)
1 #define PETSCDM_DLL
2 #include <petsc/private/dmpleximpl.h> /*I   "petscdmplex.h"   I*/
3 #include <petsc/private/hashseti.h>
4 #include <petscsf.h>
5 #include <petscdmplextransform.h> /*I   "petscdmplextransform.h"   I*/
6 #include <petscdmlabelephemeral.h>
7 #include <petsc/private/kernels/blockmatmult.h>
8 #include <petsc/private/kernels/blockinvert.h>
9 
10 #ifdef PETSC_HAVE_UNISTD_H
11   #include <unistd.h>
12 #endif
13 #include <errno.h>
14 
15 PetscLogEvent DMPLEX_CreateFromFile, DMPLEX_CreateFromOptions, DMPLEX_BuildFromCellList, DMPLEX_BuildCoordinatesFromCellList;
16 
17 /* External function declarations here */
18 static PetscErrorCode DMInitialize_Plex(DM dm);
19 
20 PETSC_EXTERN PetscErrorCode DMPlexCheckEGADS_Private(DM dm)
21 {
22   PetscObject modelObj;
23 
24   PetscFunctionBegin;
25   PetscCall(PetscObjectQuery((PetscObject)dm, "EGADS Model", &modelObj));
26   PetscCheck(modelObj, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Input DM must have attached EGADS Geometry Model");
27   PetscFunctionReturn(PETSC_SUCCESS);
28 }
29 
30 static PetscErrorCode DMPlexCopyContext_Private(DM dmin, const char name[], DM dmout)
31 {
32   PetscObject obj;
33 
34   PetscFunctionBegin;
35   PetscCall(PetscObjectQuery((PetscObject)dmin, name, &obj));
36   if (obj) PetscCall(PetscObjectCompose((PetscObject)dmout, name, obj));
37   PetscFunctionReturn(PETSC_SUCCESS);
38 }
39 
40 static PetscErrorCode DMPlexSwapContext_Private(DM dmA, const char name[], DM dmB)
41 {
42   PetscObject objA, objB;
43 
44   PetscFunctionBegin;
45   PetscCall(PetscObjectQuery((PetscObject)dmA, name, &objA));
46   PetscCall(PetscObjectQuery((PetscObject)dmB, name, &objB));
47   PetscCall(PetscObjectReference(objA));
48   PetscCall(PetscObjectReference(objB));
49   PetscCall(PetscObjectCompose((PetscObject)dmA, name, objB));
50   PetscCall(PetscObjectCompose((PetscObject)dmB, name, objA));
51   PetscCall(PetscObjectDereference(objA));
52   PetscCall(PetscObjectDereference(objB));
53   PetscFunctionReturn(PETSC_SUCCESS);
54 }
55 
56 PetscErrorCode DMPlexCopyEGADSInfo_Internal(DM dmin, DM dmout)
57 {
58   PetscFunctionBegin;
59   PetscCall(DMPlexCopyContext_Private(dmin, "EGADS Model", dmout));
60   PetscCall(DMPlexCopyContext_Private(dmin, "EGADS Context", dmout));
61   PetscCall(DMPlexCopyContext_Private(dmin, "EGADSlite Model", dmout));
62   PetscCall(DMPlexCopyContext_Private(dmin, "EGADSlite Context", dmout));
63   PetscFunctionReturn(PETSC_SUCCESS);
64 }
65 
66 static PetscErrorCode DMPlexSwapEGADSInfo_Private(DM dmA, DM dmB)
67 {
68   PetscFunctionBegin;
69   PetscCall(DMPlexSwapContext_Private(dmA, "EGADS Model", dmB));
70   PetscCall(DMPlexSwapContext_Private(dmA, "EGADS Context", dmB));
71   PetscCall(DMPlexSwapContext_Private(dmA, "EGADSlite Model", dmB));
72   PetscCall(DMPlexSwapContext_Private(dmA, "EGADSlite Context", dmB));
73   PetscFunctionReturn(PETSC_SUCCESS);
74 }
75 
76 /* This copies internal things in the Plex structure that we generally want when making a new, related Plex */
77 PetscErrorCode DMPlexCopy_Internal(DM dmin, PetscBool copyPeriodicity, PetscBool copyOverlap, DM dmout)
78 {
79   const PetscReal     *maxCell, *Lstart, *L;
80   VecType              vecType;
81   MatType              matType;
82   PetscBool            dist, useCeed, balance_partition;
83   DMReorderDefaultFlag reorder;
84 
85   PetscFunctionBegin;
86   if (dmin == dmout) PetscFunctionReturn(PETSC_SUCCESS);
87   PetscCall(DMGetVecType(dmin, &vecType));
88   PetscCall(DMSetVecType(dmout, vecType));
89   PetscCall(DMGetMatType(dmin, &matType));
90   PetscCall(DMSetMatType(dmout, matType));
91   if (copyPeriodicity) {
92     PetscCall(DMGetPeriodicity(dmin, &maxCell, &Lstart, &L));
93     PetscCall(DMSetPeriodicity(dmout, maxCell, Lstart, L));
94     PetscCall(DMLocalizeCoordinates(dmout));
95   }
96   PetscCall(DMPlexDistributeGetDefault(dmin, &dist));
97   PetscCall(DMPlexDistributeSetDefault(dmout, dist));
98   PetscCall(DMPlexReorderGetDefault(dmin, &reorder));
99   PetscCall(DMPlexReorderSetDefault(dmout, reorder));
100   PetscCall(DMPlexGetUseCeed(dmin, &useCeed));
101   PetscCall(DMPlexSetUseCeed(dmout, useCeed));
102   PetscCall(DMPlexGetPartitionBalance(dmin, &balance_partition));
103   PetscCall(DMPlexSetPartitionBalance(dmout, balance_partition));
104   ((DM_Plex *)dmout->data)->useHashLocation = ((DM_Plex *)dmin->data)->useHashLocation;
105   ((DM_Plex *)dmout->data)->printSetValues  = ((DM_Plex *)dmin->data)->printSetValues;
106   ((DM_Plex *)dmout->data)->printFEM        = ((DM_Plex *)dmin->data)->printFEM;
107   ((DM_Plex *)dmout->data)->printFVM        = ((DM_Plex *)dmin->data)->printFVM;
108   ((DM_Plex *)dmout->data)->printL2         = ((DM_Plex *)dmin->data)->printL2;
109   ((DM_Plex *)dmout->data)->printLocate     = ((DM_Plex *)dmin->data)->printLocate;
110   ((DM_Plex *)dmout->data)->printProject    = ((DM_Plex *)dmin->data)->printProject;
111   ((DM_Plex *)dmout->data)->printTol        = ((DM_Plex *)dmin->data)->printTol;
112   if (copyOverlap) PetscCall(DMPlexSetOverlap_Plex(dmout, dmin, 0));
113   PetscCall(DMPlexCopyEGADSInfo_Internal(dmin, dmout));
114   PetscFunctionReturn(PETSC_SUCCESS);
115 }
116 
117 /* Replace dm with the contents of ndm, and then destroy ndm
118    - Share the DM_Plex structure
119    - Share the coordinates
120    - Share the SF
121 */
122 PetscErrorCode DMPlexReplace_Internal(DM dm, DM *ndm)
123 {
124   PetscSF          sf;
125   DM               dmNew = *ndm, coordDM, coarseDM;
126   Vec              coords;
127   const PetscReal *maxCell, *Lstart, *L;
128   PetscInt         dim, cdim;
129   PetscBool        use_natural;
130 
131   PetscFunctionBegin;
132   if (dm == dmNew) {
133     PetscCall(DMDestroy(ndm));
134     PetscFunctionReturn(PETSC_SUCCESS);
135   }
136   dm->setupcalled = dmNew->setupcalled;
137   if (!dm->hdr.name) {
138     const char *name;
139 
140     PetscCall(PetscObjectGetName((PetscObject)*ndm, &name));
141     PetscCall(PetscObjectSetName((PetscObject)dm, name));
142   }
143   PetscCall(DMGetDimension(dmNew, &dim));
144   PetscCall(DMSetDimension(dm, dim));
145   PetscCall(DMGetCoordinateDim(dmNew, &cdim));
146   PetscCall(DMSetCoordinateDim(dm, cdim));
147   PetscCall(DMGetPointSF(dmNew, &sf));
148   PetscCall(DMSetPointSF(dm, sf));
149   PetscCall(DMGetCoordinateDM(dmNew, &coordDM));
150   PetscCall(DMGetCoordinatesLocal(dmNew, &coords));
151   PetscCall(DMSetCoordinateDM(dm, coordDM));
152   PetscCall(DMSetCoordinatesLocal(dm, coords));
153   PetscCall(DMGetCellCoordinateDM(dmNew, &coordDM));
154   PetscCall(DMGetCellCoordinatesLocal(dmNew, &coords));
155   PetscCall(DMSetCellCoordinateDM(dm, coordDM));
156   PetscCall(DMSetCellCoordinatesLocal(dm, coords));
157   /* Do not want to create the coordinate field if it does not already exist, so do not call DMGetCoordinateField() */
158   PetscCall(DMFieldDestroy(&dm->coordinates[0].field));
159   dm->coordinates[0].field            = dmNew->coordinates[0].field;
160   ((DM_Plex *)dmNew->data)->coordFunc = ((DM_Plex *)dm->data)->coordFunc;
161   PetscCall(DMGetPeriodicity(dmNew, &maxCell, &Lstart, &L));
162   PetscCall(DMSetPeriodicity(dm, maxCell, Lstart, L));
163   PetscCall(DMGetNaturalSF(dmNew, &sf));
164   PetscCall(DMSetNaturalSF(dm, sf));
165   PetscCall(DMGetUseNatural(dmNew, &use_natural));
166   PetscCall(DMSetUseNatural(dm, use_natural));
167   PetscCall(DMDestroy_Plex(dm));
168   PetscCall(DMInitialize_Plex(dm));
169   dm->data = dmNew->data;
170   ((DM_Plex *)dmNew->data)->refct++;
171   {
172     PetscInt       num_face_sfs;
173     const PetscSF *sfs;
174     PetscCall(DMPlexGetIsoperiodicFaceSF(dm, &num_face_sfs, &sfs));
175     PetscCall(DMPlexSetIsoperiodicFaceSF(dm, num_face_sfs, (PetscSF *)sfs)); // for the compose function effect on dm
176   }
177   PetscCall(DMDestroyLabelLinkList_Internal(dm));
178   PetscCall(DMCopyLabels(dmNew, dm, PETSC_OWN_POINTER, PETSC_TRUE, DM_COPY_LABELS_FAIL));
179   PetscCall(DMGetCoarseDM(dmNew, &coarseDM));
180   PetscCall(DMSetCoarseDM(dm, coarseDM));
181   PetscCall(DMPlexCopyEGADSInfo_Internal(dmNew, dm));
182   PetscCall(DMDestroy(ndm));
183   PetscFunctionReturn(PETSC_SUCCESS);
184 }
185 
186 /* Swap dm with the contents of dmNew
187    - Swap the DM_Plex structure
188    - Swap the coordinates
189    - Swap the point PetscSF
190 */
191 static PetscErrorCode DMPlexSwap_Static(DM dmA, DM dmB)
192 {
193   DM          coordDMA, coordDMB;
194   Vec         coordsA, coordsB;
195   PetscSF     sfA, sfB;
196   DMField     fieldTmp;
197   void       *tmp;
198   DMLabelLink listTmp;
199   DMLabel     depthTmp;
200   PetscInt    tmpI;
201 
202   PetscFunctionBegin;
203   if (dmA == dmB) PetscFunctionReturn(PETSC_SUCCESS);
204   PetscCall(DMGetPointSF(dmA, &sfA));
205   PetscCall(DMGetPointSF(dmB, &sfB));
206   PetscCall(PetscObjectReference((PetscObject)sfA));
207   PetscCall(DMSetPointSF(dmA, sfB));
208   PetscCall(DMSetPointSF(dmB, sfA));
209   PetscCall(PetscObjectDereference((PetscObject)sfA));
210 
211   PetscCall(DMGetCoordinateDM(dmA, &coordDMA));
212   PetscCall(DMGetCoordinateDM(dmB, &coordDMB));
213   PetscCall(PetscObjectReference((PetscObject)coordDMA));
214   PetscCall(DMSetCoordinateDM(dmA, coordDMB));
215   PetscCall(DMSetCoordinateDM(dmB, coordDMA));
216   PetscCall(PetscObjectDereference((PetscObject)coordDMA));
217 
218   PetscCall(DMGetCoordinatesLocal(dmA, &coordsA));
219   PetscCall(DMGetCoordinatesLocal(dmB, &coordsB));
220   PetscCall(PetscObjectReference((PetscObject)coordsA));
221   PetscCall(DMSetCoordinatesLocal(dmA, coordsB));
222   PetscCall(DMSetCoordinatesLocal(dmB, coordsA));
223   PetscCall(PetscObjectDereference((PetscObject)coordsA));
224 
225   PetscCall(DMGetCellCoordinateDM(dmA, &coordDMA));
226   PetscCall(DMGetCellCoordinateDM(dmB, &coordDMB));
227   PetscCall(PetscObjectReference((PetscObject)coordDMA));
228   PetscCall(DMSetCellCoordinateDM(dmA, coordDMB));
229   PetscCall(DMSetCellCoordinateDM(dmB, coordDMA));
230   PetscCall(PetscObjectDereference((PetscObject)coordDMA));
231 
232   PetscCall(DMGetCellCoordinatesLocal(dmA, &coordsA));
233   PetscCall(DMGetCellCoordinatesLocal(dmB, &coordsB));
234   PetscCall(PetscObjectReference((PetscObject)coordsA));
235   PetscCall(DMSetCellCoordinatesLocal(dmA, coordsB));
236   PetscCall(DMSetCellCoordinatesLocal(dmB, coordsA));
237   PetscCall(PetscObjectDereference((PetscObject)coordsA));
238 
239   PetscCall(DMPlexSwapEGADSInfo_Private(dmA, dmB));
240 
241   fieldTmp                  = dmA->coordinates[0].field;
242   dmA->coordinates[0].field = dmB->coordinates[0].field;
243   dmB->coordinates[0].field = fieldTmp;
244   fieldTmp                  = dmA->coordinates[1].field;
245   dmA->coordinates[1].field = dmB->coordinates[1].field;
246   dmB->coordinates[1].field = fieldTmp;
247   tmp                       = dmA->data;
248   dmA->data                 = dmB->data;
249   dmB->data                 = tmp;
250   listTmp                   = dmA->labels;
251   dmA->labels               = dmB->labels;
252   dmB->labels               = listTmp;
253   depthTmp                  = dmA->depthLabel;
254   dmA->depthLabel           = dmB->depthLabel;
255   dmB->depthLabel           = depthTmp;
256   depthTmp                  = dmA->celltypeLabel;
257   dmA->celltypeLabel        = dmB->celltypeLabel;
258   dmB->celltypeLabel        = depthTmp;
259   tmpI                      = dmA->levelup;
260   dmA->levelup              = dmB->levelup;
261   dmB->levelup              = tmpI;
262   PetscFunctionReturn(PETSC_SUCCESS);
263 }
264 
265 PetscErrorCode DMPlexInterpolateInPlace_Internal(DM dm)
266 {
267   DM idm;
268 
269   PetscFunctionBegin;
270   PetscCall(DMPlexInterpolate(dm, &idm));
271   PetscCall(DMPlexCopyCoordinates(dm, idm));
272   PetscCall(DMPlexReplace_Internal(dm, &idm));
273   PetscFunctionReturn(PETSC_SUCCESS);
274 }
275 
276 /*@C
277   DMPlexCreateCoordinateSpace - Creates a finite element space for the coordinates
278 
279   Collective
280 
281   Input Parameters:
282 + dm        - The `DMPLEX`
283 . degree    - The degree of the finite element or `PETSC_DECIDE`
284 . project   - Flag to project current coordinates into the space
285 - coordFunc - An optional function to map new points from refinement to the surface
286 
287   Level: advanced
288 
289 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `PetscPointFunc`, `PetscFECreateLagrange()`, `DMGetCoordinateDM()`
290 @*/
291 PetscErrorCode DMPlexCreateCoordinateSpace(DM dm, PetscInt degree, PetscBool project, PetscPointFunc coordFunc)
292 {
293   DM_Plex *mesh = (DM_Plex *)dm->data;
294   PetscFE  fe   = NULL;
295   DM       cdm;
296   PetscInt dim, cdim, dE, qorder, height;
297 
298   PetscFunctionBegin;
299   PetscCall(DMGetDimension(dm, &dim));
300   cdim = dim;
301   PetscCall(DMGetCoordinateDim(dm, &dE));
302   qorder = degree;
303   PetscCall(DMGetCoordinateDM(dm, &cdm));
304   PetscObjectOptionsBegin((PetscObject)cdm);
305   PetscCall(PetscOptionsBoundedInt("-default_quadrature_order", "Quadrature order is one less than quadrature points per edge", "DMPlexCreateCoordinateSpace", qorder, &qorder, NULL, 0));
306   PetscCall(PetscOptionsBoundedInt("-dm_plex_coordinate_dim", "Set the coordinate dimension", "DMPlexCreateCoordinateSpace", cdim, &cdim, NULL, dim));
307   PetscOptionsEnd();
308   PetscCall(DMPlexGetVTKCellHeight(dm, &height));
309   if (cdim > dim) {
310     DM           cdm;
311     PetscSection cs, csNew;
312     Vec          coordinates, coordinatesNew;
313     VecType      vectype;
314     IS           idx;
315     PetscInt    *indices;
316     PetscInt     bs, n;
317 
318     // Recreate coordinate section
319     {
320       const char *fieldName = NULL, *compName = NULL;
321       PetscInt    Nc, pStart, pEnd;
322 
323       PetscCall(DMGetCoordinateDM(dm, &cdm));
324       PetscCall(DMGetLocalSection(cdm, &cs));
325       PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)cs), &csNew));
326       PetscCall(PetscSectionSetNumFields(csNew, 1));
327       PetscCall(PetscSectionGetFieldName(cs, 0, &fieldName));
328       PetscCall(PetscSectionSetFieldName(csNew, 0, fieldName));
329       PetscCall(PetscSectionGetFieldComponents(cs, 0, &Nc));
330       PetscCall(PetscSectionSetFieldComponents(csNew, 0, cdim));
331       for (PetscInt c = 0; c < Nc; ++c) {
332         PetscCall(PetscSectionGetComponentName(cs, 0, c, &compName));
333         PetscCall(PetscSectionSetComponentName(csNew, 0, c, compName));
334       }
335       PetscCall(PetscSectionGetChart(cs, &pStart, &pEnd));
336       PetscCall(PetscSectionSetChart(csNew, pStart, pEnd));
337       for (PetscInt p = pStart; p < pEnd; ++p) {
338         PetscInt dof;
339 
340         PetscCall(PetscSectionGetDof(cs, p, &dof));
341         if (dof) {
342           PetscCall(PetscSectionSetDof(csNew, p, cdim));
343           PetscCall(PetscSectionSetFieldDof(csNew, p, 0, cdim));
344         }
345       }
346       PetscCall(PetscSectionSetUp(csNew));
347     }
348     PetscCall(DMSetLocalSection(cdm, csNew));
349     PetscCall(PetscSectionDestroy(&csNew));
350     // Reset coordinate dimension for coordinate DM
351     PetscCall(DMSetCoordinateDim(cdm, cdim));
352     PetscCall(DMSetCoordinateField(cdm, NULL));
353     // Inject coordinates into higher dimension
354     PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
355     PetscCall(VecGetBlockSize(coordinates, &bs));
356     PetscCheck(bs == dim, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "We can only inject simple coordinates into a higher dimension");
357     PetscCall(VecCreate(PetscObjectComm((PetscObject)coordinates), &coordinatesNew));
358     PetscCall(VecGetType(coordinates, &vectype));
359     PetscCall(VecSetType(coordinatesNew, vectype));
360     PetscCall(VecGetLocalSize(coordinates, &n));
361     PetscCheck(!(n % bs), PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "We can only inject simple coordinates into a higher dimension");
362     n /= bs;
363     PetscCall(VecSetSizes(coordinatesNew, n * cdim, PETSC_DETERMINE));
364     PetscCall(VecSetUp(coordinatesNew));
365     PetscCall(PetscMalloc1(n * bs, &indices));
366     for (PetscInt i = 0; i < n; ++i)
367       for (PetscInt b = 0; b < bs; ++b) indices[i * bs + b] = i * cdim + b;
368     PetscCall(ISCreateGeneral(PETSC_COMM_SELF, n * bs, indices, PETSC_OWN_POINTER, &idx));
369     PetscCall(VecISCopy(coordinatesNew, idx, SCATTER_FORWARD, coordinates));
370     PetscCall(ISDestroy(&idx));
371     PetscCall(DMSetCoordinatesLocal(dm, coordinatesNew));
372     PetscCall(VecDestroy(&coordinatesNew));
373     PetscCall(DMSetCoordinateDim(dm, cdim));
374     {
375       PetscInt gn;
376 
377       PetscCall(DMGetCoordinates(dm, &coordinatesNew));
378       PetscCall(VecGetLocalSize(coordinatesNew, &gn));
379       PetscCheck(gn == n * cdim, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Global coordinate size %" PetscInt_FMT " != %" PetscInt_FMT "local coordinate size", gn, n * cdim);
380     }
381     dE      = cdim;
382     project = PETSC_FALSE;
383   }
384   if (degree >= 0) {
385     DMPolytopeType ct = DM_POLYTOPE_UNKNOWN;
386     PetscInt       cStart, cEnd, gct;
387 
388     PetscCall(DMPlexGetHeightStratum(dm, height, &cStart, &cEnd));
389     if (cEnd > cStart) PetscCall(DMPlexGetCellType(dm, cStart, &ct));
390     gct = (PetscInt)ct;
391     PetscCallMPI(MPIU_Allreduce(MPI_IN_PLACE, &gct, 1, MPIU_INT, MPI_MIN, PetscObjectComm((PetscObject)dm)));
392     ct = (DMPolytopeType)gct;
393     // Work around current bug in PetscDualSpaceSetUp_Lagrange()
394     //   Can be seen in plex_tutorials-ex10_1
395     if (ct != DM_POLYTOPE_SEG_PRISM_TENSOR && ct != DM_POLYTOPE_TRI_PRISM_TENSOR && ct != DM_POLYTOPE_QUAD_PRISM_TENSOR) PetscCall(PetscFECreateLagrangeByCell(PETSC_COMM_SELF, dim, dE, ct, degree, qorder, &fe));
396   }
397   PetscCall(DMSetCoordinateDisc(dm, fe, project));
398   PetscCall(PetscFEDestroy(&fe));
399   mesh->coordFunc = coordFunc;
400   PetscFunctionReturn(PETSC_SUCCESS);
401 }
402 
403 /*@
404   DMPlexCreateDoublet - Creates a mesh of two cells of the specified type, optionally with later refinement.
405 
406   Collective
407 
408   Input Parameters:
409 + comm            - The communicator for the `DM` object
410 . dim             - The spatial dimension
411 . simplex         - Flag for simplicial cells, otherwise they are tensor product cells
412 . interpolate     - Flag to create intermediate mesh pieces (edges, faces)
413 - refinementLimit - A nonzero number indicates the largest admissible volume for a refined cell
414 
415   Output Parameter:
416 . newdm - The `DM` object
417 
418   Level: beginner
419 
420 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMSetType()`, `DMCreate()`
421 @*/
422 PetscErrorCode DMPlexCreateDoublet(MPI_Comm comm, PetscInt dim, PetscBool simplex, PetscBool interpolate, PetscReal refinementLimit, DM *newdm)
423 {
424   DM          dm;
425   PetscMPIInt rank;
426 
427   PetscFunctionBegin;
428   PetscCall(DMCreate(comm, &dm));
429   PetscCall(DMSetType(dm, DMPLEX));
430   PetscCall(DMSetDimension(dm, dim));
431   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
432   PetscCallMPI(MPI_Comm_rank(comm, &rank));
433   switch (dim) {
434   case 2:
435     if (simplex) PetscCall(PetscObjectSetName((PetscObject)dm, "triangular"));
436     else PetscCall(PetscObjectSetName((PetscObject)dm, "quadrilateral"));
437     break;
438   case 3:
439     if (simplex) PetscCall(PetscObjectSetName((PetscObject)dm, "tetrahedral"));
440     else PetscCall(PetscObjectSetName((PetscObject)dm, "hexahedral"));
441     break;
442   default:
443     SETERRQ(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %" PetscInt_FMT, dim);
444   }
445   if (rank) {
446     PetscInt numPoints[2] = {0, 0};
447     PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, NULL, NULL, NULL, NULL));
448   } else {
449     switch (dim) {
450     case 2:
451       if (simplex) {
452         PetscInt    numPoints[2]        = {4, 2};
453         PetscInt    coneSize[6]         = {3, 3, 0, 0, 0, 0};
454         PetscInt    cones[6]            = {2, 3, 4, 5, 4, 3};
455         PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
456         PetscScalar vertexCoords[8]     = {-0.5, 0.5, 0.0, 0.0, 0.0, 1.0, 0.5, 0.5};
457 
458         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
459       } else {
460         PetscInt    numPoints[2]        = {6, 2};
461         PetscInt    coneSize[8]         = {4, 4, 0, 0, 0, 0, 0, 0};
462         PetscInt    cones[8]            = {2, 3, 4, 5, 3, 6, 7, 4};
463         PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
464         PetscScalar vertexCoords[12]    = {-1.0, -0.5, 0.0, -0.5, 0.0, 0.5, -1.0, 0.5, 1.0, -0.5, 1.0, 0.5};
465 
466         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
467       }
468       break;
469     case 3:
470       if (simplex) {
471         PetscInt    numPoints[2]        = {5, 2};
472         PetscInt    coneSize[7]         = {4, 4, 0, 0, 0, 0, 0};
473         PetscInt    cones[8]            = {4, 3, 5, 2, 5, 3, 4, 6};
474         PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
475         PetscScalar vertexCoords[15]    = {-1.0, 0.0, 0.0, 0.0, -1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 1.0, 0.0, 1.0, 0.0, 0.0};
476 
477         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
478       } else {
479         PetscInt    numPoints[2]         = {12, 2};
480         PetscInt    coneSize[14]         = {8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
481         PetscInt    cones[16]            = {2, 3, 4, 5, 6, 7, 8, 9, 5, 4, 10, 11, 7, 12, 13, 8};
482         PetscInt    coneOrientations[16] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
483         PetscScalar vertexCoords[36]     = {-1.0, -0.5, -0.5, -1.0, 0.5, -0.5, 0.0, 0.5, -0.5, 0.0, -0.5, -0.5, -1.0, -0.5, 0.5, 0.0, -0.5, 0.5, 0.0, 0.5, 0.5, -1.0, 0.5, 0.5, 1.0, 0.5, -0.5, 1.0, -0.5, -0.5, 1.0, -0.5, 0.5, 1.0, 0.5, 0.5};
484 
485         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
486       }
487       break;
488     default:
489       SETERRQ(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %" PetscInt_FMT, dim);
490     }
491   }
492   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
493   *newdm = dm;
494   if (refinementLimit > 0.0) {
495     DM          rdm;
496     const char *name;
497 
498     PetscCall(DMPlexSetRefinementUniform(*newdm, PETSC_FALSE));
499     PetscCall(DMPlexSetRefinementLimit(*newdm, refinementLimit));
500     PetscCall(DMRefine(*newdm, comm, &rdm));
501     PetscCall(PetscObjectGetName((PetscObject)*newdm, &name));
502     PetscCall(PetscObjectSetName((PetscObject)rdm, name));
503     PetscCall(DMDestroy(newdm));
504     *newdm = rdm;
505   }
506   if (interpolate) {
507     DM idm;
508 
509     PetscCall(DMPlexInterpolate(*newdm, &idm));
510     PetscCall(DMDestroy(newdm));
511     *newdm = idm;
512   }
513   PetscFunctionReturn(PETSC_SUCCESS);
514 }
515 
516 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_1D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[])
517 {
518   const PetscInt numVertices    = 2;
519   PetscInt       markerRight    = 1;
520   PetscInt       markerLeft     = 1;
521   PetscBool      markerSeparate = PETSC_FALSE;
522   Vec            coordinates;
523   PetscSection   coordSection;
524   PetscScalar   *coords;
525   PetscInt       coordSize;
526   PetscMPIInt    rank;
527   PetscInt       cdim = 1, v;
528 
529   PetscFunctionBegin;
530   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
531   if (markerSeparate) {
532     markerRight = 2;
533     markerLeft  = 1;
534   }
535   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
536   if (rank == 0) {
537     PetscCall(DMPlexSetChart(dm, 0, numVertices));
538     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
539     PetscCall(DMSetLabelValue(dm, "marker", 0, markerLeft));
540     PetscCall(DMSetLabelValue(dm, "marker", 1, markerRight));
541   }
542   PetscCall(DMPlexSymmetrize(dm));
543   PetscCall(DMPlexStratify(dm));
544   /* Build coordinates */
545   PetscCall(DMSetCoordinateDim(dm, cdim));
546   PetscCall(DMGetCoordinateSection(dm, &coordSection));
547   PetscCall(PetscSectionSetNumFields(coordSection, 1));
548   PetscCall(PetscSectionSetChart(coordSection, 0, numVertices));
549   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, cdim));
550   for (v = 0; v < numVertices; ++v) {
551     PetscCall(PetscSectionSetDof(coordSection, v, cdim));
552     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, cdim));
553   }
554   PetscCall(PetscSectionSetUp(coordSection));
555   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
556   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
557   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
558   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
559   PetscCall(VecSetBlockSize(coordinates, cdim));
560   PetscCall(VecSetType(coordinates, VECSTANDARD));
561   PetscCall(VecGetArray(coordinates, &coords));
562   coords[0] = lower[0];
563   coords[1] = upper[0];
564   PetscCall(VecRestoreArray(coordinates, &coords));
565   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
566   PetscCall(VecDestroy(&coordinates));
567   PetscFunctionReturn(PETSC_SUCCESS);
568 }
569 
570 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_2D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[])
571 {
572   const PetscInt numVertices    = (edges[0] + 1) * (edges[1] + 1);
573   const PetscInt numEdges       = edges[0] * (edges[1] + 1) + (edges[0] + 1) * edges[1];
574   PetscInt       markerTop      = 1;
575   PetscInt       markerBottom   = 1;
576   PetscInt       markerRight    = 1;
577   PetscInt       markerLeft     = 1;
578   PetscBool      markerSeparate = PETSC_FALSE;
579   Vec            coordinates;
580   PetscSection   coordSection;
581   PetscScalar   *coords;
582   PetscInt       coordSize;
583   PetscMPIInt    rank;
584   PetscInt       v, vx, vy;
585 
586   PetscFunctionBegin;
587   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
588   if (markerSeparate) {
589     markerTop    = 3;
590     markerBottom = 1;
591     markerRight  = 2;
592     markerLeft   = 4;
593   }
594   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
595   if (rank == 0) {
596     PetscInt e, ex, ey;
597 
598     PetscCall(DMPlexSetChart(dm, 0, numEdges + numVertices));
599     for (e = 0; e < numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
600     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
601     for (vx = 0; vx <= edges[0]; vx++) {
602       for (ey = 0; ey < edges[1]; ey++) {
603         PetscInt edge   = vx * edges[1] + ey + edges[0] * (edges[1] + 1);
604         PetscInt vertex = ey * (edges[0] + 1) + vx + numEdges;
605         PetscInt cone[2];
606 
607         cone[0] = vertex;
608         cone[1] = vertex + edges[0] + 1;
609         PetscCall(DMPlexSetCone(dm, edge, cone));
610         if (vx == edges[0]) {
611           PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
612           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
613           if (ey == edges[1] - 1) {
614             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
615             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerRight));
616           }
617         } else if (vx == 0) {
618           PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
619           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
620           if (ey == edges[1] - 1) {
621             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
622             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerLeft));
623           }
624         }
625       }
626     }
627     for (vy = 0; vy <= edges[1]; vy++) {
628       for (ex = 0; ex < edges[0]; ex++) {
629         PetscInt edge   = vy * edges[0] + ex;
630         PetscInt vertex = vy * (edges[0] + 1) + ex + numEdges;
631         PetscInt cone[2];
632 
633         cone[0] = vertex;
634         cone[1] = vertex + 1;
635         PetscCall(DMPlexSetCone(dm, edge, cone));
636         if (vy == edges[1]) {
637           PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
638           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
639           if (ex == edges[0] - 1) {
640             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
641             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerTop));
642           }
643         } else if (vy == 0) {
644           PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
645           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
646           if (ex == edges[0] - 1) {
647             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
648             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerBottom));
649           }
650         }
651       }
652     }
653   }
654   PetscCall(DMPlexSymmetrize(dm));
655   PetscCall(DMPlexStratify(dm));
656   /* Build coordinates */
657   PetscCall(DMSetCoordinateDim(dm, 2));
658   PetscCall(DMGetCoordinateSection(dm, &coordSection));
659   PetscCall(PetscSectionSetNumFields(coordSection, 1));
660   PetscCall(PetscSectionSetChart(coordSection, numEdges, numEdges + numVertices));
661   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, 2));
662   for (v = numEdges; v < numEdges + numVertices; ++v) {
663     PetscCall(PetscSectionSetDof(coordSection, v, 2));
664     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, 2));
665   }
666   PetscCall(PetscSectionSetUp(coordSection));
667   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
668   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
669   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
670   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
671   PetscCall(VecSetBlockSize(coordinates, 2));
672   PetscCall(VecSetType(coordinates, VECSTANDARD));
673   PetscCall(VecGetArray(coordinates, &coords));
674   for (vy = 0; vy <= edges[1]; ++vy) {
675     for (vx = 0; vx <= edges[0]; ++vx) {
676       coords[(vy * (edges[0] + 1) + vx) * 2 + 0] = lower[0] + ((upper[0] - lower[0]) / edges[0]) * vx;
677       coords[(vy * (edges[0] + 1) + vx) * 2 + 1] = lower[1] + ((upper[1] - lower[1]) / edges[1]) * vy;
678     }
679   }
680   PetscCall(VecRestoreArray(coordinates, &coords));
681   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
682   PetscCall(VecDestroy(&coordinates));
683   PetscFunctionReturn(PETSC_SUCCESS);
684 }
685 
686 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_3D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt faces[])
687 {
688   PetscInt     vertices[3], numVertices;
689   PetscInt     numFaces       = 2 * faces[0] * faces[1] + 2 * faces[1] * faces[2] + 2 * faces[0] * faces[2];
690   PetscInt     markerTop      = 1;
691   PetscInt     markerBottom   = 1;
692   PetscInt     markerFront    = 1;
693   PetscInt     markerBack     = 1;
694   PetscInt     markerRight    = 1;
695   PetscInt     markerLeft     = 1;
696   PetscBool    markerSeparate = PETSC_FALSE;
697   Vec          coordinates;
698   PetscSection coordSection;
699   PetscScalar *coords;
700   PetscInt     coordSize;
701   PetscMPIInt  rank;
702   PetscInt     v, vx, vy, vz;
703   PetscInt     voffset, iface = 0, cone[4];
704 
705   PetscFunctionBegin;
706   PetscCheck(faces[0] >= 1 && faces[1] >= 1 && faces[2] >= 1, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Must have at least 1 face per side");
707   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
708   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
709   if (markerSeparate) {
710     markerBottom = 1;
711     markerTop    = 2;
712     markerFront  = 3;
713     markerBack   = 4;
714     markerRight  = 5;
715     markerLeft   = 6;
716   }
717   vertices[0] = faces[0] + 1;
718   vertices[1] = faces[1] + 1;
719   vertices[2] = faces[2] + 1;
720   numVertices = vertices[0] * vertices[1] * vertices[2];
721   if (rank == 0) {
722     PetscInt f;
723 
724     PetscCall(DMPlexSetChart(dm, 0, numFaces + numVertices));
725     for (f = 0; f < numFaces; ++f) PetscCall(DMPlexSetConeSize(dm, f, 4));
726     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
727 
728     /* Side 0 (Top) */
729     for (vy = 0; vy < faces[1]; vy++) {
730       for (vx = 0; vx < faces[0]; vx++) {
731         voffset = numFaces + vertices[0] * vertices[1] * (vertices[2] - 1) + vy * vertices[0] + vx;
732         cone[0] = voffset;
733         cone[1] = voffset + 1;
734         cone[2] = voffset + vertices[0] + 1;
735         cone[3] = voffset + vertices[0];
736         PetscCall(DMPlexSetCone(dm, iface, cone));
737         PetscCall(DMSetLabelValue(dm, "marker", iface, markerTop));
738         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerTop));
739         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerTop));
740         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerTop));
741         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 1, markerTop));
742         iface++;
743       }
744     }
745 
746     /* Side 1 (Bottom) */
747     for (vy = 0; vy < faces[1]; vy++) {
748       for (vx = 0; vx < faces[0]; vx++) {
749         voffset = numFaces + vy * (faces[0] + 1) + vx;
750         cone[0] = voffset + 1;
751         cone[1] = voffset;
752         cone[2] = voffset + vertices[0];
753         cone[3] = voffset + vertices[0] + 1;
754         PetscCall(DMPlexSetCone(dm, iface, cone));
755         PetscCall(DMSetLabelValue(dm, "marker", iface, markerBottom));
756         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerBottom));
757         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerBottom));
758         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerBottom));
759         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 1, markerBottom));
760         iface++;
761       }
762     }
763 
764     /* Side 2 (Front) */
765     for (vz = 0; vz < faces[2]; vz++) {
766       for (vx = 0; vx < faces[0]; vx++) {
767         voffset = numFaces + vz * vertices[0] * vertices[1] + vx;
768         cone[0] = voffset;
769         cone[1] = voffset + 1;
770         cone[2] = voffset + vertices[0] * vertices[1] + 1;
771         cone[3] = voffset + vertices[0] * vertices[1];
772         PetscCall(DMPlexSetCone(dm, iface, cone));
773         PetscCall(DMSetLabelValue(dm, "marker", iface, markerFront));
774         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerFront));
775         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerFront));
776         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 0, markerFront));
777         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 1, markerFront));
778         iface++;
779       }
780     }
781 
782     /* Side 3 (Back) */
783     for (vz = 0; vz < faces[2]; vz++) {
784       for (vx = 0; vx < faces[0]; vx++) {
785         voffset = numFaces + vz * vertices[0] * vertices[1] + vertices[0] * (vertices[1] - 1) + vx;
786         cone[0] = voffset + vertices[0] * vertices[1];
787         cone[1] = voffset + vertices[0] * vertices[1] + 1;
788         cone[2] = voffset + 1;
789         cone[3] = voffset;
790         PetscCall(DMPlexSetCone(dm, iface, cone));
791         PetscCall(DMSetLabelValue(dm, "marker", iface, markerBack));
792         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerBack));
793         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerBack));
794         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 0, markerBack));
795         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 1, markerBack));
796         iface++;
797       }
798     }
799 
800     /* Side 4 (Left) */
801     for (vz = 0; vz < faces[2]; vz++) {
802       for (vy = 0; vy < faces[1]; vy++) {
803         voffset = numFaces + vz * vertices[0] * vertices[1] + vy * vertices[0];
804         cone[0] = voffset;
805         cone[1] = voffset + vertices[0] * vertices[1];
806         cone[2] = voffset + vertices[0] * vertices[1] + vertices[0];
807         cone[3] = voffset + vertices[0];
808         PetscCall(DMPlexSetCone(dm, iface, cone));
809         PetscCall(DMSetLabelValue(dm, "marker", iface, markerLeft));
810         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerLeft));
811         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerLeft));
812         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[1] + 0, markerLeft));
813         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + vertices[0], markerLeft));
814         iface++;
815       }
816     }
817 
818     /* Side 5 (Right) */
819     for (vz = 0; vz < faces[2]; vz++) {
820       for (vy = 0; vy < faces[1]; vy++) {
821         voffset = numFaces + vz * vertices[0] * vertices[1] + vy * vertices[0] + faces[0];
822         cone[0] = voffset + vertices[0] * vertices[1];
823         cone[1] = voffset;
824         cone[2] = voffset + vertices[0];
825         cone[3] = voffset + vertices[0] * vertices[1] + vertices[0];
826         PetscCall(DMPlexSetCone(dm, iface, cone));
827         PetscCall(DMSetLabelValue(dm, "marker", iface, markerRight));
828         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerRight));
829         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerRight));
830         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 0, markerRight));
831         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + vertices[0], markerRight));
832         iface++;
833       }
834     }
835   }
836   PetscCall(DMPlexSymmetrize(dm));
837   PetscCall(DMPlexStratify(dm));
838   /* Build coordinates */
839   PetscCall(DMSetCoordinateDim(dm, 3));
840   PetscCall(DMGetCoordinateSection(dm, &coordSection));
841   PetscCall(PetscSectionSetNumFields(coordSection, 1));
842   PetscCall(PetscSectionSetChart(coordSection, numFaces, numFaces + numVertices));
843   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, 3));
844   for (v = numFaces; v < numFaces + numVertices; ++v) {
845     PetscCall(PetscSectionSetDof(coordSection, v, 3));
846     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, 3));
847   }
848   PetscCall(PetscSectionSetUp(coordSection));
849   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
850   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
851   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
852   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
853   PetscCall(VecSetBlockSize(coordinates, 3));
854   PetscCall(VecSetType(coordinates, VECSTANDARD));
855   PetscCall(VecGetArray(coordinates, &coords));
856   for (vz = 0; vz <= faces[2]; ++vz) {
857     for (vy = 0; vy <= faces[1]; ++vy) {
858       for (vx = 0; vx <= faces[0]; ++vx) {
859         coords[((vz * (faces[1] + 1) + vy) * (faces[0] + 1) + vx) * 3 + 0] = lower[0] + ((upper[0] - lower[0]) / faces[0]) * vx;
860         coords[((vz * (faces[1] + 1) + vy) * (faces[0] + 1) + vx) * 3 + 1] = lower[1] + ((upper[1] - lower[1]) / faces[1]) * vy;
861         coords[((vz * (faces[1] + 1) + vy) * (faces[0] + 1) + vx) * 3 + 2] = lower[2] + ((upper[2] - lower[2]) / faces[2]) * vz;
862       }
863     }
864   }
865   PetscCall(VecRestoreArray(coordinates, &coords));
866   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
867   PetscCall(VecDestroy(&coordinates));
868   PetscFunctionReturn(PETSC_SUCCESS);
869 }
870 
871 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], PetscBool interpolate)
872 {
873   PetscFunctionBegin;
874   PetscValidLogicalCollectiveInt(dm, dim, 2);
875   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
876   PetscCall(DMSetDimension(dm, dim - 1));
877   PetscCall(DMSetCoordinateDim(dm, dim));
878   switch (dim) {
879   case 1:
880     PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_1D_Internal(dm, lower, upper, faces));
881     break;
882   case 2:
883     PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_2D_Internal(dm, lower, upper, faces));
884     break;
885   case 3:
886     PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_3D_Internal(dm, lower, upper, faces));
887     break;
888   default:
889     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Dimension not supported: %" PetscInt_FMT, dim);
890   }
891   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
892   if (interpolate) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
893   PetscFunctionReturn(PETSC_SUCCESS);
894 }
895 
896 /*@C
897   DMPlexCreateBoxSurfaceMesh - Creates a mesh on the surface of the tensor product of unit intervals (box) using tensor cells (hexahedra).
898 
899   Collective
900 
901   Input Parameters:
902 + comm        - The communicator for the `DM` object
903 . dim         - The spatial dimension of the box, so the resulting mesh is has dimension `dim`-1
904 . faces       - Number of faces per dimension, or `NULL` for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
905 . lower       - The lower left corner, or `NULL` for (0, 0, 0)
906 . upper       - The upper right corner, or `NULL` for (1, 1, 1)
907 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
908 
909   Output Parameter:
910 . dm - The `DM` object
911 
912   Level: beginner
913 
914 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMSetFromOptions()`, `DMPlexCreateBoxMesh()`, `DMPlexCreateFromFile()`, `DMSetType()`, `DMCreate()`
915 @*/
916 PetscErrorCode DMPlexCreateBoxSurfaceMesh(MPI_Comm comm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], PetscBool interpolate, DM *dm)
917 {
918   PetscInt  fac[3] = {1, 1, 1};
919   PetscReal low[3] = {0, 0, 0};
920   PetscReal upp[3] = {1, 1, 1};
921 
922   PetscFunctionBegin;
923   PetscCall(DMCreate(comm, dm));
924   PetscCall(DMSetType(*dm, DMPLEX));
925   PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(*dm, dim, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, interpolate));
926   PetscFunctionReturn(PETSC_SUCCESS);
927 }
928 
929 static PetscErrorCode DMPlexCreateLineMesh_Internal(DM dm, PetscInt segments, PetscReal lower, PetscReal upper, DMBoundaryType bd)
930 {
931   PetscInt     i, fStart, fEnd, numCells = 0, numVerts = 0;
932   PetscInt     numPoints[2], *coneSize, *cones, *coneOrientations;
933   PetscScalar *vertexCoords;
934   PetscReal    L, maxCell;
935   PetscBool    markerSeparate = PETSC_FALSE;
936   PetscInt     markerLeft = 1, faceMarkerLeft = 1;
937   PetscInt     markerRight = 1, faceMarkerRight = 2;
938   PetscBool    wrap = (bd == DM_BOUNDARY_PERIODIC || bd == DM_BOUNDARY_TWIST) ? PETSC_TRUE : PETSC_FALSE;
939   PetscMPIInt  rank;
940 
941   PetscFunctionBegin;
942   PetscAssertPointer(dm, 1);
943 
944   PetscCall(DMSetDimension(dm, 1));
945   PetscCall(DMCreateLabel(dm, "marker"));
946   PetscCall(DMCreateLabel(dm, "Face Sets"));
947 
948   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
949   if (rank == 0) numCells = segments;
950   if (rank == 0) numVerts = segments + (wrap ? 0 : 1);
951 
952   numPoints[0] = numVerts;
953   numPoints[1] = numCells;
954   PetscCall(PetscMalloc4(numCells + numVerts, &coneSize, numCells * 2, &cones, numCells + numVerts, &coneOrientations, numVerts, &vertexCoords));
955   PetscCall(PetscArrayzero(coneOrientations, numCells + numVerts));
956   for (i = 0; i < numCells; ++i) coneSize[i] = 2;
957   for (i = 0; i < numVerts; ++i) coneSize[numCells + i] = 0;
958   for (i = 0; i < numCells; ++i) {
959     cones[2 * i]     = numCells + i % numVerts;
960     cones[2 * i + 1] = numCells + (i + 1) % numVerts;
961   }
962   for (i = 0; i < numVerts; ++i) vertexCoords[i] = lower + (upper - lower) * ((PetscReal)i / (PetscReal)numCells);
963   PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
964   PetscCall(PetscFree4(coneSize, cones, coneOrientations, vertexCoords));
965 
966   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
967   if (markerSeparate) {
968     markerLeft  = faceMarkerLeft;
969     markerRight = faceMarkerRight;
970   }
971   if (!wrap && rank == 0) {
972     PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
973     PetscCall(DMSetLabelValue(dm, "marker", fStart, markerLeft));
974     PetscCall(DMSetLabelValue(dm, "marker", fEnd - 1, markerRight));
975     PetscCall(DMSetLabelValue(dm, "Face Sets", fStart, faceMarkerLeft));
976     PetscCall(DMSetLabelValue(dm, "Face Sets", fEnd - 1, faceMarkerRight));
977   }
978   if (wrap) {
979     L       = upper - lower;
980     maxCell = (PetscReal)1.1 * (L / (PetscReal)PetscMax(1, segments));
981     PetscCall(DMSetPeriodicity(dm, &maxCell, &lower, &L));
982   }
983   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
984   PetscFunctionReturn(PETSC_SUCCESS);
985 }
986 
987 // Creates "Face Sets" label based on the standard box labeling conventions
988 static PetscErrorCode DMPlexSetBoxLabel_Internal(DM dm, const DMBoundaryType periodicity[])
989 {
990   DM              cdm;
991   PetscSection    csection;
992   Vec             coordinates;
993   DMLabel         label;
994   IS              faces_is;
995   PetscInt        dim, num_face = 0;
996   const PetscInt *faces;
997   PetscInt        faceMarkerBottom, faceMarkerTop, faceMarkerFront, faceMarkerBack, faceMarkerRight, faceMarkerLeft;
998 
999   PetscFunctionBeginUser;
1000   PetscCall(DMGetDimension(dm, &dim));
1001   PetscCheck((dim == 2) || (dim == 3), PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "DMPlex box labeling only supports 2D and 3D meshes, received DM of dimension %" PetscInt_FMT, dim);
1002   // Get Face Sets label
1003   PetscCall(DMGetLabel(dm, "Face Sets", &label));
1004   if (label) {
1005     PetscCall(DMLabelReset(label));
1006   } else {
1007     PetscCall(DMCreateLabel(dm, "Face Sets"));
1008     PetscCall(DMGetLabel(dm, "Face Sets", &label));
1009   }
1010   PetscCall(DMPlexMarkBoundaryFaces(dm, 1, label));
1011   PetscCall(DMGetStratumIS(dm, "Face Sets", 1, &faces_is));
1012 
1013   switch (dim) {
1014   case 2:
1015     faceMarkerTop    = 3;
1016     faceMarkerBottom = 1;
1017     faceMarkerRight  = 2;
1018     faceMarkerLeft   = 4;
1019     break;
1020   case 3:
1021     faceMarkerBottom = 1;
1022     faceMarkerTop    = 2;
1023     faceMarkerFront  = 3;
1024     faceMarkerBack   = 4;
1025     faceMarkerRight  = 5;
1026     faceMarkerLeft   = 6;
1027     break;
1028   default:
1029     SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Dimension %" PetscInt_FMT " not supported", dim);
1030   }
1031 
1032   if (faces_is) PetscCall(ISGetLocalSize(faces_is, &num_face));
1033   if (faces_is) PetscCall(ISGetIndices(faces_is, &faces));
1034   PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
1035   PetscCall(DMGetCoordinateDM(dm, &cdm));
1036   PetscCall(DMGetLocalSection(cdm, &csection));
1037   for (PetscInt f = 0; f < num_face; ++f) {
1038     PetscScalar *coords = NULL;
1039     PetscInt     face = faces[f], flip = 1, label_value = -1, coords_size;
1040 
1041     { // Determine if orientation of face is flipped
1042       PetscInt        num_cells_support, num_faces, start = -1;
1043       const PetscInt *orients, *cell_faces, *cells;
1044 
1045       PetscCall(DMPlexGetSupport(dm, face, &cells));
1046       PetscCall(DMPlexGetSupportSize(dm, face, &num_cells_support));
1047       PetscCheck(num_cells_support == 1, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "Expected one cell in support of exterior face, but got %" PetscInt_FMT " cells", num_cells_support);
1048       PetscCall(DMPlexGetCone(dm, cells[0], &cell_faces));
1049       PetscCall(DMPlexGetConeSize(dm, cells[0], &num_faces));
1050       for (PetscInt i = 0; i < num_faces; i++) {
1051         if (cell_faces[i] == face) start = i;
1052       }
1053       PetscCheck(start >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_CORRUPT, "Could not find face %" PetscInt_FMT " in cone of its support", face);
1054       PetscCall(DMPlexGetConeOrientation(dm, cells[0], &orients));
1055       if (orients[start] < 0) flip = -1;
1056     }
1057 
1058     // Cannot use DMPlexComputeCellGeometryFVM() for high-order geometry, so must calculate normal vectors manually
1059     // Use the vertices (depth 0) of coordinate DM to calculate normal vector
1060     PetscCall(DMPlexVecGetClosureAtDepth_Internal(cdm, csection, coordinates, face, 0, &coords_size, &coords));
1061     switch (dim) {
1062     case 2: {
1063       PetscScalar vec[2];
1064 
1065       for (PetscInt d = 0; d < dim; ++d) vec[d] = flip * (PetscRealPart(coords[1 * dim + d]) - PetscRealPart(coords[0 * dim + d]));
1066       PetscScalar normal[] = {vec[1], -vec[0]};
1067       if (PetscAbsScalar(normal[0]) > PetscAbsScalar(normal[1])) {
1068         label_value = PetscRealPart(normal[0]) > 0 ? faceMarkerRight : faceMarkerLeft;
1069       } else {
1070         label_value = PetscRealPart(normal[1]) > 0 ? faceMarkerTop : faceMarkerBottom;
1071       }
1072     } break;
1073     case 3: {
1074       PetscScalar vec1[3], vec2[3], normal[3];
1075 
1076       for (PetscInt d = 0; d < dim; ++d) {
1077         vec1[d] = PetscRealPart(coords[1 * dim + d]) - PetscRealPart(coords[0 * dim + d]);
1078         vec2[d] = PetscRealPart(coords[2 * dim + d]) - PetscRealPart(coords[1 * dim + d]);
1079       }
1080 
1081       // Calculate normal vector via cross-product
1082       normal[0] = flip * ((vec1[1] * vec2[2]) - (vec1[2] * vec2[1]));
1083       normal[1] = flip * ((vec1[2] * vec2[0]) - (vec1[0] * vec2[2]));
1084       normal[2] = flip * ((vec1[0] * vec2[1]) - (vec1[1] * vec2[0]));
1085 
1086       if (PetscAbsScalar(normal[0]) > PetscAbsScalar(normal[1])) {
1087         if (PetscAbsScalar(normal[0]) > PetscAbsScalar(normal[2])) {
1088           label_value = PetscRealPart(normal[0]) > 0 ? faceMarkerRight : faceMarkerLeft;
1089         } else {
1090           label_value = PetscRealPart(normal[2]) > 0 ? faceMarkerTop : faceMarkerBottom;
1091         }
1092       } else {
1093         if (PetscAbsScalar(normal[1]) > PetscAbsScalar(normal[2])) {
1094           label_value = PetscRealPart(normal[1]) > 0 ? faceMarkerBack : faceMarkerFront;
1095         } else {
1096           label_value = PetscRealPart(normal[2]) > 0 ? faceMarkerTop : faceMarkerBottom;
1097         }
1098       }
1099     } break;
1100     }
1101 
1102     PetscInt previous_label_value; // always 1 due to DMPlexMarkBoundaryFaces call above
1103     PetscCall(DMGetLabelValue(dm, "Face Sets", face, &previous_label_value));
1104     PetscCall(DMClearLabelValue(dm, "Face Sets", face, previous_label_value));
1105     PetscCall(DMSetLabelValue(dm, "Face Sets", face, label_value));
1106     PetscCall(DMPlexVecRestoreClosure(cdm, csection, coordinates, face, &coords_size, &coords));
1107   }
1108   if (faces_is) PetscCall(ISRestoreIndices(faces_is, &faces));
1109   PetscCall(ISDestroy(&faces_is));
1110 
1111   // Create Isoperiodic SF from newly-created face labels
1112   PetscSF     periodicsfs[3];
1113   PetscInt    periodic_sf_index  = 0;
1114   PetscScalar transform[3][4][4] = {{{0.}}};
1115   for (PetscInt d = 0; d < dim; d++) {
1116     IS              donor_is, periodic_is;
1117     const PetscInt *donor_faces = NULL, *periodic_faces = NULL;
1118     PetscInt        num_donor = 0, num_periodic = 0;
1119     PetscSF         centroidsf;
1120     PetscReal       donor_to_periodic_distance;
1121     const PetscInt  face_pairings[2][3][2] = {
1122       // 2D face pairings, {donor, periodic}
1123       {{4, 2}, {1, 3}},
1124       // 3D face pairings
1125       {{5, 6}, {3, 4}, {1, 2}}
1126     };
1127 
1128     if (periodicity[d] != DM_BOUNDARY_PERIODIC) continue;
1129     {
1130       // Compute centroidsf, which is the mapping from donor faces to periodic faces
1131       // Matches the centroid of the faces together, ignoring the periodic direction component (which should not match between donor and periodic face)
1132       PetscInt     coords_size, centroid_comps = dim - 1;
1133       PetscScalar *coords = NULL;
1134       PetscReal   *donor_centroids, *periodic_centroids;
1135       PetscReal    loc_periodic[2] = {PETSC_MIN_REAL, PETSC_MIN_REAL}, loc_periodic_global[2]; // Location of donor (0) and periodic (1) faces in periodic direction
1136 
1137       PetscCall(DMGetStratumIS(dm, "Face Sets", face_pairings[dim - 2][d][0], &donor_is));
1138       PetscCall(DMGetStratumIS(dm, "Face Sets", face_pairings[dim - 2][d][1], &periodic_is));
1139       if (donor_is) {
1140         PetscCall(ISGetLocalSize(donor_is, &num_donor));
1141         PetscCall(ISGetIndices(donor_is, &donor_faces));
1142       }
1143       if (periodic_is) {
1144         PetscCall(ISGetLocalSize(periodic_is, &num_periodic));
1145         PetscCall(ISGetIndices(periodic_is, &periodic_faces));
1146       }
1147       PetscCall(PetscCalloc2(num_donor * centroid_comps, &donor_centroids, num_periodic * centroid_comps, &periodic_centroids));
1148       for (PetscInt f = 0; f < num_donor; f++) {
1149         PetscInt face = donor_faces[f], num_coords;
1150         PetscCall(DMPlexVecGetClosureAtDepth_Internal(cdm, csection, coordinates, face, 0, &coords_size, &coords));
1151         num_coords = coords_size / dim;
1152         for (PetscInt c = 0; c < num_coords; c++) {
1153           PetscInt comp_index = 0;
1154           loc_periodic[0]     = PetscRealPart(coords[c * dim + d]);
1155           for (PetscInt i = 0; i < dim; i++) {
1156             if (i == d) continue; // Periodic direction not used for centroid calculation
1157             donor_centroids[f * centroid_comps + comp_index] += PetscRealPart(coords[c * dim + i]) / num_coords;
1158             comp_index++;
1159           }
1160         }
1161         PetscCall(DMPlexVecRestoreClosure(cdm, csection, coordinates, face, &coords_size, &coords));
1162       }
1163 
1164       for (PetscInt f = 0; f < num_periodic; f++) {
1165         PetscInt face = periodic_faces[f], num_coords;
1166         PetscCall(DMPlexVecGetClosureAtDepth_Internal(cdm, csection, coordinates, face, 0, &coords_size, &coords));
1167         num_coords = coords_size / dim;
1168         for (PetscInt c = 0; c < num_coords; c++) {
1169           PetscInt comp_index = 0;
1170           loc_periodic[1]     = PetscRealPart(coords[c * dim + d]);
1171           for (PetscInt i = 0; i < dim; i++) {
1172             if (i == d) continue; // Periodic direction not used for centroid calculation
1173             periodic_centroids[f * centroid_comps + comp_index] += PetscRealPart(coords[c * dim + i]) / num_coords;
1174             comp_index++;
1175           }
1176         }
1177         PetscCall(DMPlexVecRestoreClosure(cdm, csection, coordinates, face, &coords_size, &coords));
1178       }
1179       PetscCallMPI(MPIU_Allreduce(loc_periodic, loc_periodic_global, 2, MPIU_REAL, MPIU_MAX, PetscObjectComm((PetscObject)dm)));
1180       donor_to_periodic_distance = loc_periodic_global[1] - loc_periodic_global[0];
1181 
1182       PetscCall(PetscSFCreate(PetscObjectComm((PetscObject)dm), &centroidsf));
1183       PetscCall(PetscSFSetGraphFromCoordinates(centroidsf, num_donor, num_periodic, centroid_comps, 1e-10, donor_centroids, periodic_centroids));
1184       PetscCall(PetscSFViewFromOptions(centroidsf, NULL, "-dm_plex_box_label_centroid_sf_view"));
1185       PetscCall(PetscFree2(donor_centroids, periodic_centroids));
1186     }
1187 
1188     { // Create Isoperiodic SF using centroidsSF
1189       PetscInt           pStart, pEnd;
1190       PetscInt          *leaf_faces;
1191       const PetscSFNode *firemote;
1192       PetscSFNode       *isoperiodic_leaves;
1193 
1194       PetscCall(PetscMalloc1(num_periodic, &leaf_faces));
1195       PetscCall(PetscSFBcastBegin(centroidsf, MPIU_INT, donor_faces, leaf_faces, MPI_REPLACE));
1196       PetscCall(PetscSFBcastEnd(centroidsf, MPIU_INT, donor_faces, leaf_faces, MPI_REPLACE));
1197 
1198       PetscCall(PetscMalloc1(num_periodic, &isoperiodic_leaves));
1199       PetscCall(PetscSFGetGraph(centroidsf, NULL, NULL, NULL, &firemote));
1200       for (PetscInt l = 0; l < num_periodic; ++l) {
1201         isoperiodic_leaves[l].index = leaf_faces[l];
1202         isoperiodic_leaves[l].rank  = firemote[l].rank;
1203       }
1204 
1205       PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
1206       PetscCall(PetscSFCreate(PetscObjectComm((PetscObject)dm), &periodicsfs[periodic_sf_index]));
1207       PetscCall(PetscSFSetGraph(periodicsfs[periodic_sf_index], pEnd - pStart, num_periodic, (PetscInt *)periodic_faces, PETSC_COPY_VALUES, isoperiodic_leaves, PETSC_OWN_POINTER));
1208       PetscCall(PetscSFViewFromOptions(periodicsfs[periodic_sf_index], NULL, "-dm_plex_box_label_periodic_sf_view"));
1209       PetscCall(PetscFree(leaf_faces));
1210     }
1211 
1212     transform[periodic_sf_index][0][0] = 1;
1213     transform[periodic_sf_index][1][1] = 1;
1214     transform[periodic_sf_index][2][2] = 1;
1215     transform[periodic_sf_index][3][3] = 1;
1216     transform[periodic_sf_index][d][3] = donor_to_periodic_distance;
1217 
1218     periodic_sf_index++;
1219     PetscCall(PetscSFDestroy(&centroidsf));
1220     if (donor_is) {
1221       PetscCall(ISRestoreIndices(donor_is, &donor_faces));
1222       PetscCall(ISDestroy(&donor_is));
1223     }
1224     if (periodic_is) {
1225       PetscCall(ISRestoreIndices(periodic_is, &periodic_faces));
1226       PetscCall(ISDestroy(&periodic_is));
1227     }
1228     PetscCall(DMClearLabelStratum(dm, "Face Sets", face_pairings[dim - 2][d][0]));
1229     PetscCall(DMClearLabelStratum(dm, "Face Sets", face_pairings[dim - 2][d][1]));
1230   }
1231   PetscCall(DMPlexSetIsoperiodicFaceSF(dm, periodic_sf_index, periodicsfs));
1232   PetscCall(DMPlexSetIsoperiodicFaceTransform(dm, periodic_sf_index, (const PetscScalar *)transform));
1233   for (PetscInt p = 0; p < periodic_sf_index; p++) PetscCall(PetscSFDestroy(&periodicsfs[p]));
1234 
1235   { // Update coordinate DM with new Face Sets label
1236     DM      cdm;
1237     DMLabel oldFaceSets, newFaceSets;
1238     PetscCall(DMGetCoordinateDM(dm, &cdm));
1239     PetscCall(DMGetLabel(cdm, "Face Sets", &oldFaceSets));
1240     if (oldFaceSets) PetscCall(DMRemoveLabelBySelf(cdm, &oldFaceSets, PETSC_FALSE));
1241     PetscCall(DMLabelDuplicate(label, &newFaceSets));
1242     PetscCall(DMAddLabel(cdm, newFaceSets));
1243     PetscCall(DMLabelDestroy(&newFaceSets));
1244   }
1245   PetscFunctionReturn(PETSC_SUCCESS);
1246 }
1247 
1248 static PetscErrorCode DMPlexCreateSquareMesh_Simplex_CrissCross(DM dm, const PetscInt edges[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType bd[])
1249 {
1250   PetscInt       markerTop = 1, faceMarkerTop = 3;
1251   PetscInt       markerBottom = 1, faceMarkerBottom = 1;
1252   PetscInt       markerRight = 1, faceMarkerRight = 2;
1253   PetscInt       markerLeft = 1, faceMarkerLeft = 4;
1254   PetscBool      markerSeparate = PETSC_FALSE;
1255   DMBoundaryType bdX = bd[0], bdY = bd[1];
1256   PetscMPIInt    rank;
1257 
1258   PetscFunctionBegin;
1259   PetscCheck(bdX == DM_BOUNDARY_NONE || bdX == DM_BOUNDARY_PERIODIC, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Not implemented for boundary type %s", DMBoundaryTypes[bdX]);
1260   PetscCheck(bdY == DM_BOUNDARY_NONE || bdY == DM_BOUNDARY_PERIODIC, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Not implemented for boundary type %s", DMBoundaryTypes[bdY]);
1261   PetscCall(DMSetDimension(dm, 2));
1262   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
1263   PetscCall(DMCreateLabel(dm, "marker"));
1264   PetscCall(DMCreateLabel(dm, "Face Sets"));
1265   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
1266   if (markerSeparate) {
1267     markerBottom = faceMarkerBottom;
1268     markerTop    = faceMarkerTop;
1269     markerRight  = faceMarkerRight;
1270     markerLeft   = faceMarkerLeft;
1271   }
1272   {
1273     const PetscInt numXEdges    = rank == 0 ? edges[0] : 0;
1274     const PetscInt numYEdges    = rank == 0 ? edges[1] : 0;
1275     const PetscInt numZEdges    = rank == 0 ? 4 * edges[0] * edges[1] : 0; /* Z-edges are the 4 internal edges per cell */
1276     const PetscInt numXVertices = rank == 0 ? (bdX == DM_BOUNDARY_PERIODIC ? edges[0] : edges[0] + 1) : 0;
1277     const PetscInt numYVertices = rank == 0 ? (bdY == DM_BOUNDARY_PERIODIC ? edges[1] : edges[1] + 1) : 0;
1278     const PetscInt numZVertices = rank == 0 ? edges[0] * edges[1] : 0;
1279     const PetscInt numCells     = 4 * numXEdges * numYEdges;
1280     const PetscInt numTotXEdges = numXEdges * numYVertices;
1281     const PetscInt numTotYEdges = numYEdges * numXVertices;
1282     const PetscInt numVertices  = numXVertices * numYVertices + numZVertices;
1283     const PetscInt numEdges     = numTotXEdges + numTotYEdges + numZEdges;
1284     const PetscInt firstVertex  = numCells;
1285     const PetscInt firstXEdge   = numCells + numVertices;
1286     const PetscInt firstYEdge   = firstXEdge + numTotXEdges;
1287     const PetscInt firstZEdge   = firstYEdge + numTotYEdges;
1288     Vec            coordinates;
1289     PetscSection   coordSection;
1290     PetscScalar   *coords;
1291     PetscInt       coordSize;
1292     PetscInt       v, vx, vy;
1293     PetscInt       c, e, ex, ey;
1294 
1295     PetscCall(DMPlexSetChart(dm, 0, numCells + numEdges + numVertices));
1296     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 3));
1297     for (e = firstXEdge; e < firstXEdge + numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
1298     PetscCall(DMSetUp(dm));
1299 
1300     /* Build cells and Z-edges */
1301     for (ey = 0; ey < numYEdges; ++ey) {
1302       for (ex = 0; ex < numXEdges; ++ex) {
1303         const PetscInt exp1 = (ex + 1) % numXVertices;
1304         const PetscInt eyp1 = (ey + 1) % numYVertices;
1305         const PetscInt ez   = firstZEdge + 4 * (ey * numXEdges + ex);
1306         const PetscInt vc   = firstVertex + numXVertices * numYVertices + ey * numXEdges + ex;
1307         const PetscInt v0   = firstVertex + ey * numXVertices + ex;
1308         const PetscInt v1   = firstVertex + ey * numXVertices + exp1;
1309         const PetscInt v2   = firstVertex + eyp1 * numXVertices + exp1;
1310         const PetscInt v3   = firstVertex + eyp1 * numXVertices + ex;
1311         const PetscInt e0   = firstXEdge + ey * numXEdges + ex;
1312         const PetscInt e1   = firstYEdge + exp1 * numYEdges + ey;
1313         const PetscInt e2   = firstXEdge + eyp1 * numXEdges + ex;
1314         const PetscInt e3   = firstYEdge + ex * numYEdges + ey;
1315 
1316         const PetscInt cones[] = {ez, e0, ez + 1, ez + 1, e1, ez + 2, ez + 2, e2, ez + 3, ez + 3, e3, ez};
1317         const PetscInt ornts[] = {-1, 0, 0, -1, 0, 0, -1, -1, 0, -1, -1, 0};
1318         const PetscInt verts[] = {v0, vc, v1, vc, v2, vc, v3, vc};
1319 
1320         for (c = 0; c < 4; c++) {
1321           PetscInt cell = 4 * (ey * numXEdges + ex) + c;
1322           PetscInt edge = ez + c;
1323 
1324           PetscCall(DMPlexSetCone(dm, cell, cones + 3 * c));
1325           PetscCall(DMPlexSetConeOrientation(dm, cell, ornts + 3 * c));
1326           PetscCall(DMPlexSetCone(dm, edge, verts + 2 * c));
1327         }
1328       }
1329     }
1330 
1331     /* Build Y edges*/
1332     for (vx = 0; vx < numXVertices; vx++) {
1333       for (ey = 0; ey < numYEdges; ey++) {
1334         const PetscInt edge   = firstYEdge + vx * numYEdges + ey;
1335         const PetscInt v0     = firstVertex + ey * numXVertices + vx;
1336         const PetscInt v1     = firstVertex + ((ey + 1) % numYVertices) * numXVertices + vx;
1337         const PetscInt cone[] = {v0, v1};
1338 
1339         PetscCall(DMPlexSetCone(dm, edge, cone));
1340         if ((bdX != DM_BOUNDARY_PERIODIC) && (bdX != DM_BOUNDARY_TWIST)) {
1341           if (vx == numXVertices - 1) {
1342             PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerRight));
1343             PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1344             PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1345             if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1346           } else if (vx == 0) {
1347             PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerLeft));
1348             PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1349             PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1350             if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1351           }
1352         }
1353       }
1354     }
1355 
1356     /* Build X edges*/
1357     for (vy = 0; vy < numYVertices; vy++) {
1358       for (ex = 0; ex < numXEdges; ex++) {
1359         const PetscInt edge   = firstXEdge + vy * numXEdges + ex;
1360         const PetscInt v0     = firstVertex + vy * numXVertices + ex;
1361         const PetscInt v1     = firstVertex + vy * numXVertices + (ex + 1) % numXVertices;
1362         const PetscInt cone[] = {v0, v1};
1363 
1364         PetscCall(DMPlexSetCone(dm, edge, cone));
1365         if ((bdY != DM_BOUNDARY_PERIODIC) && (bdY != DM_BOUNDARY_TWIST)) {
1366           if (vy == numYVertices - 1) {
1367             PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerTop));
1368             PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1369             PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1370             if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1371           } else if (vy == 0) {
1372             PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerBottom));
1373             PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1374             PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1375             if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1376           }
1377         }
1378       }
1379     }
1380 
1381     /* Compute support, stratify, and celltype label */
1382     PetscCall(DMPlexSymmetrize(dm));
1383     PetscCall(DMPlexStratify(dm));
1384     PetscCall(DMPlexComputeCellTypes(dm));
1385 
1386     /* Build coordinates */
1387     PetscCall(DMGetCoordinateSection(dm, &coordSection));
1388     PetscCall(PetscSectionSetNumFields(coordSection, 1));
1389     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, 2));
1390     PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numVertices));
1391     for (v = firstVertex; v < firstVertex + numVertices; ++v) {
1392       PetscCall(PetscSectionSetDof(coordSection, v, 2));
1393       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, 2));
1394     }
1395     PetscCall(PetscSectionSetUp(coordSection));
1396     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
1397     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
1398     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
1399     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
1400     PetscCall(VecSetBlockSize(coordinates, 2));
1401     PetscCall(VecSetType(coordinates, VECSTANDARD));
1402     PetscCall(VecGetArray(coordinates, &coords));
1403     for (vy = 0; vy < numYVertices; ++vy) {
1404       for (vx = 0; vx < numXVertices; ++vx) {
1405         coords[2 * (vy * numXVertices + vx) + 0] = lower[0] + ((upper[0] - lower[0]) / numXEdges) * vx;
1406         coords[2 * (vy * numXVertices + vx) + 1] = lower[1] + ((upper[1] - lower[1]) / numYEdges) * vy;
1407       }
1408     }
1409     for (ey = 0; ey < numYEdges; ++ey) {
1410       for (ex = 0; ex < numXEdges; ++ex) {
1411         const PetscInt c = ey * numXEdges + ex + numYVertices * numXVertices;
1412 
1413         coords[2 * c + 0] = lower[0] + ((upper[0] - lower[0]) / numXEdges) * (ex + 0.5);
1414         coords[2 * c + 1] = lower[1] + ((upper[1] - lower[1]) / numYEdges) * (ey + 0.5);
1415       }
1416     }
1417     PetscCall(VecRestoreArray(coordinates, &coords));
1418     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
1419     PetscCall(VecDestroy(&coordinates));
1420 
1421     /* handle periodic BC */
1422     if (bdX == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_PERIODIC) {
1423       PetscReal L[2]       = {-1., -1.};
1424       PetscReal maxCell[2] = {-1., -1.};
1425 
1426       for (PetscInt d = 0; d < 2; ++d) {
1427         if (bd[d] != DM_BOUNDARY_NONE) {
1428           L[d]       = upper[d] - lower[d];
1429           maxCell[d] = 1.1 * (L[d] / PetscMax(1, edges[d]));
1430         }
1431       }
1432       PetscCall(DMSetPeriodicity(dm, maxCell, lower, L));
1433     }
1434   }
1435   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
1436   PetscFunctionReturn(PETSC_SUCCESS);
1437 }
1438 
1439 static PetscErrorCode DMPlexCreateBoxMesh_Simplex_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate)
1440 {
1441   DM        boundary, vol;
1442   DMLabel   bdlabel;
1443   PetscBool crisscross = PETSC_FALSE;
1444 
1445   PetscFunctionBegin;
1446   PetscAssertPointer(dm, 1);
1447   if (dim == 2) PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_box_crisscross", &crisscross, NULL));
1448   if (crisscross) {
1449     PetscCall(DMPlexCreateSquareMesh_Simplex_CrissCross(dm, faces, lower, upper, periodicity));
1450   } else {
1451     for (PetscInt i = 0; i < dim; ++i) PetscCheck(periodicity[i] == DM_BOUNDARY_NONE, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Periodicity is not supported for simplex meshes");
1452     PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &boundary));
1453     PetscCall(DMSetType(boundary, DMPLEX));
1454     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)boundary, ((PetscObject)dm)->prefix));
1455     PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(boundary, dim, faces, lower, upper, PETSC_FALSE));
1456     PetscCall(DMPlexGenerate(boundary, NULL, interpolate, &vol));
1457     PetscCall(DMGetLabel(vol, "marker", &bdlabel));
1458     if (bdlabel) PetscCall(DMPlexLabelComplete(vol, bdlabel));
1459     PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_FALSE, vol));
1460     PetscCall(DMPlexReplace_Internal(dm, &vol));
1461     PetscCall(DMDestroy(&boundary));
1462   }
1463   if (interpolate) {
1464     PetscCall(DMPlexInterpolateInPlace_Internal(dm));
1465     PetscCall(DMPlexSetBoxLabel_Internal(dm, periodicity));
1466   }
1467   PetscFunctionReturn(PETSC_SUCCESS);
1468 }
1469 
1470 static PetscErrorCode DMPlexCreateCubeMesh_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[], DMBoundaryType bdX, DMBoundaryType bdY, DMBoundaryType bdZ)
1471 {
1472   DMLabel     cutLabel  = NULL;
1473   PetscInt    markerTop = 1, faceMarkerTop = 1;
1474   PetscInt    markerBottom = 1, faceMarkerBottom = 1;
1475   PetscInt    markerFront = 1, faceMarkerFront = 1;
1476   PetscInt    markerBack = 1, faceMarkerBack = 1;
1477   PetscInt    markerRight = 1, faceMarkerRight = 1;
1478   PetscInt    markerLeft = 1, faceMarkerLeft = 1;
1479   PetscInt    dim;
1480   PetscBool   markerSeparate = PETSC_FALSE, cutMarker = PETSC_FALSE;
1481   PetscMPIInt rank;
1482 
1483   PetscFunctionBegin;
1484   PetscCall(DMGetDimension(dm, &dim));
1485   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
1486   PetscCall(DMCreateLabel(dm, "marker"));
1487   PetscCall(DMCreateLabel(dm, "Face Sets"));
1488   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_periodic_cut", &cutMarker, NULL));
1489   if (bdX == DM_BOUNDARY_PERIODIC || bdX == DM_BOUNDARY_TWIST || bdY == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_TWIST || bdZ == DM_BOUNDARY_PERIODIC || bdZ == DM_BOUNDARY_TWIST) {
1490     if (cutMarker) {
1491       PetscCall(DMCreateLabel(dm, "periodic_cut"));
1492       PetscCall(DMGetLabel(dm, "periodic_cut", &cutLabel));
1493     }
1494   }
1495   switch (dim) {
1496   case 2:
1497     faceMarkerTop    = 3;
1498     faceMarkerBottom = 1;
1499     faceMarkerRight  = 2;
1500     faceMarkerLeft   = 4;
1501     break;
1502   case 3:
1503     faceMarkerBottom = 1;
1504     faceMarkerTop    = 2;
1505     faceMarkerFront  = 3;
1506     faceMarkerBack   = 4;
1507     faceMarkerRight  = 5;
1508     faceMarkerLeft   = 6;
1509     break;
1510   default:
1511     SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Dimension %" PetscInt_FMT " not supported", dim);
1512   }
1513   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
1514   if (markerSeparate) {
1515     markerBottom = faceMarkerBottom;
1516     markerTop    = faceMarkerTop;
1517     markerFront  = faceMarkerFront;
1518     markerBack   = faceMarkerBack;
1519     markerRight  = faceMarkerRight;
1520     markerLeft   = faceMarkerLeft;
1521   }
1522   {
1523     const PetscInt numXEdges    = rank == 0 ? edges[0] : 0;
1524     const PetscInt numYEdges    = rank == 0 ? edges[1] : 0;
1525     const PetscInt numZEdges    = rank == 0 ? edges[2] : 0;
1526     const PetscInt numXVertices = rank == 0 ? (bdX == DM_BOUNDARY_PERIODIC || bdX == DM_BOUNDARY_TWIST ? edges[0] : edges[0] + 1) : 0;
1527     const PetscInt numYVertices = rank == 0 ? (bdY == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_TWIST ? edges[1] : edges[1] + 1) : 0;
1528     const PetscInt numZVertices = rank == 0 ? (bdZ == DM_BOUNDARY_PERIODIC || bdZ == DM_BOUNDARY_TWIST ? edges[2] : edges[2] + 1) : 0;
1529     const PetscInt numCells     = numXEdges * numYEdges * numZEdges;
1530     const PetscInt numXFaces    = numYEdges * numZEdges;
1531     const PetscInt numYFaces    = numXEdges * numZEdges;
1532     const PetscInt numZFaces    = numXEdges * numYEdges;
1533     const PetscInt numTotXFaces = numXVertices * numXFaces;
1534     const PetscInt numTotYFaces = numYVertices * numYFaces;
1535     const PetscInt numTotZFaces = numZVertices * numZFaces;
1536     const PetscInt numFaces     = numTotXFaces + numTotYFaces + numTotZFaces;
1537     const PetscInt numTotXEdges = numXEdges * numYVertices * numZVertices;
1538     const PetscInt numTotYEdges = numYEdges * numXVertices * numZVertices;
1539     const PetscInt numTotZEdges = numZEdges * numXVertices * numYVertices;
1540     const PetscInt numVertices  = numXVertices * numYVertices * numZVertices;
1541     const PetscInt numEdges     = numTotXEdges + numTotYEdges + numTotZEdges;
1542     const PetscInt firstVertex  = (dim == 2) ? numFaces : numCells;
1543     const PetscInt firstXFace   = (dim == 2) ? 0 : numCells + numVertices;
1544     const PetscInt firstYFace   = firstXFace + numTotXFaces;
1545     const PetscInt firstZFace   = firstYFace + numTotYFaces;
1546     const PetscInt firstXEdge   = numCells + numFaces + numVertices;
1547     const PetscInt firstYEdge   = firstXEdge + numTotXEdges;
1548     const PetscInt firstZEdge   = firstYEdge + numTotYEdges;
1549     Vec            coordinates;
1550     PetscSection   coordSection;
1551     PetscScalar   *coords;
1552     PetscInt       coordSize;
1553     PetscInt       v, vx, vy, vz;
1554     PetscInt       c, f, fx, fy, fz, e, ex, ey, ez;
1555 
1556     PetscCall(DMPlexSetChart(dm, 0, numCells + numFaces + numEdges + numVertices));
1557     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 6));
1558     for (f = firstXFace; f < firstXFace + numFaces; ++f) PetscCall(DMPlexSetConeSize(dm, f, 4));
1559     for (e = firstXEdge; e < firstXEdge + numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
1560     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
1561     /* Build cells */
1562     for (fz = 0; fz < numZEdges; ++fz) {
1563       for (fy = 0; fy < numYEdges; ++fy) {
1564         for (fx = 0; fx < numXEdges; ++fx) {
1565           PetscInt cell  = (fz * numYEdges + fy) * numXEdges + fx;
1566           PetscInt faceB = firstZFace + (fy * numXEdges + fx) * numZVertices + fz;
1567           PetscInt faceT = firstZFace + (fy * numXEdges + fx) * numZVertices + ((fz + 1) % numZVertices);
1568           PetscInt faceF = firstYFace + (fz * numXEdges + fx) * numYVertices + fy;
1569           PetscInt faceK = firstYFace + (fz * numXEdges + fx) * numYVertices + ((fy + 1) % numYVertices);
1570           PetscInt faceL = firstXFace + (fz * numYEdges + fy) * numXVertices + fx;
1571           PetscInt faceR = firstXFace + (fz * numYEdges + fy) * numXVertices + ((fx + 1) % numXVertices);
1572           /* B,  T,  F,  K,  R,  L */
1573           PetscInt ornt[6] = {-2, 0, 0, -3, 0, -2}; /* ??? */
1574           PetscInt cone[6];
1575 
1576           /* no boundary twisting in 3D */
1577           cone[0] = faceB;
1578           cone[1] = faceT;
1579           cone[2] = faceF;
1580           cone[3] = faceK;
1581           cone[4] = faceR;
1582           cone[5] = faceL;
1583           PetscCall(DMPlexSetCone(dm, cell, cone));
1584           PetscCall(DMPlexSetConeOrientation(dm, cell, ornt));
1585           if (bdX != DM_BOUNDARY_NONE && fx == numXEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
1586           if (bdY != DM_BOUNDARY_NONE && fy == numYEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
1587           if (bdZ != DM_BOUNDARY_NONE && fz == numZEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
1588         }
1589       }
1590     }
1591     /* Build x faces */
1592     for (fz = 0; fz < numZEdges; ++fz) {
1593       for (fy = 0; fy < numYEdges; ++fy) {
1594         for (fx = 0; fx < numXVertices; ++fx) {
1595           PetscInt face    = firstXFace + (fz * numYEdges + fy) * numXVertices + fx;
1596           PetscInt edgeL   = firstZEdge + (fy * numXVertices + fx) * numZEdges + fz;
1597           PetscInt edgeR   = firstZEdge + (((fy + 1) % numYVertices) * numXVertices + fx) * numZEdges + fz;
1598           PetscInt edgeB   = firstYEdge + (fz * numXVertices + fx) * numYEdges + fy;
1599           PetscInt edgeT   = firstYEdge + (((fz + 1) % numZVertices) * numXVertices + fx) * numYEdges + fy;
1600           PetscInt ornt[4] = {0, 0, -1, -1};
1601           PetscInt cone[4];
1602 
1603           if (dim == 3) {
1604             /* markers */
1605             if (bdX != DM_BOUNDARY_PERIODIC) {
1606               if (fx == numXVertices - 1) {
1607                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerRight));
1608                 PetscCall(DMSetLabelValue(dm, "marker", face, markerRight));
1609               } else if (fx == 0) {
1610                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerLeft));
1611                 PetscCall(DMSetLabelValue(dm, "marker", face, markerLeft));
1612               }
1613             }
1614           }
1615           cone[0] = edgeB;
1616           cone[1] = edgeR;
1617           cone[2] = edgeT;
1618           cone[3] = edgeL;
1619           PetscCall(DMPlexSetCone(dm, face, cone));
1620           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
1621         }
1622       }
1623     }
1624     /* Build y faces */
1625     for (fz = 0; fz < numZEdges; ++fz) {
1626       for (fx = 0; fx < numXEdges; ++fx) {
1627         for (fy = 0; fy < numYVertices; ++fy) {
1628           PetscInt face    = firstYFace + (fz * numXEdges + fx) * numYVertices + fy;
1629           PetscInt edgeL   = firstZEdge + (fy * numXVertices + fx) * numZEdges + fz;
1630           PetscInt edgeR   = firstZEdge + (fy * numXVertices + ((fx + 1) % numXVertices)) * numZEdges + fz;
1631           PetscInt edgeB   = firstXEdge + (fz * numYVertices + fy) * numXEdges + fx;
1632           PetscInt edgeT   = firstXEdge + (((fz + 1) % numZVertices) * numYVertices + fy) * numXEdges + fx;
1633           PetscInt ornt[4] = {0, 0, -1, -1};
1634           PetscInt cone[4];
1635 
1636           if (dim == 3) {
1637             /* markers */
1638             if (bdY != DM_BOUNDARY_PERIODIC) {
1639               if (fy == numYVertices - 1) {
1640                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerBack));
1641                 PetscCall(DMSetLabelValue(dm, "marker", face, markerBack));
1642               } else if (fy == 0) {
1643                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerFront));
1644                 PetscCall(DMSetLabelValue(dm, "marker", face, markerFront));
1645               }
1646             }
1647           }
1648           cone[0] = edgeB;
1649           cone[1] = edgeR;
1650           cone[2] = edgeT;
1651           cone[3] = edgeL;
1652           PetscCall(DMPlexSetCone(dm, face, cone));
1653           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
1654         }
1655       }
1656     }
1657     /* Build z faces */
1658     for (fy = 0; fy < numYEdges; ++fy) {
1659       for (fx = 0; fx < numXEdges; ++fx) {
1660         for (fz = 0; fz < numZVertices; fz++) {
1661           PetscInt face    = firstZFace + (fy * numXEdges + fx) * numZVertices + fz;
1662           PetscInt edgeL   = firstYEdge + (fz * numXVertices + fx) * numYEdges + fy;
1663           PetscInt edgeR   = firstYEdge + (fz * numXVertices + ((fx + 1) % numXVertices)) * numYEdges + fy;
1664           PetscInt edgeB   = firstXEdge + (fz * numYVertices + fy) * numXEdges + fx;
1665           PetscInt edgeT   = firstXEdge + (fz * numYVertices + ((fy + 1) % numYVertices)) * numXEdges + fx;
1666           PetscInt ornt[4] = {0, 0, -1, -1};
1667           PetscInt cone[4];
1668 
1669           if (dim == 2) {
1670             if (bdX == DM_BOUNDARY_TWIST && fx == numXEdges - 1) {
1671               edgeR += numYEdges - 1 - 2 * fy;
1672               ornt[1] = -1;
1673             }
1674             if (bdY == DM_BOUNDARY_TWIST && fy == numYEdges - 1) {
1675               edgeT += numXEdges - 1 - 2 * fx;
1676               ornt[2] = 0;
1677             }
1678             if (bdX != DM_BOUNDARY_NONE && fx == numXEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, face, 2));
1679             if (bdY != DM_BOUNDARY_NONE && fy == numYEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, face, 2));
1680           } else {
1681             /* markers */
1682             if (bdZ != DM_BOUNDARY_PERIODIC) {
1683               if (fz == numZVertices - 1) {
1684                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerTop));
1685                 PetscCall(DMSetLabelValue(dm, "marker", face, markerTop));
1686               } else if (fz == 0) {
1687                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerBottom));
1688                 PetscCall(DMSetLabelValue(dm, "marker", face, markerBottom));
1689               }
1690             }
1691           }
1692           cone[0] = edgeB;
1693           cone[1] = edgeR;
1694           cone[2] = edgeT;
1695           cone[3] = edgeL;
1696           PetscCall(DMPlexSetCone(dm, face, cone));
1697           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
1698         }
1699       }
1700     }
1701     /* Build Z edges*/
1702     for (vy = 0; vy < numYVertices; vy++) {
1703       for (vx = 0; vx < numXVertices; vx++) {
1704         for (ez = 0; ez < numZEdges; ez++) {
1705           const PetscInt edge    = firstZEdge + (vy * numXVertices + vx) * numZEdges + ez;
1706           const PetscInt vertexB = firstVertex + (ez * numYVertices + vy) * numXVertices + vx;
1707           const PetscInt vertexT = firstVertex + (((ez + 1) % numZVertices) * numYVertices + vy) * numXVertices + vx;
1708           PetscInt       cone[2];
1709 
1710           cone[0] = vertexB;
1711           cone[1] = vertexT;
1712           PetscCall(DMPlexSetCone(dm, edge, cone));
1713           if (dim == 3) {
1714             if (bdX != DM_BOUNDARY_PERIODIC) {
1715               if (vx == numXVertices - 1) {
1716                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1717                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1718                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1719               } else if (vx == 0) {
1720                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1721                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1722                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1723               }
1724             }
1725             if (bdY != DM_BOUNDARY_PERIODIC) {
1726               if (vy == numYVertices - 1) {
1727                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBack));
1728                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBack));
1729                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBack));
1730               } else if (vy == 0) {
1731                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerFront));
1732                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerFront));
1733                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerFront));
1734               }
1735             }
1736           }
1737         }
1738       }
1739     }
1740     /* Build Y edges*/
1741     for (vz = 0; vz < numZVertices; vz++) {
1742       for (vx = 0; vx < numXVertices; vx++) {
1743         for (ey = 0; ey < numYEdges; ey++) {
1744           const PetscInt nextv   = (dim == 2 && bdY == DM_BOUNDARY_TWIST && ey == numYEdges - 1) ? (numXVertices - vx - 1) : (vz * numYVertices + ((ey + 1) % numYVertices)) * numXVertices + vx;
1745           const PetscInt edge    = firstYEdge + (vz * numXVertices + vx) * numYEdges + ey;
1746           const PetscInt vertexF = firstVertex + (vz * numYVertices + ey) * numXVertices + vx;
1747           const PetscInt vertexK = firstVertex + nextv;
1748           PetscInt       cone[2];
1749 
1750           cone[0] = vertexF;
1751           cone[1] = vertexK;
1752           PetscCall(DMPlexSetCone(dm, edge, cone));
1753           if (dim == 2) {
1754             if ((bdX != DM_BOUNDARY_PERIODIC) && (bdX != DM_BOUNDARY_TWIST)) {
1755               if (vx == numXVertices - 1) {
1756                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerRight));
1757                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1758                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1759                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1760               } else if (vx == 0) {
1761                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerLeft));
1762                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1763                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1764                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1765               }
1766             } else {
1767               if (vx == 0 && cutLabel) {
1768                 PetscCall(DMLabelSetValue(cutLabel, edge, 1));
1769                 PetscCall(DMLabelSetValue(cutLabel, cone[0], 1));
1770                 if (ey == numYEdges - 1) PetscCall(DMLabelSetValue(cutLabel, cone[1], 1));
1771               }
1772             }
1773           } else {
1774             if (bdX != DM_BOUNDARY_PERIODIC) {
1775               if (vx == numXVertices - 1) {
1776                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1777                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1778                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1779               } else if (vx == 0) {
1780                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1781                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1782                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1783               }
1784             }
1785             if (bdZ != DM_BOUNDARY_PERIODIC) {
1786               if (vz == numZVertices - 1) {
1787                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1788                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1789                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1790               } else if (vz == 0) {
1791                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1792                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1793                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1794               }
1795             }
1796           }
1797         }
1798       }
1799     }
1800     /* Build X edges*/
1801     for (vz = 0; vz < numZVertices; vz++) {
1802       for (vy = 0; vy < numYVertices; vy++) {
1803         for (ex = 0; ex < numXEdges; ex++) {
1804           const PetscInt nextv   = (dim == 2 && bdX == DM_BOUNDARY_TWIST && ex == numXEdges - 1) ? (numYVertices - vy - 1) * numXVertices : (vz * numYVertices + vy) * numXVertices + (ex + 1) % numXVertices;
1805           const PetscInt edge    = firstXEdge + (vz * numYVertices + vy) * numXEdges + ex;
1806           const PetscInt vertexL = firstVertex + (vz * numYVertices + vy) * numXVertices + ex;
1807           const PetscInt vertexR = firstVertex + nextv;
1808           PetscInt       cone[2];
1809 
1810           cone[0] = vertexL;
1811           cone[1] = vertexR;
1812           PetscCall(DMPlexSetCone(dm, edge, cone));
1813           if (dim == 2) {
1814             if ((bdY != DM_BOUNDARY_PERIODIC) && (bdY != DM_BOUNDARY_TWIST)) {
1815               if (vy == numYVertices - 1) {
1816                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerTop));
1817                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1818                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1819                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1820               } else if (vy == 0) {
1821                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerBottom));
1822                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1823                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1824                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1825               }
1826             } else {
1827               if (vy == 0 && cutLabel) {
1828                 PetscCall(DMLabelSetValue(cutLabel, edge, 1));
1829                 PetscCall(DMLabelSetValue(cutLabel, cone[0], 1));
1830                 if (ex == numXEdges - 1) PetscCall(DMLabelSetValue(cutLabel, cone[1], 1));
1831               }
1832             }
1833           } else {
1834             if (bdY != DM_BOUNDARY_PERIODIC) {
1835               if (vy == numYVertices - 1) {
1836                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBack));
1837                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBack));
1838                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBack));
1839               } else if (vy == 0) {
1840                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerFront));
1841                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerFront));
1842                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerFront));
1843               }
1844             }
1845             if (bdZ != DM_BOUNDARY_PERIODIC) {
1846               if (vz == numZVertices - 1) {
1847                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1848                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1849                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1850               } else if (vz == 0) {
1851                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1852                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1853                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1854               }
1855             }
1856           }
1857         }
1858       }
1859     }
1860     PetscCall(DMPlexSymmetrize(dm));
1861     PetscCall(DMPlexStratify(dm));
1862     /* Build coordinates */
1863     PetscCall(DMGetCoordinateSection(dm, &coordSection));
1864     PetscCall(PetscSectionSetNumFields(coordSection, 1));
1865     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
1866     PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numVertices));
1867     for (v = firstVertex; v < firstVertex + numVertices; ++v) {
1868       PetscCall(PetscSectionSetDof(coordSection, v, dim));
1869       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
1870     }
1871     PetscCall(PetscSectionSetUp(coordSection));
1872     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
1873     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
1874     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
1875     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
1876     PetscCall(VecSetBlockSize(coordinates, dim));
1877     PetscCall(VecSetType(coordinates, VECSTANDARD));
1878     PetscCall(VecGetArray(coordinates, &coords));
1879     for (vz = 0; vz < numZVertices; ++vz) {
1880       for (vy = 0; vy < numYVertices; ++vy) {
1881         for (vx = 0; vx < numXVertices; ++vx) {
1882           coords[((vz * numYVertices + vy) * numXVertices + vx) * dim + 0] = lower[0] + ((upper[0] - lower[0]) / numXEdges) * vx;
1883           coords[((vz * numYVertices + vy) * numXVertices + vx) * dim + 1] = lower[1] + ((upper[1] - lower[1]) / numYEdges) * vy;
1884           if (dim == 3) coords[((vz * numYVertices + vy) * numXVertices + vx) * dim + 2] = lower[2] + ((upper[2] - lower[2]) / numZEdges) * vz;
1885         }
1886       }
1887     }
1888     PetscCall(VecRestoreArray(coordinates, &coords));
1889     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
1890     PetscCall(VecDestroy(&coordinates));
1891   }
1892   PetscFunctionReturn(PETSC_SUCCESS);
1893 }
1894 
1895 static PetscErrorCode DMPlexCreateBoxMesh_Tensor_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[])
1896 {
1897   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1898   PetscInt       fac[3] = {0, 0, 0}, d;
1899 
1900   PetscFunctionBegin;
1901   PetscAssertPointer(dm, 1);
1902   PetscValidLogicalCollectiveInt(dm, dim, 2);
1903   PetscCall(DMSetDimension(dm, dim));
1904   for (d = 0; d < dim; ++d) {
1905     fac[d] = faces[d];
1906     bdt[d] = periodicity[d];
1907   }
1908   PetscCall(DMPlexCreateCubeMesh_Internal(dm, lower, upper, fac, bdt[0], bdt[1], bdt[2]));
1909   if (periodicity[0] == DM_BOUNDARY_PERIODIC || periodicity[0] == DM_BOUNDARY_TWIST || periodicity[1] == DM_BOUNDARY_PERIODIC || periodicity[1] == DM_BOUNDARY_TWIST || (dim > 2 && (periodicity[2] == DM_BOUNDARY_PERIODIC || periodicity[2] == DM_BOUNDARY_TWIST))) {
1910     PetscReal L[3]       = {-1., -1., 0.};
1911     PetscReal maxCell[3] = {-1., -1., 0.};
1912 
1913     for (d = 0; d < dim; ++d) {
1914       if (periodicity[d] != DM_BOUNDARY_NONE) {
1915         L[d]       = upper[d] - lower[d];
1916         maxCell[d] = 1.1 * (L[d] / PetscMax(1, faces[d]));
1917       }
1918     }
1919     PetscCall(DMSetPeriodicity(dm, maxCell, lower, L));
1920   }
1921   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
1922   PetscFunctionReturn(PETSC_SUCCESS);
1923 }
1924 
1925 static PetscErrorCode DMPlexCreateBoxMesh_Internal(DM dm, DMPlexShape shape, PetscInt dim, PetscBool simplex, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate)
1926 {
1927   PetscFunctionBegin;
1928   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
1929   if (shape == DM_SHAPE_ZBOX) PetscCall(DMPlexCreateBoxMesh_Tensor_SFC_Internal(dm, dim, faces, lower, upper, periodicity, interpolate));
1930   else if (dim == 1) PetscCall(DMPlexCreateLineMesh_Internal(dm, faces[0], lower[0], upper[0], periodicity[0]));
1931   else if (simplex) PetscCall(DMPlexCreateBoxMesh_Simplex_Internal(dm, dim, faces, lower, upper, periodicity, interpolate));
1932   else PetscCall(DMPlexCreateBoxMesh_Tensor_Internal(dm, dim, faces, lower, upper, periodicity));
1933   if (!interpolate && dim > 1 && !simplex) {
1934     DM udm;
1935 
1936     PetscCall(DMPlexUninterpolate(dm, &udm));
1937     PetscCall(DMPlexCopyCoordinates(dm, udm));
1938     PetscCall(DMPlexReplace_Internal(dm, &udm));
1939   }
1940   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
1941   PetscFunctionReturn(PETSC_SUCCESS);
1942 }
1943 
1944 /*@
1945   DMPlexCreateBoxMesh - Creates a mesh on the tensor product of unit intervals (box) using simplices or tensor cells (hexahedra).
1946 
1947   Collective
1948 
1949   Input Parameters:
1950 + comm               - The communicator for the `DM` object
1951 . dim                - The spatial dimension
1952 . simplex            - `PETSC_TRUE` for simplices, `PETSC_FALSE` for tensor cells
1953 . faces              - Number of faces per dimension, or `NULL` for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
1954 . lower              - The lower left corner, or `NULL` for (0, 0, 0)
1955 . upper              - The upper right corner, or `NULL` for (1, 1, 1)
1956 . periodicity        - The boundary type for the X,Y,Z direction, or `NULL` for `DM_BOUNDARY_NONE`
1957 . interpolate        - Flag to create intermediate mesh pieces (edges, faces)
1958 . localizationHeight - Flag to localize edges and faces in addition to cells; only significant for periodic meshes
1959 - sparseLocalize     - Flag to localize coordinates only for cells near the periodic boundary; only significant for periodic meshes
1960 
1961   Output Parameter:
1962 . dm - The `DM` object
1963 
1964   Level: beginner
1965 
1966   Note:
1967   To customize this mesh using options, use
1968 .vb
1969   DMCreate(comm, &dm);
1970   DMSetType(dm, DMPLEX);
1971   DMSetFromOptions(dm);
1972 .ve
1973   and use the options in `DMSetFromOptions()`.
1974 
1975   Here is the numbering returned for 2 faces in each direction for tensor cells\:
1976 .vb
1977  10---17---11---18----12
1978   |         |         |
1979   |         |         |
1980  20    2   22    3    24
1981   |         |         |
1982   |         |         |
1983   7---15----8---16----9
1984   |         |         |
1985   |         |         |
1986  19    0   21    1   23
1987   |         |         |
1988   |         |         |
1989   4---13----5---14----6
1990 .ve
1991   and for simplicial cells
1992 .vb
1993  14----8---15----9----16
1994   |\     5  |\      7 |
1995   | \       | \       |
1996  13   2    14    3    15
1997   | 4   \   | 6   \   |
1998   |       \ |       \ |
1999  11----6---12----7----13
2000   |\        |\        |
2001   | \    1  | \     3 |
2002  10   0    11    1    12
2003   | 0   \   | 2   \   |
2004   |       \ |       \ |
2005   8----4----9----5----10
2006 .ve
2007 
2008 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMSetFromOptions()`, `DMPlexCreateFromFile()`, `DMPlexCreateHexCylinderMesh()`, `DMSetType()`, `DMCreate()`
2009 @*/
2010 PetscErrorCode DMPlexCreateBoxMesh(MPI_Comm comm, PetscInt dim, PetscBool simplex, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate, PetscInt localizationHeight, PetscBool sparseLocalize, DM *dm)
2011 {
2012   PetscInt       fac[3] = {1, 1, 1};
2013   PetscReal      low[3] = {0, 0, 0};
2014   PetscReal      upp[3] = {1, 1, 1};
2015   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
2016 
2017   PetscFunctionBegin;
2018   PetscCall(DMCreate(comm, dm));
2019   PetscCall(DMSetType(*dm, DMPLEX));
2020   PetscCall(DMPlexCreateBoxMesh_Internal(*dm, DM_SHAPE_BOX, dim, simplex, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, periodicity ? periodicity : bdt, interpolate));
2021   if (periodicity) {
2022     DM cdm;
2023 
2024     PetscCall(DMGetCoordinateDM(*dm, &cdm));
2025     PetscCall(DMPlexSetMaxProjectionHeight(cdm, localizationHeight));
2026     PetscCall(DMSetSparseLocalize(*dm, sparseLocalize));
2027     PetscCall(DMLocalizeCoordinates(*dm));
2028   }
2029   PetscFunctionReturn(PETSC_SUCCESS);
2030 }
2031 
2032 static PetscErrorCode DMPlexCreateWedgeBoxMesh_Internal(DM dm, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[])
2033 {
2034   DM       bdm, vol;
2035   PetscInt i;
2036 
2037   PetscFunctionBegin;
2038   // TODO Now we can support periodicity
2039   for (i = 0; i < 3; ++i) PetscCheck(periodicity[i] == DM_BOUNDARY_NONE, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Periodicity not yet supported");
2040   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &bdm));
2041   PetscCall(DMSetType(bdm, DMPLEX));
2042   PetscCall(DMSetDimension(bdm, 2));
2043   PetscCall(PetscLogEventBegin(DMPLEX_Generate, bdm, 0, 0, 0));
2044   PetscCall(DMPlexCreateBoxMesh_Simplex_Internal(bdm, 2, faces, lower, upper, periodicity, PETSC_TRUE));
2045   PetscCall(DMPlexExtrude(bdm, faces[2], upper[2] - lower[2], PETSC_TRUE, PETSC_FALSE, PETSC_FALSE, NULL, NULL, &vol));
2046   PetscCall(PetscLogEventEnd(DMPLEX_Generate, bdm, 0, 0, 0));
2047   PetscCall(DMDestroy(&bdm));
2048   PetscCall(DMPlexReplace_Internal(dm, &vol));
2049   if (lower[2] != 0.0) {
2050     Vec          v;
2051     PetscScalar *x;
2052     PetscInt     cDim, n;
2053 
2054     PetscCall(DMGetCoordinatesLocal(dm, &v));
2055     PetscCall(VecGetBlockSize(v, &cDim));
2056     PetscCall(VecGetLocalSize(v, &n));
2057     PetscCall(VecGetArray(v, &x));
2058     x += cDim;
2059     for (i = 0; i < n; i += cDim) x[i] += lower[2];
2060     PetscCall(VecRestoreArray(v, &x));
2061     PetscCall(DMSetCoordinatesLocal(dm, v));
2062   }
2063   PetscFunctionReturn(PETSC_SUCCESS);
2064 }
2065 
2066 /*@
2067   DMPlexCreateWedgeBoxMesh - Creates a 3-D mesh tessellating the (x,y) plane and extruding in the third direction using wedge cells.
2068 
2069   Collective
2070 
2071   Input Parameters:
2072 + comm        - The communicator for the `DM` object
2073 . faces       - Number of faces per dimension, or `NULL` for (1, 1, 1)
2074 . lower       - The lower left corner, or `NULL` for (0, 0, 0)
2075 . upper       - The upper right corner, or `NULL` for (1, 1, 1)
2076 . periodicity - The boundary type for the X,Y,Z direction, or `NULL` for `DM_BOUNDARY_NONE`
2077 . orderHeight - If `PETSC_TRUE`, orders the extruded cells in the height first. Otherwise, orders the cell on the layers first
2078 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
2079 
2080   Output Parameter:
2081 . dm - The `DM` object
2082 
2083   Level: beginner
2084 
2085 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateHexCylinderMesh()`, `DMPlexCreateWedgeCylinderMesh()`, `DMExtrude()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
2086 @*/
2087 PetscErrorCode DMPlexCreateWedgeBoxMesh(MPI_Comm comm, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool orderHeight, PetscBool interpolate, DM *dm)
2088 {
2089   PetscInt       fac[3] = {1, 1, 1};
2090   PetscReal      low[3] = {0, 0, 0};
2091   PetscReal      upp[3] = {1, 1, 1};
2092   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
2093 
2094   PetscFunctionBegin;
2095   PetscCall(DMCreate(comm, dm));
2096   PetscCall(DMSetType(*dm, DMPLEX));
2097   PetscCall(DMPlexCreateWedgeBoxMesh_Internal(*dm, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, periodicity ? periodicity : bdt));
2098   if (!interpolate) {
2099     DM udm;
2100 
2101     PetscCall(DMPlexUninterpolate(*dm, &udm));
2102     PetscCall(DMPlexReplace_Internal(*dm, &udm));
2103   }
2104   if (periodicity) PetscCall(DMLocalizeCoordinates(*dm));
2105   PetscFunctionReturn(PETSC_SUCCESS);
2106 }
2107 
2108 /*
2109   DMPlexTensorPointLexicographic_Private - Returns all tuples of size 'len' with nonnegative integers that are all less than or equal to 'max' for that dimension.
2110 
2111   Input Parameters:
2112 + len - The length of the tuple
2113 . max - The maximum for each dimension, so values are in [0, max)
2114 - tup - A tuple of length len+1: tup[len] > 0 indicates a stopping condition
2115 
2116   Output Parameter:
2117 . tup - A tuple of `len` integers whose entries are at most `max`
2118 
2119   Level: developer
2120 
2121   Note:
2122   Ordering is lexicographic with lowest index as least significant in ordering.
2123   e.g. for len == 2 and max == 2, this will return, in order, {0,0}, {1,0}, {2,0}, {0,1}, {1,1}, {2,1}, {0,2}, {1,2}, {2,2}.
2124 
2125 .seealso: PetscDualSpaceTensorPointLexicographic_Internal(), PetscDualSpaceLatticePointLexicographic_Internal()
2126 */
2127 static PetscErrorCode DMPlexTensorPointLexicographic_Private(PetscInt len, const PetscInt max[], PetscInt tup[])
2128 {
2129   PetscInt i;
2130 
2131   PetscFunctionBegin;
2132   for (i = 0; i < len; ++i) {
2133     if (tup[i] < max[i] - 1) {
2134       break;
2135     } else {
2136       tup[i] = 0;
2137     }
2138   }
2139   if (i == len) tup[i - 1] = max[i - 1];
2140   else ++tup[i];
2141   PetscFunctionReturn(PETSC_SUCCESS);
2142 }
2143 
2144 static PetscInt TupleToIndex_Private(PetscInt len, const PetscInt max[], const PetscInt tup[])
2145 {
2146   PetscInt idx = tup[len - 1];
2147 
2148   for (PetscInt i = len - 2; i >= 0; --i) {
2149     idx *= max[i];
2150     idx += tup[i];
2151   }
2152   return idx;
2153 }
2154 
2155 static void IndexToTuple_Private(PetscInt len, const PetscInt max[], PetscInt idx, PetscInt tup[])
2156 {
2157   for (PetscInt i = 0; i < len; ++i) {
2158     tup[i] = idx % max[i];
2159     idx    = (idx - tup[i]) / max[i];
2160   }
2161 }
2162 
2163 static void TupleToRanks_Private(PetscInt len, const PetscInt max[], const PetscInt procs[], const PetscInt tup[], PetscInt ranks[])
2164 {
2165   for (PetscInt i = 0; i < len; ++i) {
2166     const PetscInt div = max[i] / procs[i];
2167     const PetscInt rem = max[i] % procs[i];
2168     const PetscInt idx = (tup[i] < 0 ? max[i] + tup[i] : tup[i]) % max[i];
2169 
2170     if (idx < rem * (div + 1)) ranks[i] = idx / (div + 1);
2171     else ranks[i] = rem + (idx - rem * (div + 1)) / div;
2172   }
2173 }
2174 
2175 static void RanksToSizes_Private(PetscInt len, const PetscInt max[], const PetscInt procs[], const PetscInt ranks[], PetscInt sizes[])
2176 {
2177   for (PetscInt i = 0; i < len; ++i) {
2178     const PetscInt div = max[i] / procs[i];
2179     const PetscInt rem = max[i] % procs[i];
2180 
2181     sizes[i] = ranks[i] < rem ? div + 1 : div;
2182   }
2183 }
2184 
2185 /*
2186   In serial, the mesh is completely periodic. In parallel, we will include a layer of ghost vertices around the patch, so our edges will not wrap around in parallel. This will instead be handled by the SF.
2187 
2188   The cone for all edges will always be complete. Even in the presence of boundaries, we will keep all support sizes constant. When an edge would not exist in the support, we will create one to wrap back periodically. This allows for uniform stencils, and that edge will not be used for computation.
2189 
2190   All point which do not attain the vertex lower or upper bound in any dimension are owned, the rest are leaves owned by another process and present in the SF.
2191 
2192   Parallel Layout:
2193 
2194   We create a cubic process grid of dimension P^{1/d} on each side. The IndexToTuple_Private() function maps the global rank to the local rank in each dimension. We divide edges using the PETSc distribution rule in each dimension, and then add overlap. TupleToRanks_Private() returns the local rank in ech dimension for a tuple. RanksToSizes_Private() gives the size in each dimension for the domain with those local ranks.
2195 */
2196 static PetscErrorCode DMPlexCreateHypercubicMesh_Internal(DM dm, PetscInt dim, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[], PetscInt overlap, const DMBoundaryType bd[])
2197 {
2198   const PetscInt debug = ((DM_Plex *)dm->data)->printAdj;
2199   PetscSF        sf;
2200   Vec            coordinates;
2201   PetscSection   coordSection;
2202   DMLabel        cutLabel    = NULL;
2203   PetscBool      cutMarker   = PETSC_FALSE;
2204   PetscBool      periodic    = PETSC_FALSE;
2205   PetscInt       numCells    = 1;
2206   PetscInt       numVertices = 1;
2207   PetscSFNode   *remotes;
2208   PetscScalar   *coords;
2209   PetscInt      *procs;     // The number of processes along each dimension
2210   PetscInt      *lrank;     // Rank in each dimension, lrank[d] \in [0, procs[d])
2211   PetscInt      *ledges;    // The number of edges along each dimension for this process
2212   PetscInt      *vstart;    // The first vertex along each dimension on this processes
2213   PetscInt      *vertices;  // The number of vertices along each dimension on this process
2214   PetscInt      *rvert;     // The global (not local) vertex number along each dimension
2215   PetscInt      *rrank;     // The rank along each dimension for the process owning rvert[]
2216   PetscInt      *rvertices; // The number of vertices along each dimension for the process rrank[]
2217   PetscInt      *vert, *vtmp, *supp, cone[2], *leaves;
2218   PetscInt       cell = 0, coordSize, Nl = 0, Nl2 = 0;
2219   PetscMPIInt    rank, size;
2220   MPI_Comm       comm;
2221 
2222   PetscFunctionBegin;
2223   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
2224   PetscCallMPI(MPI_Comm_rank(comm, &rank));
2225   PetscCallMPI(MPI_Comm_size(comm, &size));
2226   PetscCall(DMSetDimension(dm, dim));
2227   PetscCall(DMPlexDistributeSetDefault(dm, PETSC_FALSE));
2228   PetscCall(PetscCalloc4(dim, &procs, dim, &lrank, dim, &rrank, 2 * dim, &supp));
2229   PetscCall(PetscCalloc7(dim, &ledges, dim, &vertices, dim, &rvertices, dim, &vert, dim, &rvert, dim, &vstart, dim, &vtmp));
2230   PetscCall(DMCreateLabel(dm, "marker"));
2231   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_periodic_cut", &cutMarker, NULL));
2232   for (PetscInt d = 0; d < dim; ++d) periodic = (periodic || bd[d] == DM_BOUNDARY_PERIODIC) ? PETSC_TRUE : PETSC_FALSE;
2233   if (periodic && cutMarker) {
2234     PetscCall(DMCreateLabel(dm, "periodic_cut"));
2235     PetscCall(DMGetLabel(dm, "periodic_cut", &cutLabel));
2236   }
2237   for (PetscInt d = 0; d < dim; ++d) PetscCheck(bd[d] == DM_BOUNDARY_PERIODIC, comm, PETSC_ERR_SUP, "Hypercubic mesh must be periodic now");
2238   overlap = overlap == PETSC_DETERMINE ? 1 : overlap;
2239   PetscCheck(overlap >= 1, comm, PETSC_ERR_SUP, "Overlap %" PetscInt_FMT " must be greater than 0", overlap);
2240   if (size > 1) {
2241     PetscInt Npr = 1;
2242 
2243     // Make process grid
2244     if (debug) PetscCall(PetscPrintf(comm, "Process grid:"));
2245     for (PetscInt d = 0; d < dim; ++d) {
2246       procs[d] = PetscRintReal(PetscPowReal(size, 1. / dim));
2247       Npr *= procs[d];
2248       if (debug) PetscCall(PetscPrintf(comm, " %" PetscInt_FMT, procs[d]));
2249     }
2250     if (debug) PetscCall(PetscPrintf(comm, "\n"));
2251     PetscCheck(Npr == size, comm, PETSC_ERR_PLIB, "Process grid size %" PetscInt_FMT " != %d comm size", Npr, size);
2252     IndexToTuple_Private(dim, procs, rank, lrank);
2253     for (PetscInt d = 0; d < dim; ++d) {
2254       ledges[d] = edges[d] / procs[d] + (edges[d] % procs[d] > lrank[d] ? 1 : 0);
2255       vstart[d] = 0;
2256       for (PetscInt r = 0; r < lrank[d]; ++r) vstart[d] += edges[d] / procs[d] + (edges[d] % procs[d] > r ? 1 : 0);
2257       vstart[d] -= overlap; // For halo
2258     }
2259   } else {
2260     for (PetscInt d = 0; d < dim; ++d) {
2261       procs[d]  = 1;
2262       ledges[d] = edges[d];
2263     }
2264   }
2265   // Calculate local patch size
2266   for (PetscInt d = 0; d < dim; ++d) {
2267     vertices[d] = ledges[d] + (procs[d] > 1 ? 2 * overlap : 0);
2268     numVertices *= vertices[d];
2269   }
2270   numCells = numVertices * dim;
2271   PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
2272   for (PetscInt c = 0; c < numCells; ++c) PetscCall(DMPlexSetConeSize(dm, c, 2));
2273   for (PetscInt v = numCells; v < numCells + numVertices; ++v) PetscCall(DMPlexSetSupportSize(dm, v, 2 * dim));
2274   PetscCall(DMSetUp(dm)); /* Allocate space for cones and supports */
2275   /* Build cell cones and vertex supports */
2276   PetscCall(DMCreateLabel(dm, "celltype"));
2277   if (debug) PetscCall(PetscSynchronizedPrintf(comm, "Topology for rank %d:\n", rank));
2278   while (vert[dim - 1] < vertices[dim - 1]) {
2279     const PetscInt vertex = TupleToIndex_Private(dim, vertices, vert) + numCells;
2280     PetscInt       s      = 0;
2281     PetscBool      leaf   = PETSC_FALSE;
2282 
2283     if (debug) {
2284       PetscCall(PetscSynchronizedPrintf(comm, "Vertex %" PetscInt_FMT ":", vertex));
2285       for (PetscInt d = 0; d < dim; ++d) PetscCall(PetscSynchronizedPrintf(comm, " %" PetscInt_FMT, vert[d]));
2286       PetscCall(PetscSynchronizedPrintf(comm, "\n"));
2287     }
2288     PetscCall(DMPlexSetCellType(dm, vertex, DM_POLYTOPE_POINT));
2289     // Define edge cones
2290     for (PetscInt d = 0; d < dim; ++d) {
2291       for (PetscInt e = 0; e < dim; ++e) vtmp[e] = vert[e];
2292       vtmp[d] = (vert[d] + 1) % vertices[d];
2293       cone[0] = vertex;
2294       cone[1] = TupleToIndex_Private(dim, vertices, vtmp) + numCells;
2295       if (debug) {
2296         PetscCall(PetscSynchronizedPrintf(comm, "  Vertex %" PetscInt_FMT ":", cone[1]));
2297         for (PetscInt e = 0; e < dim; ++e) PetscCall(PetscSynchronizedPrintf(comm, " %" PetscInt_FMT, vtmp[e]));
2298         PetscCall(PetscSynchronizedPrintf(comm, "\n"));
2299       }
2300       PetscCall(DMPlexSetCone(dm, cell, cone));
2301       PetscCall(DMPlexSetCellType(dm, cell, DM_POLYTOPE_SEGMENT));
2302       if (debug) PetscCall(PetscSynchronizedPrintf(comm, "  Edge %" PetscInt_FMT " (%" PetscInt_FMT " %" PetscInt_FMT ")\n", cell, cone[0], cone[1]));
2303       ++cell;
2304       // Shared vertices are any in the first or last overlap layers
2305       if (vert[d] < overlap || vert[d] >= vertices[d] - overlap) leaf = PETSC_TRUE;
2306     }
2307     if (size > 1 && leaf) ++Nl;
2308     // Define vertex supports
2309     for (PetscInt d = 0; d < dim; ++d) {
2310       for (PetscInt e = 0; e < dim; ++e) vtmp[e] = vert[e];
2311       vtmp[d]   = (vert[d] + vertices[d] - 1) % vertices[d];
2312       supp[s++] = TupleToIndex_Private(dim, vertices, vtmp) * dim + d;
2313       supp[s++] = (vertex - numCells) * dim + d;
2314       PetscCall(DMPlexSetSupport(dm, vertex, supp));
2315     }
2316     PetscCall(DMPlexTensorPointLexicographic_Private(dim, vertices, vert));
2317   }
2318   if (debug) PetscCall(PetscSynchronizedFlush(comm, NULL));
2319   PetscCall(DMPlexStratify(dm));
2320   // Allocate for SF
2321   PetscCall(PetscMalloc1(Nl, &leaves));
2322   PetscCall(PetscMalloc1(Nl, &remotes));
2323   // Build coordinates
2324   PetscCall(DMGetCoordinateSection(dm, &coordSection));
2325   PetscCall(PetscSectionSetNumFields(coordSection, 1));
2326   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
2327   PetscCall(PetscSectionSetChart(coordSection, numCells, numCells + numVertices));
2328   for (PetscInt v = numCells; v < numCells + numVertices; ++v) {
2329     PetscCall(PetscSectionSetDof(coordSection, v, dim));
2330     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
2331   }
2332   PetscCall(PetscSectionSetUp(coordSection));
2333   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
2334   PetscCall(VecCreate(comm, &coordinates));
2335   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
2336   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
2337   PetscCall(VecSetBlockSize(coordinates, dim));
2338   PetscCall(VecSetType(coordinates, VECSTANDARD));
2339   PetscCall(VecGetArray(coordinates, &coords));
2340   if (debug) PetscCall(PetscSynchronizedPrintf(comm, "Geometry for rank %d:\n", rank));
2341   for (PetscInt d = 0; d < dim; ++d) vert[d] = 0;
2342   while (vert[dim - 1] < vertices[dim - 1]) {
2343     const PetscInt vertex = TupleToIndex_Private(dim, vertices, vert);
2344     PetscBool      leaf   = PETSC_FALSE;
2345 
2346     for (PetscInt d = 0; d < dim; ++d) {
2347       coords[vertex * dim + d] = lower[d] + ((upper[d] - lower[d]) / edges[d]) * (vert[d] + vstart[d]);
2348       if (vert[d] < overlap || vert[d] >= vertices[d] - overlap) leaf = PETSC_TRUE;
2349     }
2350     if (size > 1 && leaf) {
2351       PetscInt rnumCells = 1;
2352 
2353       for (PetscInt d = 0; d < dim; ++d) rvert[d] = vert[d] + vstart[d];
2354       TupleToRanks_Private(dim, edges, procs, rvert, rrank);
2355       leaves[Nl2]       = vertex + numCells;
2356       remotes[Nl2].rank = TupleToIndex_Private(dim, procs, rrank);
2357       RanksToSizes_Private(dim, edges, procs, rrank, rvertices);
2358       for (PetscInt d = 0; d < dim; ++d) {
2359         rvertices[d] += 2 * overlap; // Add halo
2360         rnumCells *= rvertices[d];
2361       }
2362       rnumCells *= dim;
2363       for (PetscInt d = 0; d < dim; ++d) {
2364         const PetscInt diff = rrank[d] - lrank[d];
2365 
2366         if (!diff) rvert[d] = vert[d];                                     // Vertex is local
2367         else if (rvert[d] < 0) rvert[d] = rvertices[d] - 1 + rvert[d];     // Wrap around at the bottom
2368         else if (rvert[d] >= edges[d]) rvert[d] = rvert[d] - edges[d] + 1; // Wrap around at the top
2369         else if (diff == -1) rvert[d] = rvertices[d] - 1 + (vert[d] - overlap);
2370         else if (diff == 1) rvert[d] = (vertices[d] - vert[d] - 1) + overlap;
2371         else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Process distance %" PetscInt_FMT " in direction %" PetscInt_FMT " should not be possible", diff, d);
2372       }
2373       remotes[Nl2].index = TupleToIndex_Private(dim, rvertices, rvert) + rnumCells;
2374       if (debug) PetscCall(PetscSynchronizedPrintf(comm, "Shared Vertex %" PetscInt_FMT " (%" PetscInt_FMT ", %" PetscInt_FMT ")\n", leaves[Nl2], remotes[Nl2].rank, remotes[Nl2].index));
2375       ++Nl2;
2376     }
2377     if (debug) {
2378       PetscCall(PetscSynchronizedPrintf(comm, "Vertex %" PetscInt_FMT ":", vertex));
2379       for (PetscInt d = 0; d < dim; ++d) PetscCall(PetscSynchronizedPrintf(comm, " %" PetscInt_FMT, vert[d] + vstart[d]));
2380       for (PetscInt d = 0; d < dim; ++d) PetscCall(PetscSynchronizedPrintf(comm, " %g", (double)PetscRealPart(coords[vertex * dim + d])));
2381       PetscCall(PetscSynchronizedPrintf(comm, "\n"));
2382     }
2383     PetscCall(DMPlexTensorPointLexicographic_Private(dim, vertices, vert));
2384   }
2385   if (debug) PetscCall(PetscSynchronizedFlush(comm, NULL));
2386   PetscCall(VecRestoreArray(coordinates, &coords));
2387   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
2388   PetscCall(VecDestroy(&coordinates));
2389   // Build SF
2390   PetscCheck(Nl == Nl2, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Initial number of leaves %" PetscInt_FMT " != %" PetscInt_FMT " final number", Nl, Nl2);
2391   PetscCall(DMGetPointSF(dm, &sf));
2392   PetscCall(PetscSFSetGraph(sf, numCells + numVertices, Nl, leaves, PETSC_OWN_POINTER, remotes, PETSC_OWN_POINTER));
2393   if (debug) PetscCall(PetscSFView(sf, PETSC_VIEWER_STDOUT_WORLD));
2394   //PetscCall(DMSetPeriodicity(dm, NULL, lower, upper));
2395   // Attach the extent
2396   {
2397     PetscContainer c;
2398     PetscInt      *extent, *lextent;
2399 
2400     PetscCall(PetscMalloc1(dim, &extent));
2401     PetscCall(PetscMalloc1(dim, &lextent));
2402     for (PetscInt d = 0; d < dim; ++d) {
2403       extent[d]  = edges[d];
2404       lextent[d] = ledges[d];
2405     }
2406     PetscCall(PetscContainerCreate(PETSC_COMM_SELF, &c));
2407     PetscCall(PetscContainerSetCtxDestroy(c, PetscCtxDestroyDefault));
2408     PetscCall(PetscContainerSetPointer(c, extent));
2409     PetscCall(PetscObjectCompose((PetscObject)dm, "_extent", (PetscObject)c));
2410     PetscCall(PetscContainerDestroy(&c));
2411     PetscCall(PetscContainerCreate(PETSC_COMM_SELF, &c));
2412     PetscCall(PetscContainerSetCtxDestroy(c, PetscCtxDestroyDefault));
2413     PetscCall(PetscContainerSetPointer(c, lextent));
2414     PetscCall(PetscObjectCompose((PetscObject)dm, "_lextent", (PetscObject)c));
2415     PetscCall(PetscContainerDestroy(&c));
2416   }
2417   PetscCall(PetscFree4(procs, lrank, rrank, supp));
2418   PetscCall(PetscFree7(ledges, vertices, rvertices, vert, rvert, vstart, vtmp));
2419   PetscFunctionReturn(PETSC_SUCCESS);
2420 }
2421 
2422 /*@C
2423   DMPlexCreateHypercubicMesh - Creates a periodic mesh on the tensor product of unit intervals using only vertices and edges.
2424 
2425   Collective
2426 
2427   Input Parameters:
2428 + comm    - The communicator for the `DM` object
2429 . dim     - The spatial dimension
2430 . edges   - Number of edges per dimension, or `NULL` for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
2431 . lower   - The lower left corner, or `NULL` for (0, 0, 0)
2432 . upper   - The upper right corner, or `NULL` for (1, 1, 1)
2433 - overlap - The number of vertices in each direction to include in the overlap (default is 1)
2434 
2435   Output Parameter:
2436 . dm - The DM object
2437 
2438   Level: beginner
2439 
2440   Note:
2441   If you want to customize this mesh using options, you just need to
2442 .vb
2443   DMCreate(comm, &dm);
2444   DMSetType(dm, DMPLEX);
2445   DMSetFromOptions(dm);
2446 .ve
2447   and use the options on the `DMSetFromOptions()` page.
2448 
2449   The vertices are numbered is lexicographic order, and the dim edges exiting a vertex in the positive orthant are number consecutively,
2450 .vb
2451  18--0-19--2-20--4-18
2452   |     |     |     |
2453  13    15    17    13
2454   |     |     |     |
2455  24-12-25-14-26-16-24
2456   |     |     |     |
2457   7     9    11     7
2458   |     |     |     |
2459  21--6-22--8-23-10-21
2460   |     |     |     |
2461   1     3     5     1
2462   |     |     |     |
2463  18--0-19--2-20--4-18
2464 .ve
2465 
2466 .seealso: `DMSetFromOptions()`, `DMPlexCreateFromFile()`, `DMPlexCreateHexCylinderMesh()`, `DMSetType()`, `DMCreate()`
2467 @*/
2468 PetscErrorCode DMPlexCreateHypercubicMesh(MPI_Comm comm, PetscInt dim, const PetscInt edges[], const PetscReal lower[], const PetscReal upper[], PetscInt overlap, DM *dm)
2469 {
2470   PetscInt       *edg;
2471   PetscReal      *low, *upp;
2472   DMBoundaryType *bdt;
2473   PetscInt        d;
2474 
2475   PetscFunctionBegin;
2476   PetscCall(DMCreate(comm, dm));
2477   PetscCall(DMSetType(*dm, DMPLEX));
2478   PetscCall(PetscMalloc4(dim, &edg, dim, &low, dim, &upp, dim, &bdt));
2479   for (d = 0; d < dim; ++d) {
2480     edg[d] = edges ? edges[d] : 1;
2481     low[d] = lower ? lower[d] : 0.;
2482     upp[d] = upper ? upper[d] : 1.;
2483     bdt[d] = DM_BOUNDARY_PERIODIC;
2484   }
2485   PetscCall(DMPlexCreateHypercubicMesh_Internal(*dm, dim, low, upp, edg, overlap, bdt));
2486   PetscCall(PetscFree4(edg, low, upp, bdt));
2487   PetscFunctionReturn(PETSC_SUCCESS);
2488 }
2489 
2490 /*@
2491   DMPlexSetOptionsPrefix - Sets the prefix used for searching for all `DM` options in the database.
2492 
2493   Logically Collective
2494 
2495   Input Parameters:
2496 + dm     - the `DM` context
2497 - prefix - the prefix to prepend to all option names
2498 
2499   Level: advanced
2500 
2501   Note:
2502   A hyphen (-) must NOT be given at the beginning of the prefix name.
2503   The first character of all runtime options is AUTOMATICALLY the hyphen.
2504 
2505 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `SNESSetFromOptions()`
2506 @*/
2507 PetscErrorCode DMPlexSetOptionsPrefix(DM dm, const char prefix[])
2508 {
2509   DM_Plex *mesh = (DM_Plex *)dm->data;
2510 
2511   PetscFunctionBegin;
2512   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2513   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)dm, prefix));
2514   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)mesh->partitioner, prefix));
2515   PetscFunctionReturn(PETSC_SUCCESS);
2516 }
2517 
2518 /* Remap geometry to cylinder
2519    TODO: This only works for a single refinement, then it is broken
2520 
2521      Interior square: Linear interpolation is correct
2522      The other cells all have vertices on rays from the origin. We want to uniformly expand the spacing
2523      such that the last vertex is on the unit circle. So the closest and farthest vertices are at distance
2524 
2525        phi     = arctan(y/x)
2526        d_close = sqrt(1/8 + 1/4 sin^2(phi))
2527        d_far   = sqrt(1/2 + sin^2(phi))
2528 
2529      so we remap them using
2530 
2531        x_new = x_close + (x - x_close) (1 - d_close) / (d_far - d_close)
2532        y_new = y_close + (y - y_close) (1 - d_close) / (d_far - d_close)
2533 
2534      If pi/4 < phi < 3pi/4 or -3pi/4 < phi < -pi/4, then we switch x and y.
2535 */
2536 static void snapToCylinder(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
2537 {
2538   const PetscReal dis = 1.0 / PetscSqrtReal(2.0);
2539   const PetscReal ds2 = 0.5 * dis;
2540 
2541   if ((PetscAbsScalar(u[0]) <= ds2) && (PetscAbsScalar(u[1]) <= ds2)) {
2542     f0[0] = u[0];
2543     f0[1] = u[1];
2544   } else {
2545     PetscReal phi, sinp, cosp, dc, df, x, y, xc, yc;
2546 
2547     x    = PetscRealPart(u[0]);
2548     y    = PetscRealPart(u[1]);
2549     phi  = PetscAtan2Real(y, x);
2550     sinp = PetscSinReal(phi);
2551     cosp = PetscCosReal(phi);
2552     if ((PetscAbsReal(phi) > PETSC_PI / 4.0) && (PetscAbsReal(phi) < 3.0 * PETSC_PI / 4.0)) {
2553       dc = PetscAbsReal(ds2 / sinp);
2554       df = PetscAbsReal(dis / sinp);
2555       xc = ds2 * x / PetscAbsReal(y);
2556       yc = ds2 * PetscSignReal(y);
2557     } else {
2558       dc = PetscAbsReal(ds2 / cosp);
2559       df = PetscAbsReal(dis / cosp);
2560       xc = ds2 * PetscSignReal(x);
2561       yc = ds2 * y / PetscAbsReal(x);
2562     }
2563     f0[0] = xc + (u[0] - xc) * (1.0 - dc) / (df - dc);
2564     f0[1] = yc + (u[1] - yc) * (1.0 - dc) / (df - dc);
2565   }
2566   f0[2] = u[2];
2567 }
2568 
2569 static PetscErrorCode DMPlexCreateHexCylinderMesh_Internal(DM dm, DMBoundaryType periodicZ, PetscInt Nr)
2570 {
2571   const PetscInt dim = 3;
2572   PetscInt       numCells, numVertices;
2573   PetscMPIInt    rank;
2574 
2575   PetscFunctionBegin;
2576   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
2577   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
2578   PetscCall(DMSetDimension(dm, dim));
2579   /* Create topology */
2580   {
2581     PetscInt cone[8], c;
2582 
2583     numCells    = rank == 0 ? 5 : 0;
2584     numVertices = rank == 0 ? 16 : 0;
2585     if (periodicZ == DM_BOUNDARY_PERIODIC) {
2586       numCells *= 3;
2587       numVertices = rank == 0 ? 24 : 0;
2588     }
2589     PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
2590     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 8));
2591     PetscCall(DMSetUp(dm));
2592     if (rank == 0) {
2593       if (periodicZ == DM_BOUNDARY_PERIODIC) {
2594         cone[0] = 15;
2595         cone[1] = 18;
2596         cone[2] = 17;
2597         cone[3] = 16;
2598         cone[4] = 31;
2599         cone[5] = 32;
2600         cone[6] = 33;
2601         cone[7] = 34;
2602         PetscCall(DMPlexSetCone(dm, 0, cone));
2603         cone[0] = 16;
2604         cone[1] = 17;
2605         cone[2] = 24;
2606         cone[3] = 23;
2607         cone[4] = 32;
2608         cone[5] = 36;
2609         cone[6] = 37;
2610         cone[7] = 33; /* 22 25 26 21 */
2611         PetscCall(DMPlexSetCone(dm, 1, cone));
2612         cone[0] = 18;
2613         cone[1] = 27;
2614         cone[2] = 24;
2615         cone[3] = 17;
2616         cone[4] = 34;
2617         cone[5] = 33;
2618         cone[6] = 37;
2619         cone[7] = 38;
2620         PetscCall(DMPlexSetCone(dm, 2, cone));
2621         cone[0] = 29;
2622         cone[1] = 27;
2623         cone[2] = 18;
2624         cone[3] = 15;
2625         cone[4] = 35;
2626         cone[5] = 31;
2627         cone[6] = 34;
2628         cone[7] = 38;
2629         PetscCall(DMPlexSetCone(dm, 3, cone));
2630         cone[0] = 29;
2631         cone[1] = 15;
2632         cone[2] = 16;
2633         cone[3] = 23;
2634         cone[4] = 35;
2635         cone[5] = 36;
2636         cone[6] = 32;
2637         cone[7] = 31;
2638         PetscCall(DMPlexSetCone(dm, 4, cone));
2639 
2640         cone[0] = 31;
2641         cone[1] = 34;
2642         cone[2] = 33;
2643         cone[3] = 32;
2644         cone[4] = 19;
2645         cone[5] = 22;
2646         cone[6] = 21;
2647         cone[7] = 20;
2648         PetscCall(DMPlexSetCone(dm, 5, cone));
2649         cone[0] = 32;
2650         cone[1] = 33;
2651         cone[2] = 37;
2652         cone[3] = 36;
2653         cone[4] = 22;
2654         cone[5] = 25;
2655         cone[6] = 26;
2656         cone[7] = 21;
2657         PetscCall(DMPlexSetCone(dm, 6, cone));
2658         cone[0] = 34;
2659         cone[1] = 38;
2660         cone[2] = 37;
2661         cone[3] = 33;
2662         cone[4] = 20;
2663         cone[5] = 21;
2664         cone[6] = 26;
2665         cone[7] = 28;
2666         PetscCall(DMPlexSetCone(dm, 7, cone));
2667         cone[0] = 35;
2668         cone[1] = 38;
2669         cone[2] = 34;
2670         cone[3] = 31;
2671         cone[4] = 30;
2672         cone[5] = 19;
2673         cone[6] = 20;
2674         cone[7] = 28;
2675         PetscCall(DMPlexSetCone(dm, 8, cone));
2676         cone[0] = 35;
2677         cone[1] = 31;
2678         cone[2] = 32;
2679         cone[3] = 36;
2680         cone[4] = 30;
2681         cone[5] = 25;
2682         cone[6] = 22;
2683         cone[7] = 19;
2684         PetscCall(DMPlexSetCone(dm, 9, cone));
2685 
2686         cone[0] = 19;
2687         cone[1] = 20;
2688         cone[2] = 21;
2689         cone[3] = 22;
2690         cone[4] = 15;
2691         cone[5] = 16;
2692         cone[6] = 17;
2693         cone[7] = 18;
2694         PetscCall(DMPlexSetCone(dm, 10, cone));
2695         cone[0] = 22;
2696         cone[1] = 21;
2697         cone[2] = 26;
2698         cone[3] = 25;
2699         cone[4] = 16;
2700         cone[5] = 23;
2701         cone[6] = 24;
2702         cone[7] = 17;
2703         PetscCall(DMPlexSetCone(dm, 11, cone));
2704         cone[0] = 20;
2705         cone[1] = 28;
2706         cone[2] = 26;
2707         cone[3] = 21;
2708         cone[4] = 18;
2709         cone[5] = 17;
2710         cone[6] = 24;
2711         cone[7] = 27;
2712         PetscCall(DMPlexSetCone(dm, 12, cone));
2713         cone[0] = 30;
2714         cone[1] = 28;
2715         cone[2] = 20;
2716         cone[3] = 19;
2717         cone[4] = 29;
2718         cone[5] = 15;
2719         cone[6] = 18;
2720         cone[7] = 27;
2721         PetscCall(DMPlexSetCone(dm, 13, cone));
2722         cone[0] = 30;
2723         cone[1] = 19;
2724         cone[2] = 22;
2725         cone[3] = 25;
2726         cone[4] = 29;
2727         cone[5] = 23;
2728         cone[6] = 16;
2729         cone[7] = 15;
2730         PetscCall(DMPlexSetCone(dm, 14, cone));
2731       } else {
2732         cone[0] = 5;
2733         cone[1] = 8;
2734         cone[2] = 7;
2735         cone[3] = 6;
2736         cone[4] = 9;
2737         cone[5] = 12;
2738         cone[6] = 11;
2739         cone[7] = 10;
2740         PetscCall(DMPlexSetCone(dm, 0, cone));
2741         cone[0] = 6;
2742         cone[1] = 7;
2743         cone[2] = 14;
2744         cone[3] = 13;
2745         cone[4] = 12;
2746         cone[5] = 15;
2747         cone[6] = 16;
2748         cone[7] = 11;
2749         PetscCall(DMPlexSetCone(dm, 1, cone));
2750         cone[0] = 8;
2751         cone[1] = 17;
2752         cone[2] = 14;
2753         cone[3] = 7;
2754         cone[4] = 10;
2755         cone[5] = 11;
2756         cone[6] = 16;
2757         cone[7] = 18;
2758         PetscCall(DMPlexSetCone(dm, 2, cone));
2759         cone[0] = 19;
2760         cone[1] = 17;
2761         cone[2] = 8;
2762         cone[3] = 5;
2763         cone[4] = 20;
2764         cone[5] = 9;
2765         cone[6] = 10;
2766         cone[7] = 18;
2767         PetscCall(DMPlexSetCone(dm, 3, cone));
2768         cone[0] = 19;
2769         cone[1] = 5;
2770         cone[2] = 6;
2771         cone[3] = 13;
2772         cone[4] = 20;
2773         cone[5] = 15;
2774         cone[6] = 12;
2775         cone[7] = 9;
2776         PetscCall(DMPlexSetCone(dm, 4, cone));
2777       }
2778     }
2779     PetscCall(DMPlexSymmetrize(dm));
2780     PetscCall(DMPlexStratify(dm));
2781   }
2782   /* Create cube geometry */
2783   {
2784     Vec             coordinates;
2785     PetscSection    coordSection;
2786     PetscScalar    *coords;
2787     PetscInt        coordSize, v;
2788     const PetscReal dis = 1.0 / PetscSqrtReal(2.0);
2789     const PetscReal ds2 = dis / 2.0;
2790 
2791     /* Build coordinates */
2792     PetscCall(DMGetCoordinateSection(dm, &coordSection));
2793     PetscCall(PetscSectionSetNumFields(coordSection, 1));
2794     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
2795     PetscCall(PetscSectionSetChart(coordSection, numCells, numCells + numVertices));
2796     for (v = numCells; v < numCells + numVertices; ++v) {
2797       PetscCall(PetscSectionSetDof(coordSection, v, dim));
2798       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
2799     }
2800     PetscCall(PetscSectionSetUp(coordSection));
2801     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
2802     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
2803     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
2804     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
2805     PetscCall(VecSetBlockSize(coordinates, dim));
2806     PetscCall(VecSetType(coordinates, VECSTANDARD));
2807     PetscCall(VecGetArray(coordinates, &coords));
2808     if (rank == 0) {
2809       coords[0 * dim + 0]  = -ds2;
2810       coords[0 * dim + 1]  = -ds2;
2811       coords[0 * dim + 2]  = 0.0;
2812       coords[1 * dim + 0]  = ds2;
2813       coords[1 * dim + 1]  = -ds2;
2814       coords[1 * dim + 2]  = 0.0;
2815       coords[2 * dim + 0]  = ds2;
2816       coords[2 * dim + 1]  = ds2;
2817       coords[2 * dim + 2]  = 0.0;
2818       coords[3 * dim + 0]  = -ds2;
2819       coords[3 * dim + 1]  = ds2;
2820       coords[3 * dim + 2]  = 0.0;
2821       coords[4 * dim + 0]  = -ds2;
2822       coords[4 * dim + 1]  = -ds2;
2823       coords[4 * dim + 2]  = 1.0;
2824       coords[5 * dim + 0]  = -ds2;
2825       coords[5 * dim + 1]  = ds2;
2826       coords[5 * dim + 2]  = 1.0;
2827       coords[6 * dim + 0]  = ds2;
2828       coords[6 * dim + 1]  = ds2;
2829       coords[6 * dim + 2]  = 1.0;
2830       coords[7 * dim + 0]  = ds2;
2831       coords[7 * dim + 1]  = -ds2;
2832       coords[7 * dim + 2]  = 1.0;
2833       coords[8 * dim + 0]  = dis;
2834       coords[8 * dim + 1]  = -dis;
2835       coords[8 * dim + 2]  = 0.0;
2836       coords[9 * dim + 0]  = dis;
2837       coords[9 * dim + 1]  = dis;
2838       coords[9 * dim + 2]  = 0.0;
2839       coords[10 * dim + 0] = dis;
2840       coords[10 * dim + 1] = -dis;
2841       coords[10 * dim + 2] = 1.0;
2842       coords[11 * dim + 0] = dis;
2843       coords[11 * dim + 1] = dis;
2844       coords[11 * dim + 2] = 1.0;
2845       coords[12 * dim + 0] = -dis;
2846       coords[12 * dim + 1] = dis;
2847       coords[12 * dim + 2] = 0.0;
2848       coords[13 * dim + 0] = -dis;
2849       coords[13 * dim + 1] = dis;
2850       coords[13 * dim + 2] = 1.0;
2851       coords[14 * dim + 0] = -dis;
2852       coords[14 * dim + 1] = -dis;
2853       coords[14 * dim + 2] = 0.0;
2854       coords[15 * dim + 0] = -dis;
2855       coords[15 * dim + 1] = -dis;
2856       coords[15 * dim + 2] = 1.0;
2857       if (periodicZ == DM_BOUNDARY_PERIODIC) {
2858         /* 15 31 19 */ coords[16 * dim + 0] = -ds2;
2859         coords[16 * dim + 1]                = -ds2;
2860         coords[16 * dim + 2]                = 0.5;
2861         /* 16 32 22 */ coords[17 * dim + 0] = ds2;
2862         coords[17 * dim + 1]                = -ds2;
2863         coords[17 * dim + 2]                = 0.5;
2864         /* 17 33 21 */ coords[18 * dim + 0] = ds2;
2865         coords[18 * dim + 1]                = ds2;
2866         coords[18 * dim + 2]                = 0.5;
2867         /* 18 34 20 */ coords[19 * dim + 0] = -ds2;
2868         coords[19 * dim + 1]                = ds2;
2869         coords[19 * dim + 2]                = 0.5;
2870         /* 29 35 30 */ coords[20 * dim + 0] = -dis;
2871         coords[20 * dim + 1]                = -dis;
2872         coords[20 * dim + 2]                = 0.5;
2873         /* 23 36 25 */ coords[21 * dim + 0] = dis;
2874         coords[21 * dim + 1]                = -dis;
2875         coords[21 * dim + 2]                = 0.5;
2876         /* 24 37 26 */ coords[22 * dim + 0] = dis;
2877         coords[22 * dim + 1]                = dis;
2878         coords[22 * dim + 2]                = 0.5;
2879         /* 27 38 28 */ coords[23 * dim + 0] = -dis;
2880         coords[23 * dim + 1]                = dis;
2881         coords[23 * dim + 2]                = 0.5;
2882       }
2883     }
2884     PetscCall(VecRestoreArray(coordinates, &coords));
2885     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
2886     PetscCall(VecDestroy(&coordinates));
2887   }
2888   /* Create periodicity */
2889   if (periodicZ == DM_BOUNDARY_PERIODIC || periodicZ == DM_BOUNDARY_TWIST) {
2890     PetscReal L[3]       = {-1., -1., 0.};
2891     PetscReal maxCell[3] = {-1., -1., 0.};
2892     PetscReal lower[3]   = {0.0, 0.0, 0.0};
2893     PetscReal upper[3]   = {1.0, 1.0, 1.5};
2894     PetscInt  numZCells  = 3;
2895 
2896     L[2]       = upper[2] - lower[2];
2897     maxCell[2] = 1.1 * (L[2] / numZCells);
2898     PetscCall(DMSetPeriodicity(dm, maxCell, lower, L));
2899   }
2900   {
2901     DM          cdm;
2902     PetscDS     cds;
2903     PetscScalar c[2] = {1.0, 1.0};
2904 
2905     PetscCall(DMPlexCreateCoordinateSpace(dm, 1, PETSC_TRUE, NULL));
2906     PetscCall(DMGetCoordinateDM(dm, &cdm));
2907     PetscCall(DMGetDS(cdm, &cds));
2908     PetscCall(PetscDSSetConstants(cds, 2, c));
2909   }
2910   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
2911 
2912   /* Wait for coordinate creation before doing in-place modification */
2913   PetscCall(DMPlexInterpolateInPlace_Internal(dm));
2914 
2915   char        oldprefix[PETSC_MAX_PATH_LEN];
2916   const char *prefix;
2917 
2918   PetscCall(PetscObjectGetOptionsPrefix((PetscObject)dm, &prefix));
2919   PetscCall(PetscStrncpy(oldprefix, prefix, PETSC_MAX_PATH_LEN));
2920   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)dm, "petsc_cyl_ref_"));
2921   for (PetscInt r = 0; r < PetscMax(0, Nr); ++r) {
2922     DM rdm;
2923 
2924     PetscCall(DMRefine(dm, PetscObjectComm((PetscObject)dm), &rdm));
2925     PetscCall(DMPlexReplace_Internal(dm, &rdm));
2926   }
2927   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)dm, oldprefix));
2928   PetscCall(DMPlexRemapGeometry(dm, 0.0, snapToCylinder));
2929 
2930   DMLabel         bdlabel, edgelabel;
2931   IS              faceIS;
2932   const PetscInt *faces;
2933   PetscInt        Nf;
2934 
2935   PetscCall(DMCreateLabel(dm, "marker"));
2936   PetscCall(DMGetLabel(dm, "marker", &bdlabel));
2937   PetscCall(DMCreateLabel(dm, "generatrix"));
2938   PetscCall(DMGetLabel(dm, "generatrix", &edgelabel));
2939   PetscCall(DMPlexMarkBoundaryFaces(dm, PETSC_DETERMINE, bdlabel));
2940   // Remove faces on top and bottom
2941   PetscCall(DMLabelGetStratumIS(bdlabel, 1, &faceIS));
2942   if (faceIS) {
2943     PetscCall(ISGetLocalSize(faceIS, &Nf));
2944     PetscCall(ISGetIndices(faceIS, &faces));
2945     for (PetscInt f = 0; f < Nf; ++f) {
2946       PetscReal vol, normal[3];
2947 
2948       PetscCall(DMPlexComputeCellGeometryFVM(dm, faces[f], &vol, NULL, normal));
2949       if (PetscAbsReal(normal[2]) < PETSC_SMALL) PetscCall(DMLabelSetValue(edgelabel, faces[f], 1));
2950     }
2951     PetscCall(ISRestoreIndices(faceIS, &faces));
2952     PetscCall(ISDestroy(&faceIS));
2953   }
2954   PetscCall(DMPlexLabelComplete(dm, bdlabel));
2955   PetscCall(DMPlexLabelComplete(dm, edgelabel));
2956   PetscFunctionReturn(PETSC_SUCCESS);
2957 }
2958 
2959 /*@
2960   DMPlexCreateHexCylinderMesh - Creates a mesh on the tensor product of the unit interval with the circle (cylinder) using hexahedra.
2961 
2962   Collective
2963 
2964   Input Parameters:
2965 + comm      - The communicator for the `DM` object
2966 . periodicZ - The boundary type for the Z direction
2967 - Nr        - The number of refinements to carry out
2968 
2969   Output Parameter:
2970 . dm - The `DM` object
2971 
2972   Level: beginner
2973 
2974   Note:
2975   Here is the output numbering looking from the bottom of the cylinder\:
2976 .vb
2977        17-----14
2978         |     |
2979         |  2  |
2980         |     |
2981  17-----8-----7-----14
2982   |     |     |     |
2983   |  3  |  0  |  1  |
2984   |     |     |     |
2985  19-----5-----6-----13
2986         |     |
2987         |  4  |
2988         |     |
2989        19-----13
2990 
2991  and up through the top
2992 
2993        18-----16
2994         |     |
2995         |  2  |
2996         |     |
2997  18----10----11-----16
2998   |     |     |     |
2999   |  3  |  0  |  1  |
3000   |     |     |     |
3001  20-----9----12-----15
3002         |     |
3003         |  4  |
3004         |     |
3005        20-----15
3006 .ve
3007 
3008 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
3009 @*/
3010 PetscErrorCode DMPlexCreateHexCylinderMesh(MPI_Comm comm, DMBoundaryType periodicZ, PetscInt Nr, DM *dm)
3011 {
3012   PetscFunctionBegin;
3013   PetscAssertPointer(dm, 4);
3014   PetscCall(DMCreate(comm, dm));
3015   PetscCall(DMSetType(*dm, DMPLEX));
3016   PetscCall(DMPlexCreateHexCylinderMesh_Internal(*dm, periodicZ, Nr));
3017   PetscFunctionReturn(PETSC_SUCCESS);
3018 }
3019 
3020 static PetscErrorCode DMPlexCreateWedgeCylinderMesh_Internal(DM dm, PetscInt n, PetscBool interpolate)
3021 {
3022   const PetscInt dim = 3;
3023   PetscInt       numCells, numVertices, v;
3024   PetscMPIInt    rank;
3025 
3026   PetscFunctionBegin;
3027   PetscCheck(n >= 0, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of wedges %" PetscInt_FMT " cannot be negative", n);
3028   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
3029   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
3030   PetscCall(DMSetDimension(dm, dim));
3031   /* Must create the celltype label here so that we do not automatically try to compute the types */
3032   PetscCall(DMCreateLabel(dm, "celltype"));
3033   /* Create topology */
3034   {
3035     PetscInt cone[6], c;
3036 
3037     numCells    = rank == 0 ? n : 0;
3038     numVertices = rank == 0 ? 2 * (n + 1) : 0;
3039     PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
3040     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 6));
3041     PetscCall(DMSetUp(dm));
3042     for (c = 0; c < numCells; c++) {
3043       cone[0] = c + n * 1;
3044       cone[1] = (c + 1) % n + n * 1;
3045       cone[2] = 0 + 3 * n;
3046       cone[3] = c + n * 2;
3047       cone[4] = (c + 1) % n + n * 2;
3048       cone[5] = 1 + 3 * n;
3049       PetscCall(DMPlexSetCone(dm, c, cone));
3050       PetscCall(DMPlexSetCellType(dm, c, DM_POLYTOPE_TRI_PRISM_TENSOR));
3051     }
3052     PetscCall(DMPlexSymmetrize(dm));
3053     PetscCall(DMPlexStratify(dm));
3054   }
3055   for (v = numCells; v < numCells + numVertices; ++v) PetscCall(DMPlexSetCellType(dm, v, DM_POLYTOPE_POINT));
3056   /* Create cylinder geometry */
3057   {
3058     Vec          coordinates;
3059     PetscSection coordSection;
3060     PetscScalar *coords;
3061     PetscInt     coordSize, c;
3062 
3063     /* Build coordinates */
3064     PetscCall(DMGetCoordinateSection(dm, &coordSection));
3065     PetscCall(PetscSectionSetNumFields(coordSection, 1));
3066     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
3067     PetscCall(PetscSectionSetChart(coordSection, numCells, numCells + numVertices));
3068     for (v = numCells; v < numCells + numVertices; ++v) {
3069       PetscCall(PetscSectionSetDof(coordSection, v, dim));
3070       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
3071     }
3072     PetscCall(PetscSectionSetUp(coordSection));
3073     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
3074     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
3075     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
3076     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
3077     PetscCall(VecSetBlockSize(coordinates, dim));
3078     PetscCall(VecSetType(coordinates, VECSTANDARD));
3079     PetscCall(VecGetArray(coordinates, &coords));
3080     for (c = 0; c < numCells; c++) {
3081       coords[(c + 0 * n) * dim + 0] = PetscCosReal(2.0 * c * PETSC_PI / n);
3082       coords[(c + 0 * n) * dim + 1] = PetscSinReal(2.0 * c * PETSC_PI / n);
3083       coords[(c + 0 * n) * dim + 2] = 1.0;
3084       coords[(c + 1 * n) * dim + 0] = PetscCosReal(2.0 * c * PETSC_PI / n);
3085       coords[(c + 1 * n) * dim + 1] = PetscSinReal(2.0 * c * PETSC_PI / n);
3086       coords[(c + 1 * n) * dim + 2] = 0.0;
3087     }
3088     if (rank == 0) {
3089       coords[(2 * n + 0) * dim + 0] = 0.0;
3090       coords[(2 * n + 0) * dim + 1] = 0.0;
3091       coords[(2 * n + 0) * dim + 2] = 1.0;
3092       coords[(2 * n + 1) * dim + 0] = 0.0;
3093       coords[(2 * n + 1) * dim + 1] = 0.0;
3094       coords[(2 * n + 1) * dim + 2] = 0.0;
3095     }
3096     PetscCall(VecRestoreArray(coordinates, &coords));
3097     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
3098     PetscCall(VecDestroy(&coordinates));
3099   }
3100   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
3101   /* Interpolate */
3102   if (interpolate) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
3103   PetscFunctionReturn(PETSC_SUCCESS);
3104 }
3105 
3106 /*@
3107   DMPlexCreateWedgeCylinderMesh - Creates a mesh on the tensor product of the unit interval with the circle (cylinder) using wedges.
3108 
3109   Collective
3110 
3111   Input Parameters:
3112 + comm        - The communicator for the `DM` object
3113 . n           - The number of wedges around the origin
3114 - interpolate - Create edges and faces
3115 
3116   Output Parameter:
3117 . dm - The `DM` object
3118 
3119   Level: beginner
3120 
3121 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateHexCylinderMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
3122 @*/
3123 PetscErrorCode DMPlexCreateWedgeCylinderMesh(MPI_Comm comm, PetscInt n, PetscBool interpolate, DM *dm)
3124 {
3125   PetscFunctionBegin;
3126   PetscAssertPointer(dm, 4);
3127   PetscCall(DMCreate(comm, dm));
3128   PetscCall(DMSetType(*dm, DMPLEX));
3129   PetscCall(DMPlexCreateWedgeCylinderMesh_Internal(*dm, n, interpolate));
3130   PetscFunctionReturn(PETSC_SUCCESS);
3131 }
3132 
3133 static inline PetscReal DiffNormReal(PetscInt dim, const PetscReal x[], const PetscReal y[])
3134 {
3135   PetscReal prod = 0.0;
3136   PetscInt  i;
3137   for (i = 0; i < dim; ++i) prod += PetscSqr(x[i] - y[i]);
3138   return PetscSqrtReal(prod);
3139 }
3140 
3141 static inline PetscReal DotReal(PetscInt dim, const PetscReal x[], const PetscReal y[])
3142 {
3143   PetscReal prod = 0.0;
3144   PetscInt  i;
3145   for (i = 0; i < dim; ++i) prod += x[i] * y[i];
3146   return prod;
3147 }
3148 
3149 /* The first constant is the sphere radius */
3150 static void snapToSphere(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
3151 {
3152   PetscReal r     = PetscRealPart(constants[0]);
3153   PetscReal norm2 = 0.0, fac;
3154   PetscInt  n     = uOff[1] - uOff[0], d;
3155 
3156   for (d = 0; d < n; ++d) norm2 += PetscSqr(PetscRealPart(u[d]));
3157   fac = r / PetscSqrtReal(norm2);
3158   for (d = 0; d < n; ++d) f0[d] = u[d] * fac;
3159 }
3160 
3161 static PetscErrorCode DMPlexCreateSphereMesh_Internal(DM dm, PetscInt dim, PetscBool simplex, PetscReal R)
3162 {
3163   const PetscInt embedDim = dim + 1;
3164   PetscSection   coordSection;
3165   Vec            coordinates;
3166   PetscScalar   *coords;
3167   PetscReal     *coordsIn;
3168   PetscInt       numCells, numEdges, numVerts = 0, firstVertex = 0, v, firstEdge, coordSize, d, e;
3169   PetscMPIInt    rank;
3170 
3171   PetscFunctionBegin;
3172   PetscValidLogicalCollectiveBool(dm, simplex, 3);
3173   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
3174   PetscCall(DMSetDimension(dm, dim));
3175   PetscCall(DMSetCoordinateDim(dm, dim + 1));
3176   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
3177   switch (dim) {
3178   case 1:
3179     numCells = 16;
3180     numVerts = numCells;
3181 
3182     // Build Topology
3183     PetscCall(DMPlexSetChart(dm, 0, numCells + numVerts));
3184     for (PetscInt c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, embedDim));
3185     PetscCall(DMSetUp(dm));
3186     for (PetscInt c = 0; c < numCells; ++c) {
3187       PetscInt cone[2];
3188 
3189       cone[0] = c + numCells;
3190       cone[1] = (c + 1) % numVerts + numCells;
3191       PetscCall(DMPlexSetCone(dm, c, cone));
3192     }
3193     PetscCall(DMPlexSymmetrize(dm));
3194     PetscCall(DMPlexStratify(dm));
3195     PetscCall(PetscMalloc1(numVerts * embedDim, &coordsIn));
3196     for (PetscInt v = 0; v < numVerts; ++v) {
3197       const PetscReal rad = 2. * PETSC_PI * v / numVerts;
3198 
3199       coordsIn[v * embedDim + 0] = PetscCosReal(rad);
3200       coordsIn[v * embedDim + 1] = PetscSinReal(rad);
3201     }
3202     break;
3203   case 2:
3204     if (simplex) {
3205       const PetscReal radius    = PetscSqrtReal(1 + PETSC_PHI * PETSC_PHI) / (1.0 + PETSC_PHI);
3206       const PetscReal edgeLen   = 2.0 / (1.0 + PETSC_PHI) * (R / radius);
3207       const PetscInt  degree    = 5;
3208       PetscReal       vertex[3] = {0.0, 1.0 / (1.0 + PETSC_PHI), PETSC_PHI / (1.0 + PETSC_PHI)};
3209       PetscInt        s[3]      = {1, 1, 1};
3210       PetscInt        cone[3];
3211       PetscInt       *graph;
3212 
3213       vertex[0] *= R / radius;
3214       vertex[1] *= R / radius;
3215       vertex[2] *= R / radius;
3216       numCells    = rank == 0 ? 20 : 0;
3217       numVerts    = rank == 0 ? 12 : 0;
3218       firstVertex = numCells;
3219       /* Use icosahedron, which for a R-sphere has coordinates which are all cyclic permutations of
3220 
3221            (0, \pm 1/\phi+1, \pm \phi/\phi+1)
3222 
3223          where \phi^2 - \phi - 1 = 0, meaning \phi is the golden ratio \frac{1 + \sqrt{5}}{2}. The edge
3224          length is then given by 2/(1+\phi) = 2 * 0.38197 = 0.76393.
3225       */
3226       /* Construct vertices */
3227       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
3228       if (rank == 0) {
3229         for (PetscInt p = 0, i = 0; p < embedDim; ++p) {
3230           for (s[1] = -1; s[1] < 2; s[1] += 2) {
3231             for (s[2] = -1; s[2] < 2; s[2] += 2) {
3232               for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[(d + p) % embedDim] * vertex[(d + p) % embedDim];
3233               ++i;
3234             }
3235           }
3236         }
3237       }
3238       /* Construct graph */
3239       PetscCall(PetscCalloc1(numVerts * numVerts, &graph));
3240       for (PetscInt i = 0; i < numVerts; ++i) {
3241         PetscInt k = 0;
3242         for (PetscInt j = 0; j < numVerts; ++j) {
3243           if (PetscAbsReal(DiffNormReal(embedDim, &coordsIn[i * embedDim], &coordsIn[j * embedDim]) - edgeLen) < PETSC_SMALL) {
3244             graph[i * numVerts + j] = 1;
3245             ++k;
3246           }
3247         }
3248         PetscCheck(k == degree, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid icosahedron, vertex %" PetscInt_FMT " degree %" PetscInt_FMT " != %" PetscInt_FMT, i, k, degree);
3249       }
3250       /* Build Topology */
3251       PetscCall(DMPlexSetChart(dm, 0, numCells + numVerts));
3252       for (PetscInt c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, embedDim));
3253       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
3254       /* Cells */
3255       for (PetscInt i = 0, c = 0; i < numVerts; ++i) {
3256         for (PetscInt j = 0; j < i; ++j) {
3257           for (PetscInt k = 0; k < j; ++k) {
3258             if (graph[i * numVerts + j] && graph[j * numVerts + k] && graph[k * numVerts + i]) {
3259               cone[0] = firstVertex + i;
3260               cone[1] = firstVertex + j;
3261               cone[2] = firstVertex + k;
3262               /* Check orientation */
3263               {
3264                 const PetscInt epsilon[3][3][3] = {
3265                   {{0, 0, 0},  {0, 0, 1},  {0, -1, 0}},
3266                   {{0, 0, -1}, {0, 0, 0},  {1, 0, 0} },
3267                   {{0, 1, 0},  {-1, 0, 0}, {0, 0, 0} }
3268                 };
3269                 PetscReal normal[3];
3270                 PetscInt  e, f;
3271 
3272                 for (d = 0; d < embedDim; ++d) {
3273                   normal[d] = 0.0;
3274                   for (e = 0; e < embedDim; ++e) {
3275                     for (f = 0; f < embedDim; ++f) normal[d] += epsilon[d][e][f] * (coordsIn[j * embedDim + e] - coordsIn[i * embedDim + e]) * (coordsIn[k * embedDim + f] - coordsIn[i * embedDim + f]);
3276                   }
3277                 }
3278                 if (DotReal(embedDim, normal, &coordsIn[i * embedDim]) < 0) {
3279                   PetscInt tmp = cone[1];
3280                   cone[1]      = cone[2];
3281                   cone[2]      = tmp;
3282                 }
3283               }
3284               PetscCall(DMPlexSetCone(dm, c++, cone));
3285             }
3286           }
3287         }
3288       }
3289       PetscCall(DMPlexSymmetrize(dm));
3290       PetscCall(DMPlexStratify(dm));
3291       PetscCall(PetscFree(graph));
3292     } else {
3293       /*
3294         12-21--13
3295          |     |
3296         25  4  24
3297          |     |
3298   12-25--9-16--8-24--13
3299    |     |     |     |
3300   23  5 17  0 15  3  22
3301    |     |     |     |
3302   10-20--6-14--7-19--11
3303          |     |
3304         20  1  19
3305          |     |
3306         10-18--11
3307          |     |
3308         23  2  22
3309          |     |
3310         12-21--13
3311        */
3312       PetscInt cone[4], ornt[4];
3313 
3314       numCells    = rank == 0 ? 6 : 0;
3315       numEdges    = rank == 0 ? 12 : 0;
3316       numVerts    = rank == 0 ? 8 : 0;
3317       firstVertex = numCells;
3318       firstEdge   = numCells + numVerts;
3319       /* Build Topology */
3320       PetscCall(DMPlexSetChart(dm, 0, numCells + numEdges + numVerts));
3321       for (PetscInt c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 4));
3322       for (e = firstEdge; e < firstEdge + numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
3323       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
3324       if (rank == 0) {
3325         /* Cell 0 */
3326         cone[0] = 14;
3327         cone[1] = 15;
3328         cone[2] = 16;
3329         cone[3] = 17;
3330         PetscCall(DMPlexSetCone(dm, 0, cone));
3331         ornt[0] = 0;
3332         ornt[1] = 0;
3333         ornt[2] = 0;
3334         ornt[3] = 0;
3335         PetscCall(DMPlexSetConeOrientation(dm, 0, ornt));
3336         /* Cell 1 */
3337         cone[0] = 18;
3338         cone[1] = 19;
3339         cone[2] = 14;
3340         cone[3] = 20;
3341         PetscCall(DMPlexSetCone(dm, 1, cone));
3342         ornt[0] = 0;
3343         ornt[1] = 0;
3344         ornt[2] = -1;
3345         ornt[3] = 0;
3346         PetscCall(DMPlexSetConeOrientation(dm, 1, ornt));
3347         /* Cell 2 */
3348         cone[0] = 21;
3349         cone[1] = 22;
3350         cone[2] = 18;
3351         cone[3] = 23;
3352         PetscCall(DMPlexSetCone(dm, 2, cone));
3353         ornt[0] = 0;
3354         ornt[1] = 0;
3355         ornt[2] = -1;
3356         ornt[3] = 0;
3357         PetscCall(DMPlexSetConeOrientation(dm, 2, ornt));
3358         /* Cell 3 */
3359         cone[0] = 19;
3360         cone[1] = 22;
3361         cone[2] = 24;
3362         cone[3] = 15;
3363         PetscCall(DMPlexSetCone(dm, 3, cone));
3364         ornt[0] = -1;
3365         ornt[1] = -1;
3366         ornt[2] = 0;
3367         ornt[3] = -1;
3368         PetscCall(DMPlexSetConeOrientation(dm, 3, ornt));
3369         /* Cell 4 */
3370         cone[0] = 16;
3371         cone[1] = 24;
3372         cone[2] = 21;
3373         cone[3] = 25;
3374         PetscCall(DMPlexSetCone(dm, 4, cone));
3375         ornt[0] = -1;
3376         ornt[1] = -1;
3377         ornt[2] = -1;
3378         ornt[3] = 0;
3379         PetscCall(DMPlexSetConeOrientation(dm, 4, ornt));
3380         /* Cell 5 */
3381         cone[0] = 20;
3382         cone[1] = 17;
3383         cone[2] = 25;
3384         cone[3] = 23;
3385         PetscCall(DMPlexSetCone(dm, 5, cone));
3386         ornt[0] = -1;
3387         ornt[1] = -1;
3388         ornt[2] = -1;
3389         ornt[3] = -1;
3390         PetscCall(DMPlexSetConeOrientation(dm, 5, ornt));
3391         /* Edges */
3392         cone[0] = 6;
3393         cone[1] = 7;
3394         PetscCall(DMPlexSetCone(dm, 14, cone));
3395         cone[0] = 7;
3396         cone[1] = 8;
3397         PetscCall(DMPlexSetCone(dm, 15, cone));
3398         cone[0] = 8;
3399         cone[1] = 9;
3400         PetscCall(DMPlexSetCone(dm, 16, cone));
3401         cone[0] = 9;
3402         cone[1] = 6;
3403         PetscCall(DMPlexSetCone(dm, 17, cone));
3404         cone[0] = 10;
3405         cone[1] = 11;
3406         PetscCall(DMPlexSetCone(dm, 18, cone));
3407         cone[0] = 11;
3408         cone[1] = 7;
3409         PetscCall(DMPlexSetCone(dm, 19, cone));
3410         cone[0] = 6;
3411         cone[1] = 10;
3412         PetscCall(DMPlexSetCone(dm, 20, cone));
3413         cone[0] = 12;
3414         cone[1] = 13;
3415         PetscCall(DMPlexSetCone(dm, 21, cone));
3416         cone[0] = 13;
3417         cone[1] = 11;
3418         PetscCall(DMPlexSetCone(dm, 22, cone));
3419         cone[0] = 10;
3420         cone[1] = 12;
3421         PetscCall(DMPlexSetCone(dm, 23, cone));
3422         cone[0] = 13;
3423         cone[1] = 8;
3424         PetscCall(DMPlexSetCone(dm, 24, cone));
3425         cone[0] = 12;
3426         cone[1] = 9;
3427         PetscCall(DMPlexSetCone(dm, 25, cone));
3428       }
3429       PetscCall(DMPlexSymmetrize(dm));
3430       PetscCall(DMPlexStratify(dm));
3431       /* Build coordinates */
3432       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
3433       if (rank == 0) {
3434         coordsIn[0 * embedDim + 0] = -R;
3435         coordsIn[0 * embedDim + 1] = R;
3436         coordsIn[0 * embedDim + 2] = -R;
3437         coordsIn[1 * embedDim + 0] = R;
3438         coordsIn[1 * embedDim + 1] = R;
3439         coordsIn[1 * embedDim + 2] = -R;
3440         coordsIn[2 * embedDim + 0] = R;
3441         coordsIn[2 * embedDim + 1] = -R;
3442         coordsIn[2 * embedDim + 2] = -R;
3443         coordsIn[3 * embedDim + 0] = -R;
3444         coordsIn[3 * embedDim + 1] = -R;
3445         coordsIn[3 * embedDim + 2] = -R;
3446         coordsIn[4 * embedDim + 0] = -R;
3447         coordsIn[4 * embedDim + 1] = R;
3448         coordsIn[4 * embedDim + 2] = R;
3449         coordsIn[5 * embedDim + 0] = R;
3450         coordsIn[5 * embedDim + 1] = R;
3451         coordsIn[5 * embedDim + 2] = R;
3452         coordsIn[6 * embedDim + 0] = -R;
3453         coordsIn[6 * embedDim + 1] = -R;
3454         coordsIn[6 * embedDim + 2] = R;
3455         coordsIn[7 * embedDim + 0] = R;
3456         coordsIn[7 * embedDim + 1] = -R;
3457         coordsIn[7 * embedDim + 2] = R;
3458       }
3459     }
3460     break;
3461   case 3:
3462     if (simplex) {
3463       const PetscReal edgeLen         = 1.0 / PETSC_PHI;
3464       PetscReal       vertexA[4]      = {0.5, 0.5, 0.5, 0.5};
3465       PetscReal       vertexB[4]      = {1.0, 0.0, 0.0, 0.0};
3466       PetscReal       vertexC[4]      = {0.5, 0.5 * PETSC_PHI, 0.5 / PETSC_PHI, 0.0};
3467       const PetscInt  degree          = 12;
3468       PetscInt        s[4]            = {1, 1, 1};
3469       PetscInt        evenPerm[12][4] = {
3470         {0, 1, 2, 3},
3471         {0, 2, 3, 1},
3472         {0, 3, 1, 2},
3473         {1, 0, 3, 2},
3474         {1, 2, 0, 3},
3475         {1, 3, 2, 0},
3476         {2, 0, 1, 3},
3477         {2, 1, 3, 0},
3478         {2, 3, 0, 1},
3479         {3, 0, 2, 1},
3480         {3, 1, 0, 2},
3481         {3, 2, 1, 0}
3482       };
3483       PetscInt  cone[4];
3484       PetscInt *graph, p, i, j, k, l;
3485 
3486       vertexA[0] *= R;
3487       vertexA[1] *= R;
3488       vertexA[2] *= R;
3489       vertexA[3] *= R;
3490       vertexB[0] *= R;
3491       vertexB[1] *= R;
3492       vertexB[2] *= R;
3493       vertexB[3] *= R;
3494       vertexC[0] *= R;
3495       vertexC[1] *= R;
3496       vertexC[2] *= R;
3497       vertexC[3] *= R;
3498       numCells    = rank == 0 ? 600 : 0;
3499       numVerts    = rank == 0 ? 120 : 0;
3500       firstVertex = numCells;
3501       /* Use the 600-cell, which for a unit sphere has coordinates which are
3502 
3503            1/2 (\pm 1, \pm 1,    \pm 1, \pm 1)                          16
3504                (\pm 1,    0,       0,      0)  all cyclic permutations   8
3505            1/2 (\pm 1, \pm phi, \pm 1/phi, 0)  all even permutations    96
3506 
3507          where \phi^2 - \phi - 1 = 0, meaning \phi is the golden ratio \frac{1 + \sqrt{5}}{2}. The edge
3508          length is then given by 1/\phi = 0.61803.
3509 
3510          http://buzzard.pugetsound.edu/sage-practice/ch03s03.html
3511          http://mathworld.wolfram.com/600-Cell.html
3512       */
3513       /* Construct vertices */
3514       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
3515       i = 0;
3516       if (rank == 0) {
3517         for (s[0] = -1; s[0] < 2; s[0] += 2) {
3518           for (s[1] = -1; s[1] < 2; s[1] += 2) {
3519             for (s[2] = -1; s[2] < 2; s[2] += 2) {
3520               for (s[3] = -1; s[3] < 2; s[3] += 2) {
3521                 for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[d] * vertexA[d];
3522                 ++i;
3523               }
3524             }
3525           }
3526         }
3527         for (p = 0; p < embedDim; ++p) {
3528           s[1] = s[2] = s[3] = 1;
3529           for (s[0] = -1; s[0] < 2; s[0] += 2) {
3530             for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[(d + p) % embedDim] * vertexB[(d + p) % embedDim];
3531             ++i;
3532           }
3533         }
3534         for (p = 0; p < 12; ++p) {
3535           s[3] = 1;
3536           for (s[0] = -1; s[0] < 2; s[0] += 2) {
3537             for (s[1] = -1; s[1] < 2; s[1] += 2) {
3538               for (s[2] = -1; s[2] < 2; s[2] += 2) {
3539                 for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[evenPerm[p][d]] * vertexC[evenPerm[p][d]];
3540                 ++i;
3541               }
3542             }
3543           }
3544         }
3545       }
3546       PetscCheck(i == numVerts, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid 600-cell, vertices %" PetscInt_FMT " != %" PetscInt_FMT, i, numVerts);
3547       /* Construct graph */
3548       PetscCall(PetscCalloc1(numVerts * numVerts, &graph));
3549       for (i = 0; i < numVerts; ++i) {
3550         for (j = 0, k = 0; j < numVerts; ++j) {
3551           if (PetscAbsReal(DiffNormReal(embedDim, &coordsIn[i * embedDim], &coordsIn[j * embedDim]) - edgeLen) < PETSC_SMALL) {
3552             graph[i * numVerts + j] = 1;
3553             ++k;
3554           }
3555         }
3556         PetscCheck(k == degree, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid 600-cell, vertex %" PetscInt_FMT " degree %" PetscInt_FMT " != %" PetscInt_FMT, i, k, degree);
3557       }
3558       /* Build Topology */
3559       PetscCall(DMPlexSetChart(dm, 0, numCells + numVerts));
3560       for (PetscInt c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, embedDim));
3561       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
3562       /* Cells */
3563       if (rank == 0) {
3564         for (PetscInt i = 0, c = 0; i < numVerts; ++i) {
3565           for (j = 0; j < i; ++j) {
3566             for (k = 0; k < j; ++k) {
3567               for (l = 0; l < k; ++l) {
3568                 if (graph[i * numVerts + j] && graph[j * numVerts + k] && graph[k * numVerts + i] && graph[l * numVerts + i] && graph[l * numVerts + j] && graph[l * numVerts + k]) {
3569                   cone[0] = firstVertex + i;
3570                   cone[1] = firstVertex + j;
3571                   cone[2] = firstVertex + k;
3572                   cone[3] = firstVertex + l;
3573                   /* Check orientation: https://ef.gy/linear-algebra:normal-vectors-in-higher-dimensional-spaces */
3574                   {
3575                     const PetscInt epsilon[4][4][4][4] = {
3576                       {{{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}},  {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 1}, {0, 0, -1, 0}}, {{0, 0, 0, 0}, {0, 0, 0, -1}, {0, 0, 0, 0}, {0, 1, 0, 0}}, {{0, 0, 0, 0}, {0, 0, 1, 0}, {0, -1, 0, 0}, {0, 0, 0, 0}}},
3577 
3578                       {{{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, -1}, {0, 0, 1, 0}}, {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}},  {{0, 0, 0, 1}, {0, 0, 0, 0}, {0, 0, 0, 0}, {-1, 0, 0, 0}}, {{0, 0, -1, 0}, {0, 0, 0, 0}, {1, 0, 0, 0}, {0, 0, 0, 0}}},
3579 
3580                       {{{0, 0, 0, 0}, {0, 0, 0, 1}, {0, 0, 0, 0}, {0, -1, 0, 0}}, {{0, 0, 0, -1}, {0, 0, 0, 0}, {0, 0, 0, 0}, {1, 0, 0, 0}}, {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}},  {{0, 1, 0, 0}, {-1, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}}},
3581 
3582                       {{{0, 0, 0, 0}, {0, 0, -1, 0}, {0, 1, 0, 0}, {0, 0, 0, 0}}, {{0, 0, 1, 0}, {0, 0, 0, 0}, {-1, 0, 0, 0}, {0, 0, 0, 0}}, {{0, -1, 0, 0}, {1, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}}, {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}} }
3583                     };
3584                     PetscReal normal[4];
3585                     PetscInt  e, f, g;
3586 
3587                     for (d = 0; d < embedDim; ++d) {
3588                       normal[d] = 0.0;
3589                       for (e = 0; e < embedDim; ++e) {
3590                         for (f = 0; f < embedDim; ++f) {
3591                           for (g = 0; g < embedDim; ++g) {
3592                             normal[d] += epsilon[d][e][f][g] * (coordsIn[j * embedDim + e] - coordsIn[i * embedDim + e]) * (coordsIn[k * embedDim + f] - coordsIn[i * embedDim + f]) * (coordsIn[l * embedDim + f] - coordsIn[i * embedDim + f]);
3593                           }
3594                         }
3595                       }
3596                     }
3597                     if (DotReal(embedDim, normal, &coordsIn[i * embedDim]) < 0) {
3598                       PetscInt tmp = cone[1];
3599                       cone[1]      = cone[2];
3600                       cone[2]      = tmp;
3601                     }
3602                   }
3603                   PetscCall(DMPlexSetCone(dm, c++, cone));
3604                 }
3605               }
3606             }
3607           }
3608         }
3609       }
3610       PetscCall(DMPlexSymmetrize(dm));
3611       PetscCall(DMPlexStratify(dm));
3612       PetscCall(PetscFree(graph));
3613     }
3614     break;
3615   default:
3616     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Unsupported dimension for sphere: %" PetscInt_FMT, dim);
3617   }
3618   /* Create coordinates */
3619   PetscCall(DMGetCoordinateSection(dm, &coordSection));
3620   PetscCall(PetscSectionSetNumFields(coordSection, 1));
3621   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, embedDim));
3622   PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numVerts));
3623   for (v = firstVertex; v < firstVertex + numVerts; ++v) {
3624     PetscCall(PetscSectionSetDof(coordSection, v, embedDim));
3625     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, embedDim));
3626   }
3627   PetscCall(PetscSectionSetUp(coordSection));
3628   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
3629   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
3630   PetscCall(VecSetBlockSize(coordinates, embedDim));
3631   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
3632   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
3633   PetscCall(VecSetType(coordinates, VECSTANDARD));
3634   PetscCall(VecGetArray(coordinates, &coords));
3635   for (v = 0; v < numVerts; ++v)
3636     for (d = 0; d < embedDim; ++d) coords[v * embedDim + d] = coordsIn[v * embedDim + d];
3637   PetscCall(VecRestoreArray(coordinates, &coords));
3638   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
3639   PetscCall(VecDestroy(&coordinates));
3640   PetscCall(PetscFree(coordsIn));
3641   {
3642     DM          cdm;
3643     PetscDS     cds;
3644     PetscScalar c = R;
3645 
3646     PetscCall(DMPlexCreateCoordinateSpace(dm, 1, PETSC_TRUE, snapToSphere));
3647     PetscCall(DMGetCoordinateDM(dm, &cdm));
3648     PetscCall(DMGetDS(cdm, &cds));
3649     PetscCall(PetscDSSetConstants(cds, 1, &c));
3650   }
3651   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
3652   /* Wait for coordinate creation before doing in-place modification */
3653   if (simplex) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
3654   PetscFunctionReturn(PETSC_SUCCESS);
3655 }
3656 
3657 typedef void (*TPSEvaluateFunc)(const PetscReal[], PetscReal *, PetscReal[], PetscReal (*)[3]);
3658 
3659 /*
3660  The Schwarz P implicit surface is
3661 
3662      f(x) = cos(x0) + cos(x1) + cos(x2) = 0
3663 */
3664 static void TPSEvaluate_SchwarzP(const PetscReal y[3], PetscReal *f, PetscReal grad[], PetscReal (*hess)[3])
3665 {
3666   PetscReal c[3] = {PetscCosReal(y[0] * PETSC_PI), PetscCosReal(y[1] * PETSC_PI), PetscCosReal(y[2] * PETSC_PI)};
3667   PetscReal g[3] = {-PetscSinReal(y[0] * PETSC_PI), -PetscSinReal(y[1] * PETSC_PI), -PetscSinReal(y[2] * PETSC_PI)};
3668   f[0]           = c[0] + c[1] + c[2];
3669   for (PetscInt i = 0; i < 3; i++) {
3670     grad[i] = PETSC_PI * g[i];
3671     for (PetscInt j = 0; j < 3; j++) hess[i][j] = (i == j) ? -PetscSqr(PETSC_PI) * c[i] : 0.;
3672   }
3673 }
3674 
3675 // u[] is a tentative normal on input. Replace with the implicit function gradient in the same direction
3676 static PetscErrorCode TPSExtrudeNormalFunc_SchwarzP(PetscInt dim, PetscReal time, const PetscReal x[], PetscInt r, PetscScalar u[], void *ctx)
3677 {
3678   for (PetscInt i = 0; i < 3; i++) u[i] = -PETSC_PI * PetscSinReal(x[i] * PETSC_PI);
3679   return PETSC_SUCCESS;
3680 }
3681 
3682 /*
3683  The Gyroid implicit surface is
3684 
3685  f(x,y,z) = sin(pi * x) * cos (pi * (y + 1/2))  + sin(pi * (y + 1/2)) * cos(pi * (z + 1/4)) + sin(pi * (z + 1/4)) * cos(pi * x)
3686 
3687 */
3688 static void TPSEvaluate_Gyroid(const PetscReal y[3], PetscReal *f, PetscReal grad[], PetscReal (*hess)[3])
3689 {
3690   PetscReal s[3] = {PetscSinReal(PETSC_PI * y[0]), PetscSinReal(PETSC_PI * (y[1] + .5)), PetscSinReal(PETSC_PI * (y[2] + .25))};
3691   PetscReal c[3] = {PetscCosReal(PETSC_PI * y[0]), PetscCosReal(PETSC_PI * (y[1] + .5)), PetscCosReal(PETSC_PI * (y[2] + .25))};
3692   f[0]           = s[0] * c[1] + s[1] * c[2] + s[2] * c[0];
3693   grad[0]        = PETSC_PI * (c[0] * c[1] - s[2] * s[0]);
3694   grad[1]        = PETSC_PI * (c[1] * c[2] - s[0] * s[1]);
3695   grad[2]        = PETSC_PI * (c[2] * c[0] - s[1] * s[2]);
3696   hess[0][0]     = -PetscSqr(PETSC_PI) * (s[0] * c[1] + s[2] * c[0]);
3697   hess[0][1]     = -PetscSqr(PETSC_PI) * (c[0] * s[1]);
3698   hess[0][2]     = -PetscSqr(PETSC_PI) * (c[2] * s[0]);
3699   hess[1][0]     = -PetscSqr(PETSC_PI) * (s[1] * c[2] + s[0] * c[1]);
3700   hess[1][1]     = -PetscSqr(PETSC_PI) * (c[1] * s[2]);
3701   hess[2][2]     = -PetscSqr(PETSC_PI) * (c[0] * s[1]);
3702   hess[2][0]     = -PetscSqr(PETSC_PI) * (s[2] * c[0] + s[1] * c[2]);
3703   hess[2][1]     = -PetscSqr(PETSC_PI) * (c[2] * s[0]);
3704   hess[2][2]     = -PetscSqr(PETSC_PI) * (c[1] * s[2]);
3705 }
3706 
3707 // u[] is a tentative normal on input. Replace with the implicit function gradient in the same direction
3708 static PetscErrorCode TPSExtrudeNormalFunc_Gyroid(PetscInt dim, PetscReal time, const PetscReal x[], PetscInt r, PetscScalar u[], void *ctx)
3709 {
3710   PetscReal s[3] = {PetscSinReal(PETSC_PI * x[0]), PetscSinReal(PETSC_PI * (x[1] + .5)), PetscSinReal(PETSC_PI * (x[2] + .25))};
3711   PetscReal c[3] = {PetscCosReal(PETSC_PI * x[0]), PetscCosReal(PETSC_PI * (x[1] + .5)), PetscCosReal(PETSC_PI * (x[2] + .25))};
3712   u[0]           = PETSC_PI * (c[0] * c[1] - s[2] * s[0]);
3713   u[1]           = PETSC_PI * (c[1] * c[2] - s[0] * s[1]);
3714   u[2]           = PETSC_PI * (c[2] * c[0] - s[1] * s[2]);
3715   return PETSC_SUCCESS;
3716 }
3717 
3718 /*
3719    We wish to solve
3720 
3721          min_y || y - x ||^2  subject to f(y) = 0
3722 
3723    Let g(y) = grad(f).  The minimization problem is equivalent to asking to satisfy
3724    f(y) = 0 and (y-x) is parallel to g(y).  We do this by using Householder QR to obtain a basis for the
3725    tangent space and ask for both components in the tangent space to be zero.
3726 
3727    Take g to be a column vector and compute the "full QR" factorization Q R = g,
3728    where Q = I - 2 n n^T is a symmetric orthogonal matrix.
3729    The first column of Q is parallel to g so the remaining two columns span the null space.
3730    Let Qn = Q[:,1:] be those remaining columns.  Then Qn Qn^T is an orthogonal projector into the tangent space.
3731    Since Q is symmetric, this is equivalent to multiplying by Q and taking the last two entries.
3732    In total, we have a system of 3 equations in 3 unknowns:
3733 
3734      f(y) = 0                       1 equation
3735      Qn^T (y - x) = 0               2 equations
3736 
3737    Here, we compute the residual and Jacobian of this system.
3738 */
3739 static void TPSNearestPointResJac(TPSEvaluateFunc feval, const PetscScalar x[], const PetscScalar y[], PetscScalar res[], PetscScalar J[])
3740 {
3741   PetscReal yreal[3] = {PetscRealPart(y[0]), PetscRealPart(y[1]), PetscRealPart(y[2])};
3742   PetscReal d[3]     = {PetscRealPart(y[0] - x[0]), PetscRealPart(y[1] - x[1]), PetscRealPart(y[2] - x[2])};
3743   PetscReal f, grad[3], n[3], norm, norm_y[3], nd, nd_y[3], sign;
3744   PetscReal n_y[3][3] = {
3745     {0, 0, 0},
3746     {0, 0, 0},
3747     {0, 0, 0}
3748   };
3749 
3750   feval(yreal, &f, grad, n_y);
3751 
3752   for (PetscInt i = 0; i < 3; i++) n[i] = grad[i];
3753   norm = PetscSqrtReal(PetscSqr(n[0]) + PetscSqr(n[1]) + PetscSqr(n[2]));
3754   for (PetscInt i = 0; i < 3; i++) norm_y[i] = 1. / norm * n[i] * n_y[i][i];
3755 
3756   // Define the Householder reflector
3757   sign = n[0] >= 0 ? 1. : -1.;
3758   n[0] += norm * sign;
3759   for (PetscInt i = 0; i < 3; i++) n_y[0][i] += norm_y[i] * sign;
3760 
3761   norm      = PetscSqrtReal(PetscSqr(n[0]) + PetscSqr(n[1]) + PetscSqr(n[2]));
3762   norm_y[0] = 1. / norm * (n[0] * n_y[0][0]);
3763   norm_y[1] = 1. / norm * (n[0] * n_y[0][1] + n[1] * n_y[1][1]);
3764   norm_y[2] = 1. / norm * (n[0] * n_y[0][2] + n[2] * n_y[2][2]);
3765 
3766   for (PetscInt i = 0; i < 3; i++) {
3767     n[i] /= norm;
3768     for (PetscInt j = 0; j < 3; j++) {
3769       // note that n[i] is n_old[i]/norm when executing the code below
3770       n_y[i][j] = n_y[i][j] / norm - n[i] / norm * norm_y[j];
3771     }
3772   }
3773 
3774   nd = n[0] * d[0] + n[1] * d[1] + n[2] * d[2];
3775   for (PetscInt i = 0; i < 3; i++) nd_y[i] = n[i] + n_y[0][i] * d[0] + n_y[1][i] * d[1] + n_y[2][i] * d[2];
3776 
3777   res[0] = f;
3778   res[1] = d[1] - 2 * n[1] * nd;
3779   res[2] = d[2] - 2 * n[2] * nd;
3780   // J[j][i] is J_{ij} (column major)
3781   for (PetscInt j = 0; j < 3; j++) {
3782     J[0 + j * 3] = grad[j];
3783     J[1 + j * 3] = (j == 1) * 1. - 2 * (n_y[1][j] * nd + n[1] * nd_y[j]);
3784     J[2 + j * 3] = (j == 2) * 1. - 2 * (n_y[2][j] * nd + n[2] * nd_y[j]);
3785   }
3786 }
3787 
3788 /*
3789    Project x to the nearest point on the implicit surface using Newton's method.
3790 */
3791 static PetscErrorCode TPSNearestPoint(TPSEvaluateFunc feval, PetscScalar x[])
3792 {
3793   PetscScalar y[3] = {x[0], x[1], x[2]}; // Initial guess
3794 
3795   PetscFunctionBegin;
3796   for (PetscInt iter = 0; iter < 10; iter++) {
3797     PetscScalar res[3], J[9];
3798     PetscReal   resnorm;
3799     TPSNearestPointResJac(feval, x, y, res, J);
3800     resnorm = PetscSqrtReal(PetscSqr(PetscRealPart(res[0])) + PetscSqr(PetscRealPart(res[1])) + PetscSqr(PetscRealPart(res[2])));
3801     if (0) { // Turn on this monitor if you need to confirm quadratic convergence
3802       PetscCall(PetscPrintf(PETSC_COMM_SELF, "[%" PetscInt_FMT "] res [%g %g %g]\n", iter, (double)PetscRealPart(res[0]), (double)PetscRealPart(res[1]), (double)PetscRealPart(res[2])));
3803     }
3804     if (resnorm < PETSC_SMALL) break;
3805 
3806     // Take the Newton step
3807     PetscCall(PetscKernel_A_gets_inverse_A_3(J, 0., PETSC_FALSE, NULL));
3808     PetscKernel_v_gets_v_minus_A_times_w_3(y, J, res);
3809   }
3810   for (PetscInt i = 0; i < 3; i++) x[i] = y[i];
3811   PetscFunctionReturn(PETSC_SUCCESS);
3812 }
3813 
3814 const char *const DMPlexTPSTypes[] = {"SCHWARZ_P", "GYROID", "DMPlexTPSType", "DMPLEX_TPS_", NULL};
3815 
3816 static PetscErrorCode DMPlexCreateTPSMesh_Internal(DM dm, DMPlexTPSType tpstype, const PetscInt extent[], const DMBoundaryType periodic[], PetscBool tps_distribute, PetscInt refinements, PetscInt layers, PetscReal thickness)
3817 {
3818   PetscMPIInt rank;
3819   PetscInt    topoDim = 2, spaceDim = 3, numFaces = 0, numVertices = 0, numEdges = 0;
3820   PetscInt (*edges)[2] = NULL, *edgeSets = NULL;
3821   PetscInt           *cells_flat = NULL;
3822   PetscReal          *vtxCoords  = NULL;
3823   TPSEvaluateFunc     evalFunc   = NULL;
3824   PetscSimplePointFn *normalFunc = NULL;
3825   DMLabel             label;
3826 
3827   PetscFunctionBegin;
3828   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
3829   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
3830   PetscCheck((layers != 0) ^ (thickness == 0.), PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_INCOMP, "Layers %" PetscInt_FMT " must be nonzero iff thickness %g is nonzero", layers, (double)thickness);
3831   switch (tpstype) {
3832   case DMPLEX_TPS_SCHWARZ_P:
3833     PetscCheck(!periodic || (periodic[0] == DM_BOUNDARY_NONE && periodic[1] == DM_BOUNDARY_NONE && periodic[2] == DM_BOUNDARY_NONE), PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Schwarz P does not support periodic meshes");
3834     if (rank == 0) {
3835       PetscInt (*cells)[6][4][4] = NULL; // [junction, junction-face, cell, conn]
3836       PetscInt  Njunctions = 0, Ncuts = 0, Npipes[3], vcount;
3837       PetscReal L = 1;
3838 
3839       Npipes[0]   = (extent[0] + 1) * extent[1] * extent[2];
3840       Npipes[1]   = extent[0] * (extent[1] + 1) * extent[2];
3841       Npipes[2]   = extent[0] * extent[1] * (extent[2] + 1);
3842       Njunctions  = extent[0] * extent[1] * extent[2];
3843       Ncuts       = 2 * (extent[0] * extent[1] + extent[1] * extent[2] + extent[2] * extent[0]);
3844       numVertices = 4 * (Npipes[0] + Npipes[1] + Npipes[2]) + 8 * Njunctions;
3845       PetscCall(PetscMalloc1(3 * numVertices, &vtxCoords));
3846       PetscCall(PetscMalloc1(Njunctions, &cells));
3847       PetscCall(PetscMalloc1(Ncuts * 4, &edges));
3848       PetscCall(PetscMalloc1(Ncuts * 4, &edgeSets));
3849       // x-normal pipes
3850       vcount = 0;
3851       for (PetscInt i = 0; i < extent[0] + 1; i++) {
3852         for (PetscInt j = 0; j < extent[1]; j++) {
3853           for (PetscInt k = 0; k < extent[2]; k++) {
3854             for (PetscInt l = 0; l < 4; l++) {
3855               vtxCoords[vcount++] = (2 * i - 1) * L;
3856               vtxCoords[vcount++] = 2 * j * L + PetscCosReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
3857               vtxCoords[vcount++] = 2 * k * L + PetscSinReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
3858             }
3859           }
3860         }
3861       }
3862       // y-normal pipes
3863       for (PetscInt i = 0; i < extent[0]; i++) {
3864         for (PetscInt j = 0; j < extent[1] + 1; j++) {
3865           for (PetscInt k = 0; k < extent[2]; k++) {
3866             for (PetscInt l = 0; l < 4; l++) {
3867               vtxCoords[vcount++] = 2 * i * L + PetscSinReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
3868               vtxCoords[vcount++] = (2 * j - 1) * L;
3869               vtxCoords[vcount++] = 2 * k * L + PetscCosReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
3870             }
3871           }
3872         }
3873       }
3874       // z-normal pipes
3875       for (PetscInt i = 0; i < extent[0]; i++) {
3876         for (PetscInt j = 0; j < extent[1]; j++) {
3877           for (PetscInt k = 0; k < extent[2] + 1; k++) {
3878             for (PetscInt l = 0; l < 4; l++) {
3879               vtxCoords[vcount++] = 2 * i * L + PetscCosReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
3880               vtxCoords[vcount++] = 2 * j * L + PetscSinReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
3881               vtxCoords[vcount++] = (2 * k - 1) * L;
3882             }
3883           }
3884         }
3885       }
3886       // junctions
3887       for (PetscInt i = 0; i < extent[0]; i++) {
3888         for (PetscInt j = 0; j < extent[1]; j++) {
3889           for (PetscInt k = 0; k < extent[2]; k++) {
3890             const PetscInt J = (i * extent[1] + j) * extent[2] + k, Jvoff = (Npipes[0] + Npipes[1] + Npipes[2]) * 4 + J * 8;
3891             PetscCheck(vcount / 3 == Jvoff, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Unexpected vertex count");
3892             for (PetscInt ii = 0; ii < 2; ii++) {
3893               for (PetscInt jj = 0; jj < 2; jj++) {
3894                 for (PetscInt kk = 0; kk < 2; kk++) {
3895                   double Ls           = (1 - sqrt(2) / 4) * L;
3896                   vtxCoords[vcount++] = 2 * i * L + (2 * ii - 1) * Ls;
3897                   vtxCoords[vcount++] = 2 * j * L + (2 * jj - 1) * Ls;
3898                   vtxCoords[vcount++] = 2 * k * L + (2 * kk - 1) * Ls;
3899                 }
3900               }
3901             }
3902             const PetscInt jfaces[3][2][4] = {
3903               {{3, 1, 0, 2}, {7, 5, 4, 6}}, // x-aligned
3904               {{5, 4, 0, 1}, {7, 6, 2, 3}}, // y-aligned
3905               {{6, 2, 0, 4}, {7, 3, 1, 5}}  // z-aligned
3906             };
3907             const PetscInt pipe_lo[3] = {// vertex numbers of pipes
3908                                          ((i * extent[1] + j) * extent[2] + k) * 4, ((i * (extent[1] + 1) + j) * extent[2] + k + Npipes[0]) * 4, ((i * extent[1] + j) * (extent[2] + 1) + k + Npipes[0] + Npipes[1]) * 4};
3909             const PetscInt pipe_hi[3] = {// vertex numbers of pipes
3910                                          (((i + 1) * extent[1] + j) * extent[2] + k) * 4, ((i * (extent[1] + 1) + j + 1) * extent[2] + k + Npipes[0]) * 4, ((i * extent[1] + j) * (extent[2] + 1) + k + 1 + Npipes[0] + Npipes[1]) * 4};
3911             for (PetscInt dir = 0; dir < 3; dir++) { // x,y,z
3912               const PetscInt ijk[3] = {i, j, k};
3913               for (PetscInt l = 0; l < 4; l++) { // rotations
3914                 cells[J][dir * 2 + 0][l][0] = pipe_lo[dir] + l;
3915                 cells[J][dir * 2 + 0][l][1] = Jvoff + jfaces[dir][0][l];
3916                 cells[J][dir * 2 + 0][l][2] = Jvoff + jfaces[dir][0][(l - 1 + 4) % 4];
3917                 cells[J][dir * 2 + 0][l][3] = pipe_lo[dir] + (l - 1 + 4) % 4;
3918                 cells[J][dir * 2 + 1][l][0] = Jvoff + jfaces[dir][1][l];
3919                 cells[J][dir * 2 + 1][l][1] = pipe_hi[dir] + l;
3920                 cells[J][dir * 2 + 1][l][2] = pipe_hi[dir] + (l - 1 + 4) % 4;
3921                 cells[J][dir * 2 + 1][l][3] = Jvoff + jfaces[dir][1][(l - 1 + 4) % 4];
3922                 if (ijk[dir] == 0) {
3923                   edges[numEdges][0] = pipe_lo[dir] + l;
3924                   edges[numEdges][1] = pipe_lo[dir] + (l + 1) % 4;
3925                   edgeSets[numEdges] = dir * 2 + 1;
3926                   numEdges++;
3927                 }
3928                 if (ijk[dir] + 1 == extent[dir]) {
3929                   edges[numEdges][0] = pipe_hi[dir] + l;
3930                   edges[numEdges][1] = pipe_hi[dir] + (l + 1) % 4;
3931                   edgeSets[numEdges] = dir * 2 + 2;
3932                   numEdges++;
3933                 }
3934               }
3935             }
3936           }
3937         }
3938       }
3939       PetscCheck(numEdges == Ncuts * 4, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Edge count %" PetscInt_FMT " incompatible with number of cuts %" PetscInt_FMT, numEdges, Ncuts);
3940       numFaces   = 24 * Njunctions;
3941       cells_flat = cells[0][0][0];
3942     }
3943     evalFunc   = TPSEvaluate_SchwarzP;
3944     normalFunc = TPSExtrudeNormalFunc_SchwarzP;
3945     break;
3946   case DMPLEX_TPS_GYROID:
3947     if (rank == 0) {
3948       // This is a coarse mesh approximation of the gyroid shifted to being the zero of the level set
3949       //
3950       //     sin(pi*x)*cos(pi*(y+1/2)) + sin(pi*(y+1/2))*cos(pi*(z+1/4)) + sin(pi*(z+1/4))*cos(x)
3951       //
3952       // on the cell [0,2]^3.
3953       //
3954       // Think about dividing that cell into four columns, and focus on the column [0,1]x[0,1]x[0,2].
3955       // If you looked at the gyroid in that column at different slices of z you would see that it kind of spins
3956       // like a boomerang:
3957       //
3958       //     z = 0          z = 1/4        z = 1/2        z = 3/4     //
3959       //     -----          -------        -------        -------     //
3960       //                                                              //
3961       //     +       +      +       +      +       +      +   \   +   //
3962       //      \                                   /            \      //
3963       //       \            `-_   _-'            /              }     //
3964       //        *-_            `-'            _-'              /      //
3965       //     +     `-+      +       +      +-'     +      +   /   +   //
3966       //                                                              //
3967       //                                                              //
3968       //     z = 1          z = 5/4        z = 3/2        z = 7/4     //
3969       //     -----          -------        -------        -------     //
3970       //                                                              //
3971       //     +-_     +      +       +      +     _-+      +   /   +   //
3972       //        `-_            _-_            _-`            /        //
3973       //           \        _-'   `-_        /              {         //
3974       //            \                       /                \        //
3975       //     +       +      +       +      +       +      +   \   +   //
3976       //
3977       //
3978       // This course mesh approximates each of these slices by two line segments,
3979       // and then connects the segments in consecutive layers with quadrilateral faces.
3980       // All of the end points of the segments are multiples of 1/4 except for the
3981       // point * in the picture for z = 0 above and the similar points in other layers.
3982       // That point is at (gamma, gamma, 0), where gamma is calculated below.
3983       //
3984       // The column  [1,2]x[1,2]x[0,2] looks the same as this column;
3985       // The columns [1,2]x[0,1]x[0,2] and [0,1]x[1,2]x[0,2] are mirror images.
3986       //
3987       // As for how this method turned into the names given to the vertices:
3988       // that was not systematic, it was just the way it worked out in my handwritten notes.
3989 
3990       PetscInt facesPerBlock = 64;
3991       PetscInt vertsPerBlock = 56;
3992       PetscInt extentPlus[3];
3993       PetscInt numBlocks, numBlocksPlus;
3994       const PetscInt A = 0, B = 1, C = 2, D = 3, E = 4, F = 5, G = 6, H = 7, II = 8, J = 9, K = 10, L = 11, M = 12, N = 13, O = 14, P = 15, Q = 16, R = 17, S = 18, T = 19, U = 20, V = 21, W = 22, X = 23, Y = 24, Z = 25, Ap = 26, Bp = 27, Cp = 28, Dp = 29, Ep = 30, Fp = 31, Gp = 32, Hp = 33, Ip = 34, Jp = 35, Kp = 36, Lp = 37, Mp = 38, Np = 39, Op = 40, Pp = 41, Qp = 42, Rp = 43, Sp = 44, Tp = 45, Up = 46, Vp = 47, Wp = 48, Xp = 49, Yp = 50, Zp = 51, Aq = 52, Bq = 53, Cq = 54, Dq = 55;
3995       const PetscInt pattern[64][4] = {
3996         /* face to vertex within the coarse discretization of a single gyroid block */
3997         /* layer 0 */
3998         {A,           C,           K,           G          },
3999         {C,           B,           II,          K          },
4000         {D,           A,           H,           L          },
4001         {B + 56 * 1,  D,           L,           J          },
4002         {E,           B + 56 * 1,  J,           N          },
4003         {A + 56 * 2,  E,           N,           H + 56 * 2 },
4004         {F,           A + 56 * 2,  G + 56 * 2,  M          },
4005         {B,           F,           M,           II         },
4006         /* layer 1 */
4007         {G,           K,           Q,           O          },
4008         {K,           II,          P,           Q          },
4009         {L,           H,           O + 56 * 1,  R          },
4010         {J,           L,           R,           P          },
4011         {N,           J,           P,           S          },
4012         {H + 56 * 2,  N,           S,           O + 56 * 3 },
4013         {M,           G + 56 * 2,  O + 56 * 2,  T          },
4014         {II,          M,           T,           P          },
4015         /* layer 2 */
4016         {O,           Q,           Y,           U          },
4017         {Q,           P,           W,           Y          },
4018         {R,           O + 56 * 1,  U + 56 * 1,  Ap         },
4019         {P,           R,           Ap,          W          },
4020         {S,           P,           X,           Bp         },
4021         {O + 56 * 3,  S,           Bp,          V + 56 * 1 },
4022         {T,           O + 56 * 2,  V,           Z          },
4023         {P,           T,           Z,           X          },
4024         /* layer 3 */
4025         {U,           Y,           Ep,          Dp         },
4026         {Y,           W,           Cp,          Ep         },
4027         {Ap,          U + 56 * 1,  Dp + 56 * 1, Gp         },
4028         {W,           Ap,          Gp,          Cp         },
4029         {Bp,          X,           Cp + 56 * 2, Fp         },
4030         {V + 56 * 1,  Bp,          Fp,          Dp + 56 * 1},
4031         {Z,           V,           Dp,          Hp         },
4032         {X,           Z,           Hp,          Cp + 56 * 2},
4033         /* layer 4 */
4034         {Dp,          Ep,          Mp,          Kp         },
4035         {Ep,          Cp,          Ip,          Mp         },
4036         {Gp,          Dp + 56 * 1, Lp,          Np         },
4037         {Cp,          Gp,          Np,          Jp         },
4038         {Fp,          Cp + 56 * 2, Jp + 56 * 2, Pp         },
4039         {Dp + 56 * 1, Fp,          Pp,          Lp         },
4040         {Hp,          Dp,          Kp,          Op         },
4041         {Cp + 56 * 2, Hp,          Op,          Ip + 56 * 2},
4042         /* layer 5 */
4043         {Kp,          Mp,          Sp,          Rp         },
4044         {Mp,          Ip,          Qp,          Sp         },
4045         {Np,          Lp,          Rp,          Tp         },
4046         {Jp,          Np,          Tp,          Qp + 56 * 1},
4047         {Pp,          Jp + 56 * 2, Qp + 56 * 3, Up         },
4048         {Lp,          Pp,          Up,          Rp         },
4049         {Op,          Kp,          Rp,          Vp         },
4050         {Ip + 56 * 2, Op,          Vp,          Qp + 56 * 2},
4051         /* layer 6 */
4052         {Rp,          Sp,          Aq,          Yp         },
4053         {Sp,          Qp,          Wp,          Aq         },
4054         {Tp,          Rp,          Yp,          Cq         },
4055         {Qp + 56 * 1, Tp,          Cq,          Wp + 56 * 1},
4056         {Up,          Qp + 56 * 3, Xp + 56 * 1, Dq         },
4057         {Rp,          Up,          Dq,          Zp         },
4058         {Vp,          Rp,          Zp,          Bq         },
4059         {Qp + 56 * 2, Vp,          Bq,          Xp         },
4060         /* layer 7 (the top is the periodic image of the bottom of layer 0) */
4061         {Yp,          Aq,          C + 56 * 4,  A + 56 * 4 },
4062         {Aq,          Wp,          B + 56 * 4,  C + 56 * 4 },
4063         {Cq,          Yp,          A + 56 * 4,  D + 56 * 4 },
4064         {Wp + 56 * 1, Cq,          D + 56 * 4,  B + 56 * 5 },
4065         {Dq,          Xp + 56 * 1, B + 56 * 5,  E + 56 * 4 },
4066         {Zp,          Dq,          E + 56 * 4,  A + 56 * 6 },
4067         {Bq,          Zp,          A + 56 * 6,  F + 56 * 4 },
4068         {Xp,          Bq,          F + 56 * 4,  B + 56 * 4 }
4069       };
4070       const PetscReal gamma                = PetscAcosReal((PetscSqrtReal(3.) - 1.) / PetscSqrtReal(2.)) / PETSC_PI;
4071       const PetscReal patternCoords[56][3] = {
4072         {1.,        0.,        0.  }, /* A  */
4073         {0.,        1.,        0.  }, /* B  */
4074         {gamma,     gamma,     0.  }, /* C  */
4075         {1 + gamma, 1 - gamma, 0.  }, /* D  */
4076         {2 - gamma, 2 - gamma, 0.  }, /* E  */
4077         {1 - gamma, 1 + gamma, 0.  }, /* F  */
4078 
4079         {.5,        0,         .25 }, /* G  */
4080         {1.5,       0.,        .25 }, /* H  */
4081         {.5,        1.,        .25 }, /* II */
4082         {1.5,       1.,        .25 }, /* J  */
4083         {.25,       .5,        .25 }, /* K  */
4084         {1.25,      .5,        .25 }, /* L  */
4085         {.75,       1.5,       .25 }, /* M  */
4086         {1.75,      1.5,       .25 }, /* N  */
4087 
4088         {0.,        0.,        .5  }, /* O  */
4089         {1.,        1.,        .5  }, /* P  */
4090         {gamma,     1 - gamma, .5  }, /* Q  */
4091         {1 + gamma, gamma,     .5  }, /* R  */
4092         {2 - gamma, 1 + gamma, .5  }, /* S  */
4093         {1 - gamma, 2 - gamma, .5  }, /* T  */
4094 
4095         {0.,        .5,        .75 }, /* U  */
4096         {0.,        1.5,       .75 }, /* V  */
4097         {1.,        .5,        .75 }, /* W  */
4098         {1.,        1.5,       .75 }, /* X  */
4099         {.5,        .75,       .75 }, /* Y  */
4100         {.5,        1.75,      .75 }, /* Z  */
4101         {1.5,       .25,       .75 }, /* Ap */
4102         {1.5,       1.25,      .75 }, /* Bp */
4103 
4104         {1.,        0.,        1.  }, /* Cp */
4105         {0.,        1.,        1.  }, /* Dp */
4106         {1 - gamma, 1 - gamma, 1.  }, /* Ep */
4107         {1 + gamma, 1 + gamma, 1.  }, /* Fp */
4108         {2 - gamma, gamma,     1.  }, /* Gp */
4109         {gamma,     2 - gamma, 1.  }, /* Hp */
4110 
4111         {.5,        0.,        1.25}, /* Ip */
4112         {1.5,       0.,        1.25}, /* Jp */
4113         {.5,        1.,        1.25}, /* Kp */
4114         {1.5,       1.,        1.25}, /* Lp */
4115         {.75,       .5,        1.25}, /* Mp */
4116         {1.75,      .5,        1.25}, /* Np */
4117         {.25,       1.5,       1.25}, /* Op */
4118         {1.25,      1.5,       1.25}, /* Pp */
4119 
4120         {0.,        0.,        1.5 }, /* Qp */
4121         {1.,        1.,        1.5 }, /* Rp */
4122         {1 - gamma, gamma,     1.5 }, /* Sp */
4123         {2 - gamma, 1 - gamma, 1.5 }, /* Tp */
4124         {1 + gamma, 2 - gamma, 1.5 }, /* Up */
4125         {gamma,     1 + gamma, 1.5 }, /* Vp */
4126 
4127         {0.,        .5,        1.75}, /* Wp */
4128         {0.,        1.5,       1.75}, /* Xp */
4129         {1.,        .5,        1.75}, /* Yp */
4130         {1.,        1.5,       1.75}, /* Zp */
4131         {.5,        .25,       1.75}, /* Aq */
4132         {.5,        1.25,      1.75}, /* Bq */
4133         {1.5,       .75,       1.75}, /* Cq */
4134         {1.5,       1.75,      1.75}, /* Dq */
4135       };
4136       PetscInt (*cells)[64][4] = NULL;
4137       PetscBool *seen;
4138       PetscInt  *vertToTrueVert;
4139       PetscInt   count;
4140 
4141       for (PetscInt i = 0; i < 3; i++) extentPlus[i] = extent[i] + 1;
4142       numBlocks = 1;
4143       for (PetscInt i = 0; i < 3; i++) numBlocks *= extent[i];
4144       numBlocksPlus = 1;
4145       for (PetscInt i = 0; i < 3; i++) numBlocksPlus *= extentPlus[i];
4146       numFaces = numBlocks * facesPerBlock;
4147       PetscCall(PetscMalloc1(numBlocks, &cells));
4148       PetscCall(PetscCalloc1(numBlocksPlus * vertsPerBlock, &seen));
4149       for (PetscInt k = 0; k < extent[2]; k++) {
4150         for (PetscInt j = 0; j < extent[1]; j++) {
4151           for (PetscInt i = 0; i < extent[0]; i++) {
4152             for (PetscInt f = 0; f < facesPerBlock; f++) {
4153               for (PetscInt v = 0; v < 4; v++) {
4154                 PetscInt vertRaw     = pattern[f][v];
4155                 PetscInt blockidx    = vertRaw / 56;
4156                 PetscInt patternvert = vertRaw % 56;
4157                 PetscInt xplus       = (blockidx & 1);
4158                 PetscInt yplus       = (blockidx & 2) >> 1;
4159                 PetscInt zplus       = (blockidx & 4) >> 2;
4160                 PetscInt zcoord      = (periodic && periodic[2] == DM_BOUNDARY_PERIODIC) ? ((k + zplus) % extent[2]) : (k + zplus);
4161                 PetscInt ycoord      = (periodic && periodic[1] == DM_BOUNDARY_PERIODIC) ? ((j + yplus) % extent[1]) : (j + yplus);
4162                 PetscInt xcoord      = (periodic && periodic[0] == DM_BOUNDARY_PERIODIC) ? ((i + xplus) % extent[0]) : (i + xplus);
4163                 PetscInt vert        = ((zcoord * extentPlus[1] + ycoord) * extentPlus[0] + xcoord) * 56 + patternvert;
4164 
4165                 cells[(k * extent[1] + j) * extent[0] + i][f][v] = vert;
4166                 seen[vert]                                       = PETSC_TRUE;
4167               }
4168             }
4169           }
4170         }
4171       }
4172       for (PetscInt i = 0; i < numBlocksPlus * vertsPerBlock; i++)
4173         if (seen[i]) numVertices++;
4174       count = 0;
4175       PetscCall(PetscMalloc1(numBlocksPlus * vertsPerBlock, &vertToTrueVert));
4176       PetscCall(PetscMalloc1(numVertices * 3, &vtxCoords));
4177       for (PetscInt i = 0; i < numBlocksPlus * vertsPerBlock; i++) vertToTrueVert[i] = -1;
4178       for (PetscInt k = 0; k < extentPlus[2]; k++) {
4179         for (PetscInt j = 0; j < extentPlus[1]; j++) {
4180           for (PetscInt i = 0; i < extentPlus[0]; i++) {
4181             for (PetscInt v = 0; v < vertsPerBlock; v++) {
4182               PetscInt vIdx = ((k * extentPlus[1] + j) * extentPlus[0] + i) * vertsPerBlock + v;
4183 
4184               if (seen[vIdx]) {
4185                 PetscInt thisVert;
4186 
4187                 vertToTrueVert[vIdx] = thisVert = count++;
4188 
4189                 for (PetscInt d = 0; d < 3; d++) vtxCoords[3 * thisVert + d] = patternCoords[v][d];
4190                 vtxCoords[3 * thisVert + 0] += i * 2;
4191                 vtxCoords[3 * thisVert + 1] += j * 2;
4192                 vtxCoords[3 * thisVert + 2] += k * 2;
4193               }
4194             }
4195           }
4196         }
4197       }
4198       for (PetscInt i = 0; i < numBlocks; i++) {
4199         for (PetscInt f = 0; f < facesPerBlock; f++) {
4200           for (PetscInt v = 0; v < 4; v++) cells[i][f][v] = vertToTrueVert[cells[i][f][v]];
4201         }
4202       }
4203       PetscCall(PetscFree(vertToTrueVert));
4204       PetscCall(PetscFree(seen));
4205       cells_flat = cells[0][0];
4206       numEdges   = 0;
4207       for (PetscInt i = 0; i < numFaces; i++) {
4208         for (PetscInt e = 0; e < 4; e++) {
4209           PetscInt         ev[]       = {cells_flat[i * 4 + e], cells_flat[i * 4 + ((e + 1) % 4)]};
4210           const PetscReal *evCoords[] = {&vtxCoords[3 * ev[0]], &vtxCoords[3 * ev[1]]};
4211 
4212           for (PetscInt d = 0; d < 3; d++) {
4213             if (!periodic || periodic[0] != DM_BOUNDARY_PERIODIC) {
4214               if (evCoords[0][d] == 0. && evCoords[1][d] == 0.) numEdges++;
4215               if (evCoords[0][d] == 2. * extent[d] && evCoords[1][d] == 2. * extent[d]) numEdges++;
4216             }
4217           }
4218         }
4219       }
4220       PetscCall(PetscMalloc1(numEdges, &edges));
4221       PetscCall(PetscMalloc1(numEdges, &edgeSets));
4222       for (PetscInt edge = 0, i = 0; i < numFaces; i++) {
4223         for (PetscInt e = 0; e < 4; e++) {
4224           PetscInt         ev[]       = {cells_flat[i * 4 + e], cells_flat[i * 4 + ((e + 1) % 4)]};
4225           const PetscReal *evCoords[] = {&vtxCoords[3 * ev[0]], &vtxCoords[3 * ev[1]]};
4226 
4227           for (PetscInt d = 0; d < 3; d++) {
4228             if (!periodic || periodic[d] != DM_BOUNDARY_PERIODIC) {
4229               if (evCoords[0][d] == 0. && evCoords[1][d] == 0.) {
4230                 edges[edge][0]   = ev[0];
4231                 edges[edge][1]   = ev[1];
4232                 edgeSets[edge++] = 2 * d;
4233               }
4234               if (evCoords[0][d] == 2. * extent[d] && evCoords[1][d] == 2. * extent[d]) {
4235                 edges[edge][0]   = ev[0];
4236                 edges[edge][1]   = ev[1];
4237                 edgeSets[edge++] = 2 * d + 1;
4238               }
4239             }
4240           }
4241         }
4242       }
4243     }
4244     evalFunc   = TPSEvaluate_Gyroid;
4245     normalFunc = TPSExtrudeNormalFunc_Gyroid;
4246     break;
4247   }
4248 
4249   PetscCall(DMSetDimension(dm, topoDim));
4250   if (rank == 0) PetscCall(DMPlexBuildFromCellList(dm, numFaces, numVertices, 4, cells_flat));
4251   else PetscCall(DMPlexBuildFromCellList(dm, 0, 0, 0, NULL));
4252   PetscCall(PetscFree(cells_flat));
4253   {
4254     DM idm;
4255     PetscCall(DMPlexInterpolate(dm, &idm));
4256     PetscCall(DMPlexReplace_Internal(dm, &idm));
4257   }
4258   if (rank == 0) PetscCall(DMPlexBuildCoordinatesFromCellList(dm, spaceDim, vtxCoords));
4259   else PetscCall(DMPlexBuildCoordinatesFromCellList(dm, spaceDim, NULL));
4260   PetscCall(PetscFree(vtxCoords));
4261 
4262   PetscCall(DMCreateLabel(dm, "Face Sets"));
4263   PetscCall(DMGetLabel(dm, "Face Sets", &label));
4264   for (PetscInt e = 0; e < numEdges; e++) {
4265     PetscInt        njoin;
4266     const PetscInt *join, verts[] = {numFaces + edges[e][0], numFaces + edges[e][1]};
4267     PetscCall(DMPlexGetJoin(dm, 2, verts, &njoin, &join));
4268     PetscCheck(njoin == 1, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Expected unique join of vertices %" PetscInt_FMT " and %" PetscInt_FMT, edges[e][0], edges[e][1]);
4269     PetscCall(DMLabelSetValue(label, join[0], edgeSets[e]));
4270     PetscCall(DMPlexRestoreJoin(dm, 2, verts, &njoin, &join));
4271   }
4272   PetscCall(PetscFree(edges));
4273   PetscCall(PetscFree(edgeSets));
4274   if (tps_distribute) {
4275     DM               pdm = NULL;
4276     PetscPartitioner part;
4277 
4278     PetscCall(DMPlexGetPartitioner(dm, &part));
4279     PetscCall(PetscPartitionerSetFromOptions(part));
4280     PetscCall(DMPlexDistribute(dm, 0, NULL, &pdm));
4281     if (pdm) PetscCall(DMPlexReplace_Internal(dm, &pdm));
4282     // Do not auto-distribute again
4283     PetscCall(DMPlexDistributeSetDefault(dm, PETSC_FALSE));
4284   }
4285 
4286   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
4287   for (PetscInt refine = 0; refine < refinements; refine++) {
4288     PetscInt     m;
4289     DM           dmf;
4290     Vec          X;
4291     PetscScalar *x;
4292     PetscCall(DMRefine(dm, MPI_COMM_NULL, &dmf));
4293     PetscCall(DMPlexReplace_Internal(dm, &dmf));
4294 
4295     PetscCall(DMGetCoordinatesLocal(dm, &X));
4296     PetscCall(VecGetLocalSize(X, &m));
4297     PetscCall(VecGetArray(X, &x));
4298     for (PetscInt i = 0; i < m; i += 3) PetscCall(TPSNearestPoint(evalFunc, &x[i]));
4299     PetscCall(VecRestoreArray(X, &x));
4300   }
4301 
4302   // Face Sets has already been propagated to new vertices during refinement; this propagates to the initial vertices.
4303   PetscCall(DMGetLabel(dm, "Face Sets", &label));
4304   PetscCall(DMPlexLabelComplete(dm, label));
4305 
4306   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
4307 
4308   if (thickness > 0) {
4309     DM              edm, cdm, ecdm;
4310     DMPlexTransform tr;
4311     const char     *prefix;
4312     PetscOptions    options;
4313     // Code from DMPlexExtrude
4314     PetscCall(DMPlexTransformCreate(PetscObjectComm((PetscObject)dm), &tr));
4315     PetscCall(DMPlexTransformSetDM(tr, dm));
4316     PetscCall(DMPlexTransformSetType(tr, DMPLEXEXTRUDETYPE));
4317     PetscCall(PetscObjectGetOptionsPrefix((PetscObject)dm, &prefix));
4318     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)tr, prefix));
4319     PetscCall(PetscObjectGetOptions((PetscObject)dm, &options));
4320     PetscCall(PetscObjectSetOptions((PetscObject)tr, options));
4321     PetscCall(DMPlexTransformExtrudeSetLayers(tr, layers));
4322     PetscCall(DMPlexTransformExtrudeSetThickness(tr, thickness));
4323     PetscCall(DMPlexTransformExtrudeSetTensor(tr, PETSC_FALSE));
4324     PetscCall(DMPlexTransformExtrudeSetSymmetric(tr, PETSC_TRUE));
4325     PetscCall(DMPlexTransformExtrudeSetNormalFunction(tr, normalFunc));
4326     PetscCall(DMPlexTransformSetFromOptions(tr));
4327     PetscCall(PetscObjectSetOptions((PetscObject)tr, NULL));
4328     PetscCall(DMPlexTransformSetUp(tr));
4329     PetscCall(PetscObjectViewFromOptions((PetscObject)tr, NULL, "-dm_plex_tps_transform_view"));
4330     PetscCall(DMPlexTransformApply(tr, dm, &edm));
4331     PetscCall(DMCopyDisc(dm, edm));
4332     PetscCall(DMGetCoordinateDM(dm, &cdm));
4333     PetscCall(DMGetCoordinateDM(edm, &ecdm));
4334     PetscCall(DMCopyDisc(cdm, ecdm));
4335     PetscCall(DMPlexTransformCreateDiscLabels(tr, edm));
4336     PetscCall(DMPlexTransformDestroy(&tr));
4337     PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, edm));
4338     PetscCall(DMPlexReplace_Internal(dm, &edm));
4339   }
4340   PetscFunctionReturn(PETSC_SUCCESS);
4341 }
4342 
4343 /*@
4344   DMPlexCreateTPSMesh - Create a distributed, interpolated mesh of a triply-periodic surface
4345 
4346   Collective
4347 
4348   Input Parameters:
4349 + comm           - The communicator for the `DM` object
4350 . tpstype        - Type of triply-periodic surface
4351 . extent         - Array of length 3 containing number of periods in each direction
4352 . periodic       - array of length 3 with periodicity, or `NULL` for non-periodic
4353 . tps_distribute - Distribute 2D manifold mesh prior to refinement and extrusion (more scalable)
4354 . refinements    - Number of factor-of-2 refinements of 2D manifold mesh
4355 . layers         - Number of cell layers extruded in normal direction
4356 - thickness      - Thickness in normal direction
4357 
4358   Output Parameter:
4359 . dm - The `DM` object
4360 
4361   Level: beginner
4362 
4363   Notes:
4364   This meshes the surface of the Schwarz P or Gyroid surfaces.  Schwarz P is the simplest member of the triply-periodic minimal surfaces.
4365   <https://en.wikipedia.org/wiki/Schwarz_minimal_surface#Schwarz_P_(%22Primitive%22)> and can be cut with "clean" boundaries.
4366   The Gyroid <https://en.wikipedia.org/wiki/Gyroid> is another triply-periodic minimal surface with applications in additive manufacturing; it is much more difficult to "cut" since there are no planes of symmetry.
4367   Our implementation creates a very coarse mesh of the surface and refines (by 4-way splitting) as many times as requested.
4368   On each refinement, all vertices are projected to their nearest point on the surface.
4369   This projection could readily be extended to related surfaces.
4370 
4371   See {cite}`maskery2018insights`
4372 
4373   The face (edge) sets for the Schwarz P surface are numbered $1(-x), 2(+x), 3(-y), 4(+y), 5(-z), 6(+z)$.
4374   When the mesh is refined, "Face Sets" contain the new vertices (created during refinement).
4375   Use `DMPlexLabelComplete()` to propagate to coarse-level vertices.
4376 
4377   Developer Notes:
4378   The Gyroid mesh does not currently mark boundary sets.
4379 
4380 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateSphereMesh()`, `DMSetType()`, `DMCreate()`
4381 @*/
4382 PetscErrorCode DMPlexCreateTPSMesh(MPI_Comm comm, DMPlexTPSType tpstype, const PetscInt extent[], const DMBoundaryType periodic[], PetscBool tps_distribute, PetscInt refinements, PetscInt layers, PetscReal thickness, DM *dm)
4383 {
4384   PetscFunctionBegin;
4385   PetscCall(DMCreate(comm, dm));
4386   PetscCall(DMSetType(*dm, DMPLEX));
4387   PetscCall(DMPlexCreateTPSMesh_Internal(*dm, tpstype, extent, periodic, tps_distribute, refinements, layers, thickness));
4388   PetscFunctionReturn(PETSC_SUCCESS);
4389 }
4390 
4391 /*@
4392   DMPlexCreateSphereMesh - Creates a mesh on the d-dimensional sphere, S^d.
4393 
4394   Collective
4395 
4396   Input Parameters:
4397 + comm    - The communicator for the `DM` object
4398 . dim     - The dimension
4399 . simplex - Use simplices, or tensor product cells
4400 - R       - The radius
4401 
4402   Output Parameter:
4403 . dm - The `DM` object
4404 
4405   Level: beginner
4406 
4407 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateBallMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
4408 @*/
4409 PetscErrorCode DMPlexCreateSphereMesh(MPI_Comm comm, PetscInt dim, PetscBool simplex, PetscReal R, DM *dm)
4410 {
4411   PetscFunctionBegin;
4412   PetscAssertPointer(dm, 5);
4413   PetscCall(DMCreate(comm, dm));
4414   PetscCall(DMSetType(*dm, DMPLEX));
4415   PetscCall(DMPlexCreateSphereMesh_Internal(*dm, dim, simplex, R));
4416   PetscFunctionReturn(PETSC_SUCCESS);
4417 }
4418 
4419 static PetscErrorCode DMPlexCreateBallMesh_Internal(DM dm, PetscInt dim, PetscReal R)
4420 {
4421   DM          sdm, vol;
4422   DMLabel     bdlabel;
4423   const char *prefix;
4424 
4425   PetscFunctionBegin;
4426   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &sdm));
4427   PetscCall(DMSetType(sdm, DMPLEX));
4428   PetscCall(DMGetOptionsPrefix(dm, &prefix));
4429   PetscCall(DMSetOptionsPrefix(sdm, prefix));
4430   PetscCall(DMAppendOptionsPrefix(sdm, "bd_"));
4431   PetscCall(DMPlexDistributeSetDefault(sdm, PETSC_FALSE));
4432   PetscCall(DMPlexCreateSphereMesh_Internal(sdm, dim - 1, PETSC_TRUE, R));
4433   PetscCall(DMSetFromOptions(sdm));
4434   PetscCall(DMViewFromOptions(sdm, NULL, "-dm_view"));
4435   PetscCall(DMPlexGenerate(sdm, NULL, PETSC_TRUE, &vol));
4436   PetscCall(DMDestroy(&sdm));
4437   PetscCall(DMPlexReplace_Internal(dm, &vol));
4438   PetscCall(DMCreateLabel(dm, "marker"));
4439   PetscCall(DMGetLabel(dm, "marker", &bdlabel));
4440   PetscCall(DMPlexMarkBoundaryFaces(dm, PETSC_DETERMINE, bdlabel));
4441   PetscCall(DMPlexLabelComplete(dm, bdlabel));
4442   PetscFunctionReturn(PETSC_SUCCESS);
4443 }
4444 
4445 /*@
4446   DMPlexCreateBallMesh - Creates a simplex mesh on the d-dimensional ball, B^d.
4447 
4448   Collective
4449 
4450   Input Parameters:
4451 + comm - The communicator for the `DM` object
4452 . dim  - The dimension
4453 - R    - The radius
4454 
4455   Output Parameter:
4456 . dm - The `DM` object
4457 
4458   Options Database Key:
4459 . bd_dm_refine - This will refine the surface mesh preserving the sphere geometry
4460 
4461   Level: beginner
4462 
4463 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateSphereMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
4464 @*/
4465 PetscErrorCode DMPlexCreateBallMesh(MPI_Comm comm, PetscInt dim, PetscReal R, DM *dm)
4466 {
4467   PetscFunctionBegin;
4468   PetscCall(DMCreate(comm, dm));
4469   PetscCall(DMSetType(*dm, DMPLEX));
4470   PetscCall(DMPlexCreateBallMesh_Internal(*dm, dim, R));
4471   PetscFunctionReturn(PETSC_SUCCESS);
4472 }
4473 
4474 static PetscErrorCode DMPlexCreateReferenceCell_Internal(DM rdm, DMPolytopeType ct)
4475 {
4476   PetscFunctionBegin;
4477   switch (ct) {
4478   case DM_POLYTOPE_POINT: {
4479     PetscInt    numPoints[1]        = {1};
4480     PetscInt    coneSize[1]         = {0};
4481     PetscInt    cones[1]            = {0};
4482     PetscInt    coneOrientations[1] = {0};
4483     PetscScalar vertexCoords[1]     = {0.0};
4484 
4485     PetscCall(DMSetDimension(rdm, 0));
4486     PetscCall(DMPlexCreateFromDAG(rdm, 0, numPoints, coneSize, cones, coneOrientations, vertexCoords));
4487   } break;
4488   case DM_POLYTOPE_SEGMENT: {
4489     PetscInt    numPoints[2]        = {2, 1};
4490     PetscInt    coneSize[3]         = {2, 0, 0};
4491     PetscInt    cones[2]            = {1, 2};
4492     PetscInt    coneOrientations[2] = {0, 0};
4493     PetscScalar vertexCoords[2]     = {-1.0, 1.0};
4494 
4495     PetscCall(DMSetDimension(rdm, 1));
4496     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
4497   } break;
4498   case DM_POLYTOPE_POINT_PRISM_TENSOR: {
4499     PetscInt    numPoints[2]        = {2, 1};
4500     PetscInt    coneSize[3]         = {2, 0, 0};
4501     PetscInt    cones[2]            = {1, 2};
4502     PetscInt    coneOrientations[2] = {0, 0};
4503     PetscScalar vertexCoords[2]     = {-1.0, 1.0};
4504 
4505     PetscCall(DMSetDimension(rdm, 1));
4506     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
4507   } break;
4508   case DM_POLYTOPE_TRIANGLE: {
4509     PetscInt    numPoints[2]        = {3, 1};
4510     PetscInt    coneSize[4]         = {3, 0, 0, 0};
4511     PetscInt    cones[3]            = {1, 2, 3};
4512     PetscInt    coneOrientations[3] = {0, 0, 0};
4513     PetscScalar vertexCoords[6]     = {-1.0, -1.0, 1.0, -1.0, -1.0, 1.0};
4514 
4515     PetscCall(DMSetDimension(rdm, 2));
4516     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
4517   } break;
4518   case DM_POLYTOPE_QUADRILATERAL: {
4519     PetscInt    numPoints[2]        = {4, 1};
4520     PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
4521     PetscInt    cones[4]            = {1, 2, 3, 4};
4522     PetscInt    coneOrientations[4] = {0, 0, 0, 0};
4523     PetscScalar vertexCoords[8]     = {-1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0};
4524 
4525     PetscCall(DMSetDimension(rdm, 2));
4526     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
4527   } break;
4528   case DM_POLYTOPE_SEG_PRISM_TENSOR: {
4529     PetscInt    numPoints[2]        = {4, 1};
4530     PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
4531     PetscInt    cones[4]            = {1, 2, 3, 4};
4532     PetscInt    coneOrientations[4] = {0, 0, 0, 0};
4533     PetscScalar vertexCoords[8]     = {-1.0, -1.0, 1.0, -1.0, -1.0, 1.0, 1.0, 1.0};
4534 
4535     PetscCall(DMSetDimension(rdm, 2));
4536     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
4537   } break;
4538   case DM_POLYTOPE_TETRAHEDRON: {
4539     PetscInt    numPoints[2]        = {4, 1};
4540     PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
4541     PetscInt    cones[4]            = {1, 2, 3, 4};
4542     PetscInt    coneOrientations[4] = {0, 0, 0, 0};
4543     PetscScalar vertexCoords[12]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0};
4544 
4545     PetscCall(DMSetDimension(rdm, 3));
4546     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
4547   } break;
4548   case DM_POLYTOPE_HEXAHEDRON: {
4549     PetscInt    numPoints[2]        = {8, 1};
4550     PetscInt    coneSize[9]         = {8, 0, 0, 0, 0, 0, 0, 0, 0};
4551     PetscInt    cones[8]            = {1, 2, 3, 4, 5, 6, 7, 8};
4552     PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
4553     PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, 1.0, 1.0, 1.0, -1.0, 1.0, 1.0};
4554 
4555     PetscCall(DMSetDimension(rdm, 3));
4556     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
4557   } break;
4558   case DM_POLYTOPE_TRI_PRISM: {
4559     PetscInt    numPoints[2]        = {6, 1};
4560     PetscInt    coneSize[7]         = {6, 0, 0, 0, 0, 0, 0};
4561     PetscInt    cones[6]            = {1, 2, 3, 4, 5, 6};
4562     PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
4563     PetscScalar vertexCoords[18]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, 1.0, 1.0};
4564 
4565     PetscCall(DMSetDimension(rdm, 3));
4566     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
4567   } break;
4568   case DM_POLYTOPE_TRI_PRISM_TENSOR: {
4569     PetscInt    numPoints[2]        = {6, 1};
4570     PetscInt    coneSize[7]         = {6, 0, 0, 0, 0, 0, 0};
4571     PetscInt    cones[6]            = {1, 2, 3, 4, 5, 6};
4572     PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
4573     PetscScalar vertexCoords[18]    = {-1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, 1.0, 1.0};
4574 
4575     PetscCall(DMSetDimension(rdm, 3));
4576     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
4577   } break;
4578   case DM_POLYTOPE_QUAD_PRISM_TENSOR: {
4579     PetscInt    numPoints[2]        = {8, 1};
4580     PetscInt    coneSize[9]         = {8, 0, 0, 0, 0, 0, 0, 0, 0};
4581     PetscInt    cones[8]            = {1, 2, 3, 4, 5, 6, 7, 8};
4582     PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
4583     PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0, 1.0, -1.0, -1.0, 1.0, 1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, 1.0, 1.0, 1.0, -1.0, 1.0, 1.0};
4584 
4585     PetscCall(DMSetDimension(rdm, 3));
4586     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
4587   } break;
4588   case DM_POLYTOPE_PYRAMID: {
4589     PetscInt    numPoints[2]        = {5, 1};
4590     PetscInt    coneSize[6]         = {5, 0, 0, 0, 0, 0};
4591     PetscInt    cones[5]            = {1, 2, 3, 4, 5};
4592     PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
4593     PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, -1.0, 0.0, 0.0, 1.0};
4594 
4595     PetscCall(DMSetDimension(rdm, 3));
4596     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
4597   } break;
4598   default:
4599     SETERRQ(PetscObjectComm((PetscObject)rdm), PETSC_ERR_ARG_WRONG, "Cannot create reference cell for cell type %s", DMPolytopeTypes[ct]);
4600   }
4601   {
4602     PetscInt Nv, v;
4603 
4604     /* Must create the celltype label here so that we do not automatically try to compute the types */
4605     PetscCall(DMCreateLabel(rdm, "celltype"));
4606     PetscCall(DMPlexSetCellType(rdm, 0, ct));
4607     PetscCall(DMPlexGetChart(rdm, NULL, &Nv));
4608     for (v = 1; v < Nv; ++v) PetscCall(DMPlexSetCellType(rdm, v, DM_POLYTOPE_POINT));
4609   }
4610   PetscCall(DMPlexInterpolateInPlace_Internal(rdm));
4611   PetscCall(PetscObjectSetName((PetscObject)rdm, DMPolytopeTypes[ct]));
4612   PetscFunctionReturn(PETSC_SUCCESS);
4613 }
4614 
4615 /*@
4616   DMPlexCreateReferenceCell - Create a `DMPLEX` with the appropriate FEM reference cell
4617 
4618   Collective
4619 
4620   Input Parameters:
4621 + comm - The communicator
4622 - ct   - The cell type of the reference cell
4623 
4624   Output Parameter:
4625 . refdm - The reference cell
4626 
4627   Level: intermediate
4628 
4629 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateBoxMesh()`
4630 @*/
4631 PetscErrorCode DMPlexCreateReferenceCell(MPI_Comm comm, DMPolytopeType ct, DM *refdm)
4632 {
4633   PetscFunctionBegin;
4634   PetscCall(DMCreate(comm, refdm));
4635   PetscCall(DMSetType(*refdm, DMPLEX));
4636   PetscCall(DMPlexCreateReferenceCell_Internal(*refdm, ct));
4637   PetscFunctionReturn(PETSC_SUCCESS);
4638 }
4639 
4640 static PetscErrorCode DMPlexCreateBoundaryLabel_Private(DM dm, const char name[])
4641 {
4642   DM        plex;
4643   DMLabel   label;
4644   PetscBool hasLabel;
4645 
4646   PetscFunctionBegin;
4647   PetscCall(DMHasLabel(dm, name, &hasLabel));
4648   if (hasLabel) PetscFunctionReturn(PETSC_SUCCESS);
4649   PetscCall(DMCreateLabel(dm, name));
4650   PetscCall(DMGetLabel(dm, name, &label));
4651   PetscCall(DMConvert(dm, DMPLEX, &plex));
4652   PetscCall(DMPlexMarkBoundaryFaces(plex, 1, label));
4653   PetscCall(DMPlexLabelComplete(plex, label));
4654   PetscCall(DMDestroy(&plex));
4655   PetscFunctionReturn(PETSC_SUCCESS);
4656 }
4657 
4658 /*
4659   We use the last coordinate as the radius, the inner radius is lower[dim-1] and the outer radius is upper[dim-1]. Then we map the first coordinate around the circle.
4660 
4661     (x, y) -> (r, theta) = (x[1], (x[0] - lower[0]) * 2\pi/(upper[0] - lower[0]))
4662 */
4663 static void boxToAnnulus(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
4664 {
4665   const PetscReal low = PetscRealPart(constants[0]);
4666   const PetscReal upp = PetscRealPart(constants[1]);
4667   const PetscReal r   = PetscRealPart(u[1]);
4668   const PetscReal th  = 2. * PETSC_PI * (PetscRealPart(u[0]) - low) / (upp - low);
4669 
4670   f0[0] = r * PetscCosReal(th);
4671   f0[1] = r * PetscSinReal(th);
4672 }
4673 
4674 // Insert vertices and their joins, marked by depth
4675 static PetscErrorCode ProcessCohesiveLabel_Vertices(DM dm, DMLabel label, DMLabel vlabel, PetscInt val, PetscInt n, const PetscInt vertices[])
4676 {
4677   PetscFunctionBegin;
4678   PetscCall(DMPlexMarkSubmesh_Interpolated(dm, vlabel, val, PETSC_FALSE, PETSC_FALSE, label, NULL));
4679   PetscFunctionReturn(PETSC_SUCCESS);
4680 }
4681 
4682 // Insert faces and their closures, marked by depth
4683 static PetscErrorCode ProcessCohesiveLabel_Faces(DM dm, DMLabel label, PetscInt n, const PetscInt faces[])
4684 {
4685   PetscFunctionBegin;
4686   for (PetscInt p = 0; p < n; ++p) {
4687     const PetscInt point   = faces[p];
4688     PetscInt      *closure = NULL;
4689     PetscInt       clSize, pdepth;
4690 
4691     PetscCall(DMPlexGetPointDepth(dm, point, &pdepth));
4692     PetscCall(DMLabelSetValue(label, point, pdepth));
4693     PetscCall(DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &clSize, &closure));
4694     for (PetscInt cl = 0; cl < clSize * 2; cl += 2) {
4695       PetscCall(DMPlexGetPointDepth(dm, closure[cl], &pdepth));
4696       PetscCall(DMLabelSetValue(label, closure[cl], pdepth));
4697     }
4698     PetscCall(DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &clSize, &closure));
4699   }
4700   PetscFunctionReturn(PETSC_SUCCESS);
4701 }
4702 
4703 PETSC_EXTERN PetscErrorCode PetscOptionsFindPairPrefix_Private(PetscOptions, const char pre[], const char name[], const char *option[], const char *value[], PetscBool *flg);
4704 
4705 const char *const DMPlexShapes[] = {"box", "box_surface", "ball", "sphere", "cylinder", "schwarz_p", "gyroid", "doublet", "annulus", "hypercubic", "zbox", "unknown", "DMPlexShape", "DM_SHAPE_", NULL};
4706 
4707 static PetscErrorCode DMPlexCreateFromOptions_Internal(PetscOptionItems PetscOptionsObject, PetscBool *useCoordSpace, DM dm)
4708 {
4709   DMPlexShape    shape   = DM_SHAPE_BOX;
4710   DMPolytopeType cell    = DM_POLYTOPE_TRIANGLE;
4711   PetscInt       dim     = 2;
4712   PetscBool      simplex = PETSC_TRUE, interpolate = PETSC_TRUE, orient = PETSC_FALSE, adjCone = PETSC_FALSE, adjClosure = PETSC_TRUE, refDomain = PETSC_FALSE;
4713   PetscBool      flg, flg2, fflg, strflg, bdfflg, nameflg;
4714   MPI_Comm       comm;
4715   char           filename[PETSC_MAX_PATH_LEN]   = "<unspecified>";
4716   char           bdFilename[PETSC_MAX_PATH_LEN] = "<unspecified>";
4717   char           plexname[PETSC_MAX_PATH_LEN]   = "";
4718   const char    *option;
4719 
4720   PetscFunctionBegin;
4721   PetscCall(PetscLogEventBegin(DMPLEX_CreateFromOptions, dm, 0, 0, 0));
4722   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
4723   /* TODO Turn this into a registration interface */
4724   PetscCall(PetscOptionsString("-dm_plex_filename", "File containing a mesh", "DMPlexCreateFromFile", filename, filename, sizeof(filename), &fflg));
4725   PetscCall(PetscOptionsString("-dm_plex_file_contents", "Contents of a file format in a string", "DMPlexCreateFromFile", filename, filename, sizeof(filename), &strflg));
4726   PetscCall(PetscOptionsString("-dm_plex_boundary_filename", "File containing a mesh boundary", "DMPlexCreateFromFile", bdFilename, bdFilename, sizeof(bdFilename), &bdfflg));
4727   PetscCall(PetscOptionsString("-dm_plex_name", "Name of the mesh in the file", "DMPlexCreateFromFile", plexname, plexname, sizeof(plexname), &nameflg));
4728   PetscCall(PetscOptionsEnum("-dm_plex_cell", "Cell shape", "", DMPolytopeTypes, (PetscEnum)cell, (PetscEnum *)&cell, NULL));
4729   PetscCall(PetscOptionsBool("-dm_plex_reference_cell_domain", "Use a reference cell domain", "", refDomain, &refDomain, NULL));
4730   PetscCall(PetscOptionsEnum("-dm_plex_shape", "Shape for built-in mesh", "", DMPlexShapes, (PetscEnum)shape, (PetscEnum *)&shape, &flg));
4731   PetscCall(PetscOptionsBoundedInt("-dm_plex_dim", "Topological dimension of the mesh", "DMGetDimension", dim, &dim, &flg, 0));
4732   PetscCall(PetscOptionsBool("-dm_plex_simplex", "Mesh cell shape", "", simplex, &simplex, &flg));
4733   PetscCall(PetscOptionsBool("-dm_plex_interpolate", "Flag to create edges and faces automatically", "", interpolate, &interpolate, &flg));
4734   PetscCall(PetscOptionsBool("-dm_plex_orient", "Orient the constructed mesh", "DMPlexOrient", orient, &orient, &flg));
4735   PetscCall(PetscOptionsBool("-dm_plex_adj_cone", "Set adjacency direction", "DMSetBasicAdjacency", adjCone, &adjCone, &flg));
4736   PetscCall(PetscOptionsBool("-dm_plex_adj_closure", "Set adjacency size", "DMSetBasicAdjacency", adjClosure, &adjClosure, &flg2));
4737   if (flg || flg2) PetscCall(DMSetBasicAdjacency(dm, adjCone, adjClosure));
4738   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_adj", "Debug output level all adjacency computations", "", 0, &((DM_Plex *)dm->data)->printAdj, NULL, 0));
4739 
4740   switch (cell) {
4741   case DM_POLYTOPE_POINT:
4742   case DM_POLYTOPE_SEGMENT:
4743   case DM_POLYTOPE_POINT_PRISM_TENSOR:
4744   case DM_POLYTOPE_TRIANGLE:
4745   case DM_POLYTOPE_QUADRILATERAL:
4746   case DM_POLYTOPE_TETRAHEDRON:
4747   case DM_POLYTOPE_HEXAHEDRON:
4748     *useCoordSpace = PETSC_TRUE;
4749     break;
4750   default:
4751     *useCoordSpace = PETSC_FALSE;
4752     break;
4753   }
4754 
4755   if (fflg) {
4756     DM          dmnew;
4757     const char *name;
4758 
4759     PetscCall(PetscObjectGetName((PetscObject)dm, &name));
4760     PetscCall(DMPlexCreateFromFile(PetscObjectComm((PetscObject)dm), filename, nameflg ? plexname : name, interpolate, &dmnew));
4761     PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
4762     PetscCall(DMPlexReplace_Internal(dm, &dmnew));
4763   } else if (refDomain) {
4764     PetscCall(DMPlexCreateReferenceCell_Internal(dm, cell));
4765   } else if (bdfflg) {
4766     DM          bdm, dmnew;
4767     const char *name;
4768 
4769     PetscCall(PetscObjectGetName((PetscObject)dm, &name));
4770     PetscCall(DMPlexCreateFromFile(PetscObjectComm((PetscObject)dm), bdFilename, nameflg ? plexname : name, interpolate, &bdm));
4771     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)bdm, "bd_"));
4772     PetscCall(DMSetFromOptions(bdm));
4773     PetscCall(DMPlexGenerate(bdm, NULL, interpolate, &dmnew));
4774     PetscCall(DMDestroy(&bdm));
4775     PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
4776     PetscCall(DMPlexReplace_Internal(dm, &dmnew));
4777   } else if (strflg) {
4778     DM          dmnew;
4779     PetscViewer viewer;
4780     const char *contents;
4781     char       *strname;
4782     char        tmpdir[PETSC_MAX_PATH_LEN];
4783     char        tmpfilename[PETSC_MAX_PATH_LEN];
4784     char        name[PETSC_MAX_PATH_LEN];
4785     MPI_Comm    comm;
4786     PetscMPIInt rank;
4787 
4788     PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
4789     PetscCallMPI(MPI_Comm_rank(comm, &rank));
4790     PetscCall(PetscStrchr(filename, ':', &strname));
4791     PetscCheck(strname, comm, PETSC_ERR_ARG_WRONG, "File contents must have the form \"ext:string_name\", not %s", filename);
4792     strname[0] = '\0';
4793     ++strname;
4794     PetscCall(PetscDLSym(NULL, strname, (void **)&contents));
4795     PetscCheck(contents, comm, PETSC_ERR_ARG_WRONG, "Could not locate mesh string %s", strname);
4796     PetscCall(PetscGetTmp(comm, tmpdir, PETSC_MAX_PATH_LEN));
4797     PetscCall(PetscStrlcat(tmpdir, "/meshXXXXXX", PETSC_MAX_PATH_LEN));
4798     PetscCall(PetscMkdtemp(tmpdir));
4799     PetscCall(PetscSNPrintf(tmpfilename, PETSC_MAX_PATH_LEN, "%s/mesh.%s", tmpdir, filename));
4800     PetscCall(PetscViewerASCIIOpen(comm, tmpfilename, &viewer));
4801     PetscCall(PetscViewerASCIIPrintf(viewer, "%s\n", contents));
4802     PetscCall(PetscViewerDestroy(&viewer));
4803     PetscCall(DMPlexCreateFromFile(PetscObjectComm((PetscObject)dm), tmpfilename, plexname, interpolate, &dmnew));
4804     PetscCall(PetscRMTree(tmpdir));
4805     PetscCall(PetscSNPrintf(name, PETSC_MAX_PATH_LEN, "%s Mesh", strname));
4806     PetscCall(PetscObjectSetName((PetscObject)dm, name));
4807     PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
4808     PetscCall(DMPlexReplace_Internal(dm, &dmnew));
4809   } else {
4810     PetscCall(PetscObjectSetName((PetscObject)dm, DMPlexShapes[shape]));
4811     switch (shape) {
4812     case DM_SHAPE_BOX:
4813     case DM_SHAPE_ZBOX:
4814     case DM_SHAPE_ANNULUS: {
4815       PetscInt       faces[3]  = {0, 0, 0};
4816       PetscReal      lower[3]  = {0, 0, 0};
4817       PetscReal      upper[3]  = {1, 1, 1};
4818       DMBoundaryType bdt[3]    = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
4819       PetscBool      isAnnular = shape == DM_SHAPE_ANNULUS ? PETSC_TRUE : PETSC_FALSE;
4820       PetscInt       i, n;
4821 
4822       n = dim;
4823       for (i = 0; i < dim; ++i) faces[i] = (dim == 1 ? 1 : 4 - dim);
4824       PetscCall(PetscOptionsIntArray("-dm_plex_box_faces", "Number of faces along each dimension", "", faces, &n, &flg));
4825       n = 3;
4826       PetscCall(PetscOptionsRealArray("-dm_plex_box_lower", "Lower left corner of box", "", lower, &n, &flg));
4827       PetscCheck(!flg || !(n != dim), comm, PETSC_ERR_ARG_SIZ, "Lower box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
4828       n = 3;
4829       PetscCall(PetscOptionsRealArray("-dm_plex_box_upper", "Upper right corner of box", "", upper, &n, &flg));
4830       PetscCheck(!flg || !(n != dim), comm, PETSC_ERR_ARG_SIZ, "Upper box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
4831       n = 3;
4832       PetscCall(PetscOptionsEnumArray("-dm_plex_box_bd", "Boundary type for each dimension", "", DMBoundaryTypes, (PetscEnum *)bdt, &n, &flg));
4833       PetscCheck(!flg || !(n != dim), comm, PETSC_ERR_ARG_SIZ, "Box boundary types had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
4834 
4835       PetscCheck(!isAnnular || dim == 2, comm, PETSC_ERR_ARG_OUTOFRANGE, "Only two dimensional annuli have been implemented");
4836       if (isAnnular)
4837         for (i = 0; i < dim - 1; ++i) bdt[i] = DM_BOUNDARY_PERIODIC;
4838 
4839       switch (cell) {
4840       case DM_POLYTOPE_TRI_PRISM_TENSOR:
4841         PetscCall(DMPlexCreateWedgeBoxMesh_Internal(dm, faces, lower, upper, bdt));
4842         if (!interpolate) {
4843           DM udm;
4844 
4845           PetscCall(DMPlexUninterpolate(dm, &udm));
4846           PetscCall(DMPlexReplace_Internal(dm, &udm));
4847         }
4848         break;
4849       default:
4850         PetscCall(DMPlexCreateBoxMesh_Internal(dm, shape, dim, simplex, faces, lower, upper, bdt, interpolate));
4851         break;
4852       }
4853       if (isAnnular) {
4854         DM          cdm;
4855         PetscDS     cds;
4856         PetscScalar bounds[2] = {lower[0], upper[0]};
4857 
4858         // Fix coordinates for annular region
4859         PetscCall(DMSetPeriodicity(dm, NULL, NULL, NULL));
4860         PetscCall(DMSetCellCoordinatesLocal(dm, NULL));
4861         PetscCall(DMSetCellCoordinates(dm, NULL));
4862         PetscCall(DMPlexCreateCoordinateSpace(dm, 1, PETSC_TRUE, NULL));
4863         PetscCall(DMGetCoordinateDM(dm, &cdm));
4864         PetscCall(DMGetDS(cdm, &cds));
4865         PetscCall(PetscDSSetConstants(cds, 2, bounds));
4866         PetscCall(DMPlexRemapGeometry(dm, 0.0, boxToAnnulus));
4867       }
4868     } break;
4869     case DM_SHAPE_BOX_SURFACE: {
4870       PetscInt  faces[3] = {0, 0, 0};
4871       PetscReal lower[3] = {0, 0, 0};
4872       PetscReal upper[3] = {1, 1, 1};
4873       PetscInt  i, n;
4874 
4875       n = dim + 1;
4876       for (i = 0; i < dim + 1; ++i) faces[i] = (dim + 1 == 1 ? 1 : 4 - (dim + 1));
4877       PetscCall(PetscOptionsIntArray("-dm_plex_box_faces", "Number of faces along each dimension", "", faces, &n, &flg));
4878       n = 3;
4879       PetscCall(PetscOptionsRealArray("-dm_plex_box_lower", "Lower left corner of box", "", lower, &n, &flg));
4880       PetscCheck(!flg || !(n != dim + 1), comm, PETSC_ERR_ARG_SIZ, "Lower box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim + 1);
4881       n = 3;
4882       PetscCall(PetscOptionsRealArray("-dm_plex_box_upper", "Upper right corner of box", "", upper, &n, &flg));
4883       PetscCheck(!flg || !(n != dim + 1), comm, PETSC_ERR_ARG_SIZ, "Upper box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim + 1);
4884       PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(dm, dim + 1, faces, lower, upper, interpolate));
4885     } break;
4886     case DM_SHAPE_SPHERE: {
4887       PetscReal R = 1.0;
4888 
4889       PetscCall(PetscOptionsReal("-dm_plex_sphere_radius", "Radius of the sphere", "", R, &R, &flg));
4890       PetscCall(DMPlexCreateSphereMesh_Internal(dm, dim, simplex, R));
4891     } break;
4892     case DM_SHAPE_BALL: {
4893       PetscReal R = 1.0;
4894 
4895       PetscCall(PetscOptionsReal("-dm_plex_ball_radius", "Radius of the ball", "", R, &R, &flg));
4896       PetscCall(DMPlexCreateBallMesh_Internal(dm, dim, R));
4897     } break;
4898     case DM_SHAPE_CYLINDER: {
4899       DMBoundaryType bdt = DM_BOUNDARY_NONE;
4900       PetscInt       Nw  = 6;
4901       PetscInt       Nr  = 0;
4902 
4903       PetscCall(PetscOptionsEnum("-dm_plex_cylinder_bd", "Boundary type in the z direction", "", DMBoundaryTypes, (PetscEnum)bdt, (PetscEnum *)&bdt, NULL));
4904       PetscCall(PetscOptionsInt("-dm_plex_cylinder_num_wedges", "Number of wedges around the cylinder", "", Nw, &Nw, NULL));
4905       PetscCall(PetscOptionsInt("-dm_plex_cylinder_num_refine", "Number of refinements before projection", "", Nr, &Nr, NULL));
4906       switch (cell) {
4907       case DM_POLYTOPE_TRI_PRISM_TENSOR:
4908         PetscCall(DMPlexCreateWedgeCylinderMesh_Internal(dm, Nw, interpolate));
4909         break;
4910       default:
4911         PetscCall(DMPlexCreateHexCylinderMesh_Internal(dm, bdt, Nr));
4912         break;
4913       }
4914     } break;
4915     case DM_SHAPE_SCHWARZ_P: // fallthrough
4916     case DM_SHAPE_GYROID: {
4917       PetscInt       extent[3] = {1, 1, 1}, refine = 0, layers = 0, three;
4918       PetscReal      thickness   = 0.;
4919       DMBoundaryType periodic[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
4920       DMPlexTPSType  tps_type    = shape == DM_SHAPE_SCHWARZ_P ? DMPLEX_TPS_SCHWARZ_P : DMPLEX_TPS_GYROID;
4921       PetscBool      tps_distribute;
4922       PetscCall(PetscOptionsIntArray("-dm_plex_tps_extent", "Number of replicas for each of three dimensions", NULL, extent, (three = 3, &three), NULL));
4923       PetscCall(PetscOptionsInt("-dm_plex_tps_refine", "Number of refinements", NULL, refine, &refine, NULL));
4924       PetscCall(PetscOptionsEnumArray("-dm_plex_tps_periodic", "Periodicity in each of three dimensions", NULL, DMBoundaryTypes, (PetscEnum *)periodic, (three = 3, &three), NULL));
4925       PetscCall(PetscOptionsInt("-dm_plex_tps_layers", "Number of layers in volumetric extrusion (or zero to not extrude)", NULL, layers, &layers, NULL));
4926       PetscCall(PetscOptionsReal("-dm_plex_tps_thickness", "Thickness of volumetric extrusion", NULL, thickness, &thickness, NULL));
4927       PetscCall(DMPlexDistributeGetDefault(dm, &tps_distribute));
4928       PetscCall(PetscOptionsBool("-dm_plex_tps_distribute", "Distribute the 2D mesh prior to refinement and extrusion", NULL, tps_distribute, &tps_distribute, NULL));
4929       PetscCall(DMPlexCreateTPSMesh_Internal(dm, tps_type, extent, periodic, tps_distribute, refine, layers, thickness));
4930     } break;
4931     case DM_SHAPE_DOUBLET: {
4932       DM        dmnew;
4933       PetscReal rl = 0.0;
4934 
4935       PetscCall(PetscOptionsReal("-dm_plex_doublet_refinementlimit", "Refinement limit", NULL, rl, &rl, NULL));
4936       PetscCall(DMPlexCreateDoublet(PetscObjectComm((PetscObject)dm), dim, simplex, interpolate, rl, &dmnew));
4937       PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
4938       PetscCall(DMPlexReplace_Internal(dm, &dmnew));
4939     } break;
4940     case DM_SHAPE_HYPERCUBIC: {
4941       PetscInt       *edges, overlap = 1;
4942       PetscReal      *lower, *upper;
4943       DMBoundaryType *bdt;
4944       PetscInt        n, d;
4945 
4946       *useCoordSpace = PETSC_FALSE;
4947       PetscCall(PetscMalloc4(dim, &edges, dim, &lower, dim, &upper, dim, &bdt));
4948       for (d = 0; d < dim; ++d) {
4949         edges[d] = 1;
4950         lower[d] = 0.;
4951         upper[d] = 1.;
4952         bdt[d]   = DM_BOUNDARY_PERIODIC;
4953       }
4954       n = dim;
4955       PetscCall(PetscOptionsIntArray("-dm_plex_box_faces", "Number of faces along each dimension", "", edges, &n, &flg));
4956       n = dim;
4957       PetscCall(PetscOptionsRealArray("-dm_plex_box_lower", "Lower left corner of box", "", lower, &n, &flg));
4958       PetscCheck(!flg || n == dim, comm, PETSC_ERR_ARG_SIZ, "Lower box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
4959       n = dim;
4960       PetscCall(PetscOptionsRealArray("-dm_plex_box_upper", "Upper right corner of box", "", upper, &n, &flg));
4961       PetscCheck(!flg || n == dim, comm, PETSC_ERR_ARG_SIZ, "Upper box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
4962       n = dim;
4963       PetscCall(PetscOptionsEnumArray("-dm_plex_box_bd", "Boundary type for each dimension", "", DMBoundaryTypes, (PetscEnum *)bdt, &n, &flg));
4964       PetscCheck(!flg || n == dim, comm, PETSC_ERR_ARG_SIZ, "Box boundary types had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
4965       PetscCall(PetscOptionsBoundedInt("-dm_distribute_overlap", "The size of the overlap halo", "DMPlexDistribute", overlap, &overlap, NULL, 0));
4966       PetscCall(DMPlexCreateHypercubicMesh_Internal(dm, dim, lower, upper, edges, overlap, bdt));
4967       PetscCall(PetscFree4(edges, lower, upper, bdt));
4968     } break;
4969     default:
4970       SETERRQ(comm, PETSC_ERR_SUP, "Domain shape %s is unsupported", DMPlexShapes[shape]);
4971     }
4972   }
4973   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
4974   if (!((PetscObject)dm)->name && nameflg) PetscCall(PetscObjectSetName((PetscObject)dm, plexname));
4975   if (orient) PetscCall(DMPlexOrient(dm));
4976   // Allow label creation
4977   PetscCall(PetscOptionsFindPairPrefix_Private(NULL, ((PetscObject)dm)->prefix, "-dm_plex_label_", &option, NULL, &flg));
4978   if (flg) {
4979     DMLabel     label;
4980     PetscInt    points[1024], n = 1024;
4981     char        fulloption[PETSC_MAX_PATH_LEN];
4982     const char *name = &option[14];
4983 
4984     PetscCall(DMCreateLabel(dm, name));
4985     PetscCall(DMGetLabel(dm, name, &label));
4986     fulloption[0] = '-';
4987     fulloption[1] = 0;
4988     PetscCall(PetscStrlcat(fulloption, option, PETSC_MAX_PATH_LEN));
4989     PetscCall(PetscOptionsGetIntArray(NULL, ((PetscObject)dm)->prefix, fulloption, points, &n, NULL));
4990     for (PetscInt p = 0; p < n; ++p) PetscCall(DMLabelSetValue(label, points[p], 1));
4991   }
4992   // Allow cohesive label creation
4993   //   Faces are input, completed, and all points are marked with their depth
4994   PetscCall(PetscOptionsFindPairPrefix_Private(NULL, ((PetscObject)dm)->prefix, "-dm_plex_cohesive_label_", &option, NULL, &flg));
4995   if (flg) {
4996     DMLabel   label;
4997     PetscInt  points[1024], n, pStart, pEnd, Nl = 1;
4998     PetscBool noCreate = PETSC_FALSE;
4999     char      fulloption[PETSC_MAX_PATH_LEN];
5000     char      name[PETSC_MAX_PATH_LEN];
5001     size_t    len;
5002 
5003     PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
5004     PetscCall(PetscStrncpy(name, &option[23], PETSC_MAX_PATH_LEN));
5005     PetscCall(PetscStrlen(name, &len));
5006     if (name[len - 1] == '0') Nl = 10;
5007     for (PetscInt l = 0; l < Nl; ++l) {
5008       if (l > 0) name[len - 1] = (char)('0' + l);
5009       fulloption[0] = 0;
5010       PetscCall(PetscStrlcat(fulloption, "-dm_plex_cohesive_label_", 32));
5011       PetscCall(PetscStrlcat(fulloption, name, PETSC_MAX_PATH_LEN - 32));
5012       n = 1024;
5013       PetscCall(PetscOptionsGetIntArray(NULL, ((PetscObject)dm)->prefix, fulloption, points, &n, &flg));
5014       if (!flg) break;
5015       PetscCall(DMHasLabel(dm, name, &noCreate));
5016       if (noCreate) {
5017         DMLabel         inlabel;
5018         IS              pointIS;
5019         const PetscInt *lpoints;
5020         PetscInt        pdep, ln, inval = points[0];
5021         char            newname[PETSC_MAX_PATH_LEN];
5022 
5023         PetscCheck(n == 1, comm, PETSC_ERR_ARG_WRONG, "Must specify a label value with this option");
5024         PetscCall(DMGetLabel(dm, name, &inlabel));
5025         PetscCall(DMLabelGetStratumIS(inlabel, inval, &pointIS));
5026         PetscCall(ISGetLocalSize(pointIS, &ln));
5027         PetscCall(ISGetIndices(pointIS, &lpoints));
5028         PetscCall(DMPlexGetPointDepth(dm, lpoints[0], &pdep));
5029         PetscCall(PetscSNPrintf(newname, PETSC_MAX_PATH_LEN, "%s%" PetscInt_FMT, name, points[0]));
5030         PetscCall(DMCreateLabel(dm, newname));
5031         PetscCall(DMGetLabel(dm, newname, &label));
5032         if (!pdep) PetscCall(ProcessCohesiveLabel_Vertices(dm, label, inlabel, inval, ln, lpoints));
5033         else PetscCall(ProcessCohesiveLabel_Faces(dm, label, ln, lpoints));
5034         PetscCall(ISRestoreIndices(pointIS, &lpoints));
5035         PetscCall(ISDestroy(&pointIS));
5036       } else {
5037         PetscCall(DMCreateLabel(dm, name));
5038         PetscCall(DMGetLabel(dm, name, &label));
5039         if (pStart >= pEnd) n = 0;
5040         PetscCall(ProcessCohesiveLabel_Faces(dm, label, n, points));
5041       }
5042       PetscCall(DMPlexOrientLabel(dm, label));
5043       PetscCall(DMPlexLabelCohesiveComplete(dm, label, NULL, 1, PETSC_FALSE, PETSC_FALSE, NULL));
5044     }
5045   }
5046   PetscCall(DMViewFromOptions(dm, NULL, "-created_dm_view"));
5047   PetscCall(PetscLogEventEnd(DMPLEX_CreateFromOptions, dm, 0, 0, 0));
5048   PetscFunctionReturn(PETSC_SUCCESS);
5049 }
5050 
5051 PetscErrorCode DMSetFromOptions_NonRefinement_Plex(DM dm, PetscOptionItems PetscOptionsObject)
5052 {
5053   DM_Plex  *mesh = (DM_Plex *)dm->data;
5054   PetscBool flg, flg2;
5055   char      bdLabel[PETSC_MAX_PATH_LEN];
5056   char      method[PETSC_MAX_PATH_LEN];
5057 
5058   PetscFunctionBegin;
5059   /* Handle viewing */
5060   PetscCall(PetscOptionsBool("-dm_plex_print_set_values", "Output all set values info", "DMPlexMatSetClosure", PETSC_FALSE, &mesh->printSetValues, NULL));
5061   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_fem", "Debug output level for all fem computations", "DMPlexSNESComputeResidualFEM", 0, &mesh->printFEM, NULL, 0));
5062   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_fvm", "Debug output level for all fvm computations", "DMPlexSNESComputeResidualFVM", 0, &mesh->printFVM, NULL, 0));
5063   PetscCall(PetscOptionsReal("-dm_plex_print_tol", "Tolerance for FEM output", "DMPlexSNESComputeResidualFEM", mesh->printTol, &mesh->printTol, NULL));
5064   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_l2", "Debug output level all L2 diff computations", "DMComputeL2Diff", 0, &mesh->printL2, NULL, 0));
5065   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_locate", "Debug output level all point location computations", "DMLocatePoints", 0, &mesh->printLocate, NULL, 0));
5066   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_project", "Debug output level all projection computations", "DMPlexProject", 0, &mesh->printProject, NULL, 0));
5067   PetscCall(DMMonitorSetFromOptions(dm, "-dm_plex_monitor_throughput", "Monitor the simulation throughput", "DMPlexMonitorThroughput", DMPlexMonitorThroughput, NULL, &flg));
5068   if (flg) PetscCall(PetscLogDefaultBegin());
5069   // Interpolation
5070   PetscCall(PetscOptionsBool("-dm_plex_interpolate_prefer_tensor", "When different orderings exist, prefer the tensor order", "DMPlexSetInterpolationPreferTensor", mesh->interpolatePreferTensor, &mesh->interpolatePreferTensor, NULL));
5071   /* Labeling */
5072   PetscCall(PetscOptionsString("-dm_plex_boundary_label", "Label to mark the mesh boundary", "", bdLabel, bdLabel, sizeof(bdLabel), &flg));
5073   if (flg) PetscCall(DMPlexCreateBoundaryLabel_Private(dm, bdLabel));
5074   /* Point Location */
5075   PetscCall(PetscOptionsBool("-dm_plex_hash_location", "Use grid hashing for point location", "DMInterpolate", PETSC_FALSE, &mesh->useHashLocation, NULL));
5076   /* Partitioning and distribution */
5077   PetscCall(PetscOptionsBool("-dm_plex_partition_balance", "Attempt to evenly divide points on partition boundary between processes", "DMPlexSetPartitionBalance", PETSC_FALSE, &mesh->partitionBalance, NULL));
5078   /* Reordering */
5079   PetscCall(PetscOptionsBool("-dm_reorder_section", "Compute point permutation for local section", "DMReorderSectionSetDefault", PETSC_FALSE, &flg2, &flg));
5080   if (flg) PetscCall(DMReorderSectionSetDefault(dm, flg2 ? DM_REORDER_DEFAULT_TRUE : DM_REORDER_DEFAULT_FALSE));
5081   PetscCall(PetscOptionsString("-dm_reorder_section_type", "Reordering method for local section", "DMReorderSectionSetType", method, method, PETSC_MAX_PATH_LEN, &flg));
5082   if (flg) PetscCall(DMReorderSectionSetType(dm, method));
5083   /* Generation and remeshing */
5084   PetscCall(PetscOptionsBool("-dm_plex_remesh_bd", "Allow changes to the boundary on remeshing", "DMAdapt", PETSC_FALSE, &mesh->remeshBd, NULL));
5085   /* Projection behavior */
5086   PetscCall(PetscOptionsBoundedInt("-dm_plex_max_projection_height", "Maximum mesh point height used to project locally", "DMPlexSetMaxProjectionHeight", 0, &mesh->maxProjectionHeight, NULL, 0));
5087   PetscCall(PetscOptionsBool("-dm_plex_regular_refinement", "Use special nested projection algorithm for regular refinement", "DMPlexSetRegularRefinement", mesh->regularRefinement, &mesh->regularRefinement, NULL));
5088   /* Checking structure */
5089   {
5090     PetscBool all = PETSC_FALSE;
5091 
5092     PetscCall(PetscOptionsBool("-dm_plex_check_all", "Perform all basic checks", "DMPlexCheck", PETSC_FALSE, &all, NULL));
5093     if (all) {
5094       PetscCall(DMPlexCheck(dm));
5095     } else {
5096       PetscCall(PetscOptionsBool("-dm_plex_check_symmetry", "Check that the adjacency information in the mesh is symmetric", "DMPlexCheckSymmetry", PETSC_FALSE, &flg, &flg2));
5097       if (flg && flg2) PetscCall(DMPlexCheckSymmetry(dm));
5098       PetscCall(PetscOptionsBool("-dm_plex_check_skeleton", "Check that each cell has the correct number of vertices (only for homogeneous simplex or tensor meshes)", "DMPlexCheckSkeleton", PETSC_FALSE, &flg, &flg2));
5099       if (flg && flg2) PetscCall(DMPlexCheckSkeleton(dm, 0));
5100       PetscCall(PetscOptionsBool("-dm_plex_check_faces", "Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type", "DMPlexCheckFaces", PETSC_FALSE, &flg, &flg2));
5101       if (flg && flg2) PetscCall(DMPlexCheckFaces(dm, 0));
5102       PetscCall(PetscOptionsBool("-dm_plex_check_geometry", "Check that cells have positive volume", "DMPlexCheckGeometry", PETSC_FALSE, &flg, &flg2));
5103       if (flg && flg2) PetscCall(DMPlexCheckGeometry(dm));
5104       PetscCall(PetscOptionsBool("-dm_plex_check_pointsf", "Check some necessary conditions for PointSF", "DMPlexCheckPointSF", PETSC_FALSE, &flg, &flg2));
5105       if (flg && flg2) PetscCall(DMPlexCheckPointSF(dm, NULL, PETSC_FALSE));
5106       PetscCall(PetscOptionsBool("-dm_plex_check_interface_cones", "Check points on inter-partition interfaces have conforming order of cone points", "DMPlexCheckInterfaceCones", PETSC_FALSE, &flg, &flg2));
5107       if (flg && flg2) PetscCall(DMPlexCheckInterfaceCones(dm));
5108     }
5109     PetscCall(PetscOptionsBool("-dm_plex_check_cell_shape", "Check cell shape", "DMPlexCheckCellShape", PETSC_FALSE, &flg, &flg2));
5110     if (flg && flg2) PetscCall(DMPlexCheckCellShape(dm, PETSC_TRUE, PETSC_DETERMINE));
5111   }
5112   {
5113     PetscReal scale = 1.0;
5114 
5115     PetscCall(PetscOptionsReal("-dm_plex_scale", "Scale factor for mesh coordinates", "DMPlexScale", scale, &scale, &flg));
5116     if (flg) {
5117       Vec coordinates, coordinatesLocal;
5118 
5119       PetscCall(DMGetCoordinates(dm, &coordinates));
5120       PetscCall(DMGetCoordinatesLocal(dm, &coordinatesLocal));
5121       PetscCall(VecScale(coordinates, scale));
5122       PetscCall(VecScale(coordinatesLocal, scale));
5123     }
5124   }
5125   PetscCall(PetscPartitionerSetFromOptions(mesh->partitioner));
5126   PetscFunctionReturn(PETSC_SUCCESS);
5127 }
5128 
5129 PetscErrorCode DMSetFromOptions_Overlap_Plex(DM dm, PetscOptionItems PetscOptionsObject, PetscInt *overlap)
5130 {
5131   PetscInt  numOvLabels = 16, numOvExLabels = 16;
5132   char     *ovLabelNames[16], *ovExLabelNames[16];
5133   PetscInt  numOvValues = 16, numOvExValues = 16, l;
5134   PetscBool flg;
5135 
5136   PetscFunctionBegin;
5137   PetscCall(PetscOptionsBoundedInt("-dm_distribute_overlap", "The size of the overlap halo", "DMPlexDistribute", *overlap, overlap, NULL, 0));
5138   PetscCall(PetscOptionsStringArray("-dm_distribute_overlap_labels", "List of overlap label names", "DMPlexDistribute", ovLabelNames, &numOvLabels, &flg));
5139   if (!flg) numOvLabels = 0;
5140   if (numOvLabels) {
5141     ((DM_Plex *)dm->data)->numOvLabels = numOvLabels;
5142     for (l = 0; l < numOvLabels; ++l) {
5143       PetscCall(DMGetLabel(dm, ovLabelNames[l], &((DM_Plex *)dm->data)->ovLabels[l]));
5144       PetscCheck(((DM_Plex *)dm->data)->ovLabels[l], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid label name %s", ovLabelNames[l]);
5145       PetscCall(PetscFree(ovLabelNames[l]));
5146     }
5147     PetscCall(PetscOptionsIntArray("-dm_distribute_overlap_values", "List of overlap label values", "DMPlexDistribute", ((DM_Plex *)dm->data)->ovValues, &numOvValues, &flg));
5148     if (!flg) numOvValues = 0;
5149     PetscCheck(numOvLabels == numOvValues, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "The number of labels %" PetscInt_FMT " must match the number of values %" PetscInt_FMT, numOvLabels, numOvValues);
5150 
5151     PetscCall(PetscOptionsStringArray("-dm_distribute_overlap_exclude_labels", "List of overlap exclude label names", "DMPlexDistribute", ovExLabelNames, &numOvExLabels, &flg));
5152     if (!flg) numOvExLabels = 0;
5153     ((DM_Plex *)dm->data)->numOvExLabels = numOvExLabels;
5154     for (l = 0; l < numOvExLabels; ++l) {
5155       PetscCall(DMGetLabel(dm, ovExLabelNames[l], &((DM_Plex *)dm->data)->ovExLabels[l]));
5156       PetscCheck(((DM_Plex *)dm->data)->ovExLabels[l], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid label name %s", ovExLabelNames[l]);
5157       PetscCall(PetscFree(ovExLabelNames[l]));
5158     }
5159     PetscCall(PetscOptionsIntArray("-dm_distribute_overlap_exclude_values", "List of overlap exclude label values", "DMPlexDistribute", ((DM_Plex *)dm->data)->ovExValues, &numOvExValues, &flg));
5160     if (!flg) numOvExValues = 0;
5161     PetscCheck(numOvExLabels == numOvExValues, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "The number of exclude labels %" PetscInt_FMT " must match the number of values %" PetscInt_FMT, numOvExLabels, numOvExValues);
5162   }
5163   PetscFunctionReturn(PETSC_SUCCESS);
5164 }
5165 
5166 static PetscErrorCode DMSetFromOptions_Plex(DM dm, PetscOptionItems PetscOptionsObject)
5167 {
5168   PetscFunctionList    ordlist;
5169   char                 oname[256];
5170   char                 sublabelname[PETSC_MAX_PATH_LEN] = "";
5171   DMReorderDefaultFlag reorder;
5172   PetscReal            volume    = -1.0;
5173   PetscInt             prerefine = 0, refine = 0, r, coarsen = 0, overlap = 0, extLayers = 0, dim;
5174   PetscBool            uniformOrig = PETSC_FALSE, uniform = PETSC_TRUE, distribute, saveSF = PETSC_FALSE, interpolate = PETSC_TRUE, coordSpace = PETSC_FALSE, remap = PETSC_TRUE, ghostCells = PETSC_FALSE, isHierarchy, flg;
5175 
5176   PetscFunctionBegin;
5177   PetscOptionsHeadBegin(PetscOptionsObject, "DMPlex Options");
5178   if (dm->cloneOpts) goto non_refine;
5179   /* Handle automatic creation */
5180   PetscCall(DMGetDimension(dm, &dim));
5181   if (dim < 0) PetscCall(DMPlexCreateFromOptions_Internal(PetscOptionsObject, &coordSpace, dm));
5182   PetscCall(DMGetDimension(dm, &dim));
5183   /* Handle interpolation before distribution */
5184   PetscCall(PetscOptionsBool("-dm_plex_interpolate_pre", "Flag to interpolate mesh before distribution", "", interpolate, &interpolate, &flg));
5185   if (flg) {
5186     DMPlexInterpolatedFlag interpolated;
5187 
5188     PetscCall(DMPlexIsInterpolated(dm, &interpolated));
5189     if (interpolated == DMPLEX_INTERPOLATED_FULL && !interpolate) {
5190       DM udm;
5191 
5192       PetscCall(DMPlexUninterpolate(dm, &udm));
5193       PetscCall(DMPlexReplace_Internal(dm, &udm));
5194     } else if (interpolated != DMPLEX_INTERPOLATED_FULL && interpolate) {
5195       DM idm;
5196 
5197       PetscCall(DMPlexInterpolate(dm, &idm));
5198       PetscCall(DMPlexReplace_Internal(dm, &idm));
5199     }
5200   }
5201   // Handle submesh selection before distribution
5202   PetscCall(PetscOptionsString("-dm_plex_submesh", "Label to use for submesh selection", "", sublabelname, sublabelname, PETSC_MAX_PATH_LEN, &flg));
5203   if (flg) {
5204     DM              subdm;
5205     DMLabel         label;
5206     IS              valueIS, pointIS;
5207     const PetscInt *values, *points;
5208     PetscBool       markedFaces = PETSC_FALSE;
5209     PetscInt        Nv, value, Np;
5210 
5211     PetscCall(DMGetLabel(dm, sublabelname, &label));
5212     PetscCall(DMLabelGetNumValues(label, &Nv));
5213     PetscCheck(Nv == 1, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Only a single label value is currently supported for submesh selection, not %" PetscInt_FMT, Nv);
5214     PetscCall(DMLabelGetValueIS(label, &valueIS));
5215     PetscCall(ISGetIndices(valueIS, &values));
5216     value = values[0];
5217     PetscCall(ISRestoreIndices(valueIS, &values));
5218     PetscCall(ISDestroy(&valueIS));
5219     PetscCall(DMLabelGetStratumSize(label, value, &Np));
5220     PetscCall(DMLabelGetStratumIS(label, value, &pointIS));
5221     PetscCall(ISGetIndices(pointIS, &points));
5222     for (PetscInt p = 0; p < Np; ++p) {
5223       PetscInt pdepth;
5224 
5225       PetscCall(DMPlexGetPointDepth(dm, points[p], &pdepth));
5226       if (pdepth) {
5227         markedFaces = PETSC_TRUE;
5228         break;
5229       }
5230     }
5231     PetscCall(ISRestoreIndices(pointIS, &points));
5232     PetscCall(ISDestroy(&pointIS));
5233     PetscCall(DMPlexCreateSubmesh(dm, label, value, markedFaces, &subdm));
5234     PetscCall(DMPlexReplace_Internal(dm, &subdm));
5235     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
5236   }
5237   /* Handle DMPlex refinement before distribution */
5238   PetscCall(DMPlexGetRefinementUniform(dm, &uniformOrig));
5239   PetscCall(PetscOptionsBoundedInt("-dm_refine_pre", "The number of refinements before distribution", "DMCreate", prerefine, &prerefine, NULL, 0));
5240   PetscCall(PetscOptionsBool("-dm_refine_remap_pre", "Flag to control coordinate remapping", "DMCreate", remap, &remap, NULL));
5241   PetscCall(PetscOptionsBool("-dm_refine_uniform_pre", "Flag for uniform refinement before distribution", "DMCreate", uniform, &uniform, &flg));
5242   if (flg) PetscCall(DMPlexSetRefinementUniform(dm, uniform));
5243   PetscCall(PetscOptionsReal("-dm_refine_volume_limit_pre", "The maximum cell volume after refinement before distribution", "DMCreate", volume, &volume, &flg));
5244   if (flg) {
5245     PetscCall(DMPlexSetRefinementUniform(dm, PETSC_FALSE));
5246     PetscCall(DMPlexSetRefinementLimit(dm, volume));
5247     prerefine = PetscMax(prerefine, 1);
5248   }
5249   if (prerefine) PetscCall(DMLocalizeCoordinates(dm));
5250   for (r = 0; r < prerefine; ++r) {
5251     DM             rdm;
5252     PetscPointFunc coordFunc = ((DM_Plex *)dm->data)->coordFunc;
5253 
5254     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
5255     PetscCall(DMRefine(dm, PetscObjectComm((PetscObject)dm), &rdm));
5256     PetscCall(DMPlexReplace_Internal(dm, &rdm));
5257     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
5258     if (coordFunc && remap) {
5259       PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
5260       ((DM_Plex *)dm->data)->coordFunc = coordFunc;
5261     }
5262   }
5263   PetscCall(DMPlexSetRefinementUniform(dm, uniformOrig));
5264   /* Handle DMPlex extrusion before distribution */
5265   PetscCall(PetscOptionsBoundedInt("-dm_extrude", "The number of layers to extrude", "", extLayers, &extLayers, NULL, 0));
5266   if (extLayers) {
5267     DM edm;
5268 
5269     PetscCall(DMExtrude(dm, extLayers, &edm));
5270     PetscCall(DMPlexReplace_Internal(dm, &edm));
5271     ((DM_Plex *)dm->data)->coordFunc = NULL;
5272     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
5273     extLayers = 0;
5274     PetscCall(DMGetDimension(dm, &dim));
5275   }
5276   /* Handle DMPlex reordering before distribution */
5277   PetscCall(DMPlexReorderGetDefault(dm, &reorder));
5278   PetscCall(MatGetOrderingList(&ordlist));
5279   PetscCall(PetscStrncpy(oname, MATORDERINGNATURAL, sizeof(oname)));
5280   PetscCall(PetscOptionsFList("-dm_plex_reorder", "Set mesh reordering type", "DMPlexGetOrdering", ordlist, MATORDERINGNATURAL, oname, sizeof(oname), &flg));
5281   if (reorder == DM_REORDER_DEFAULT_TRUE || flg) {
5282     DM pdm;
5283     IS perm;
5284 
5285     PetscCall(DMPlexGetOrdering(dm, oname, NULL, &perm));
5286     PetscCall(DMPlexPermute(dm, perm, &pdm));
5287     PetscCall(ISDestroy(&perm));
5288     PetscCall(DMPlexReplace_Internal(dm, &pdm));
5289     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
5290   }
5291   /* Handle DMPlex distribution */
5292   PetscCall(DMPlexDistributeGetDefault(dm, &distribute));
5293   PetscCall(PetscOptionsBool("-dm_distribute", "Flag to redistribute a mesh among processes", "DMPlexDistribute", distribute, &distribute, NULL));
5294   PetscCall(PetscOptionsBool("-dm_distribute_save_sf", "Flag to save the migration SF", "DMPlexSetMigrationSF", saveSF, &saveSF, NULL));
5295   PetscCall(DMSetFromOptions_Overlap_Plex(dm, PetscOptionsObject, &overlap));
5296   if (distribute) {
5297     DM               pdm = NULL;
5298     PetscPartitioner part;
5299     PetscSF          sfMigration;
5300 
5301     PetscCall(DMPlexGetPartitioner(dm, &part));
5302     PetscCall(PetscPartitionerSetFromOptions(part));
5303     PetscCall(DMPlexDistribute(dm, overlap, &sfMigration, &pdm));
5304     if (pdm) {
5305       // Delete the local section to force the existing one to be rebuilt with the distributed DM
5306       PetscCall(DMSetLocalSection(dm, pdm->localSection));
5307       PetscCall(DMPlexReplace_Internal(dm, &pdm));
5308     }
5309     if (saveSF) PetscCall(DMPlexSetMigrationSF(dm, sfMigration));
5310     PetscCall(PetscSFDestroy(&sfMigration));
5311   }
5312 
5313   {
5314     PetscBool useBoxLabel = PETSC_FALSE;
5315     PetscCall(PetscOptionsBool("-dm_plex_box_label", "Create 'Face Sets' assuming boundary faces align with cartesian directions", "DMCreate", useBoxLabel, &useBoxLabel, NULL));
5316     if (useBoxLabel) {
5317       PetscInt       n      = 3;
5318       DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
5319 
5320       PetscCall(PetscOptionsEnumArray("-dm_plex_box_label_bd", "Boundary type for each dimension when using -dm_plex_box_label", "", DMBoundaryTypes, (PetscEnum *)bdt, &n, &flg));
5321       PetscCheck(!flg || !(n != dim), PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_SIZ, "Box boundary types had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
5322       PetscCall(DMPlexSetBoxLabel_Internal(dm, bdt));
5323     }
5324   }
5325   /* Must check CEED options before creating function space for coordinates */
5326   {
5327     PetscBool useCeed = PETSC_FALSE, flg;
5328 
5329     PetscCall(PetscOptionsBool("-dm_plex_use_ceed", "Use LibCEED as the FEM backend", "DMPlexSetUseCeed", useCeed, &useCeed, &flg));
5330     if (flg) PetscCall(DMPlexSetUseCeed(dm, useCeed));
5331   }
5332   /* Create coordinate space */
5333   PetscCall(PetscOptionsBool("-dm_coord_space", "Use an FEM space for coordinates", "", coordSpace, &coordSpace, NULL));
5334   if (coordSpace) {
5335     DM_Plex  *mesh   = (DM_Plex *)dm->data;
5336     PetscInt  degree = 1, deg;
5337     PetscInt  height = 0;
5338     DM        cdm;
5339     PetscBool localize = PETSC_TRUE, sparseLocalize = PETSC_TRUE;
5340 
5341     PetscCall(PetscOptionsInt("-dm_coord_petscspace_degree", "FEM degree for coordinate space", "", degree, &degree, NULL));
5342     PetscCall(DMGetCoordinateDegree_Internal(dm, &deg));
5343     if (coordSpace && deg <= 1) PetscCall(DMPlexCreateCoordinateSpace(dm, degree, PETSC_TRUE, mesh->coordFunc));
5344     PetscCall(DMGetCoordinateDM(dm, &cdm));
5345     if (!coordSpace) {
5346       PetscDS      cds;
5347       PetscObject  obj;
5348       PetscClassId id;
5349 
5350       PetscCall(DMGetDS(cdm, &cds));
5351       PetscCall(PetscDSGetDiscretization(cds, 0, &obj));
5352       PetscCall(PetscObjectGetClassId(obj, &id));
5353       if (id == PETSCFE_CLASSID) {
5354         PetscContainer dummy;
5355 
5356         PetscCall(PetscContainerCreate(PETSC_COMM_SELF, &dummy));
5357         PetscCall(PetscObjectSetName((PetscObject)dummy, "coordinates"));
5358         PetscCall(DMSetField(cdm, 0, NULL, (PetscObject)dummy));
5359         PetscCall(PetscContainerDestroy(&dummy));
5360         PetscCall(DMClearDS(cdm));
5361       }
5362       mesh->coordFunc = NULL;
5363     }
5364     PetscCall(PetscOptionsBool("-dm_localize", "Localize mesh coordinates", "", localize, &localize, NULL));
5365     PetscCall(PetscOptionsBool("-dm_sparse_localize", "Localize only necessary cells", "DMSetSparseLocalize", sparseLocalize, &sparseLocalize, &flg));
5366     if (flg) PetscCall(DMSetSparseLocalize(dm, sparseLocalize));
5367     PetscCall(PetscOptionsInt("-dm_localize_height", "Localize edges and faces in addition to cells", "", height, &height, &flg));
5368     if (flg) PetscCall(DMPlexSetMaxProjectionHeight(cdm, height));
5369     if (localize) PetscCall(DMLocalizeCoordinates(dm));
5370   }
5371   /* Handle DMPlex refinement */
5372   remap = PETSC_TRUE;
5373   PetscCall(PetscOptionsBoundedInt("-dm_refine", "The number of uniform refinements", "DMCreate", refine, &refine, NULL, 0));
5374   PetscCall(PetscOptionsBool("-dm_refine_remap", "Flag to control coordinate remapping", "DMCreate", remap, &remap, NULL));
5375   PetscCall(PetscOptionsBoundedInt("-dm_refine_hierarchy", "The number of uniform refinements", "DMCreate", refine, &refine, &isHierarchy, 0));
5376   if (refine) PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
5377   if (refine && isHierarchy) {
5378     DM *dms, coarseDM;
5379 
5380     PetscCall(DMGetCoarseDM(dm, &coarseDM));
5381     PetscCall(PetscObjectReference((PetscObject)coarseDM));
5382     PetscCall(PetscMalloc1(refine, &dms));
5383     PetscCall(DMRefineHierarchy(dm, refine, dms));
5384     /* Total hack since we do not pass in a pointer */
5385     PetscCall(DMPlexSwap_Static(dm, dms[refine - 1]));
5386     if (refine == 1) {
5387       PetscCall(DMSetCoarseDM(dm, dms[0]));
5388       PetscCall(DMPlexSetRegularRefinement(dm, PETSC_TRUE));
5389     } else {
5390       PetscCall(DMSetCoarseDM(dm, dms[refine - 2]));
5391       PetscCall(DMPlexSetRegularRefinement(dm, PETSC_TRUE));
5392       PetscCall(DMSetCoarseDM(dms[0], dms[refine - 1]));
5393       PetscCall(DMPlexSetRegularRefinement(dms[0], PETSC_TRUE));
5394     }
5395     PetscCall(DMSetCoarseDM(dms[refine - 1], coarseDM));
5396     PetscCall(PetscObjectDereference((PetscObject)coarseDM));
5397     /* Free DMs */
5398     for (r = 0; r < refine; ++r) {
5399       PetscCall(DMSetFromOptions_NonRefinement_Plex(dms[r], PetscOptionsObject));
5400       PetscCall(DMDestroy(&dms[r]));
5401     }
5402     PetscCall(PetscFree(dms));
5403   } else {
5404     for (r = 0; r < refine; ++r) {
5405       DM             rdm;
5406       PetscPointFunc coordFunc = ((DM_Plex *)dm->data)->coordFunc;
5407 
5408       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
5409       PetscCall(DMRefine(dm, PetscObjectComm((PetscObject)dm), &rdm));
5410       /* Total hack since we do not pass in a pointer */
5411       PetscCall(DMPlexReplace_Internal(dm, &rdm));
5412       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
5413       if (coordFunc && remap) {
5414         PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
5415         ((DM_Plex *)dm->data)->coordFunc = coordFunc;
5416       }
5417     }
5418   }
5419   /* Handle DMPlex coarsening */
5420   PetscCall(PetscOptionsBoundedInt("-dm_coarsen", "Coarsen the mesh", "DMCreate", coarsen, &coarsen, NULL, 0));
5421   PetscCall(PetscOptionsBoundedInt("-dm_coarsen_hierarchy", "The number of coarsenings", "DMCreate", coarsen, &coarsen, &isHierarchy, 0));
5422   if (coarsen && isHierarchy) {
5423     DM *dms;
5424 
5425     PetscCall(PetscMalloc1(coarsen, &dms));
5426     PetscCall(DMCoarsenHierarchy(dm, coarsen, dms));
5427     /* Free DMs */
5428     for (r = 0; r < coarsen; ++r) {
5429       PetscCall(DMSetFromOptions_NonRefinement_Plex(dms[r], PetscOptionsObject));
5430       PetscCall(DMDestroy(&dms[r]));
5431     }
5432     PetscCall(PetscFree(dms));
5433   } else {
5434     for (r = 0; r < coarsen; ++r) {
5435       DM             cdm;
5436       PetscPointFunc coordFunc = ((DM_Plex *)dm->data)->coordFunc;
5437 
5438       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
5439       PetscCall(DMCoarsen(dm, PetscObjectComm((PetscObject)dm), &cdm));
5440       /* Total hack since we do not pass in a pointer */
5441       PetscCall(DMPlexReplace_Internal(dm, &cdm));
5442       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
5443       if (coordFunc) {
5444         PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
5445         ((DM_Plex *)dm->data)->coordFunc = coordFunc;
5446       }
5447     }
5448   }
5449   // Handle coordinate remapping
5450   remap = PETSC_FALSE;
5451   PetscCall(PetscOptionsBool("-dm_coord_remap", "Flag to control coordinate remapping", "", remap, &remap, NULL));
5452   if (remap) {
5453     DMPlexCoordMap map     = DM_COORD_MAP_NONE;
5454     PetscPointFunc mapFunc = NULL;
5455     PetscScalar    params[16];
5456     PetscInt       Np = PETSC_STATIC_ARRAY_LENGTH(params), cdim;
5457     MPI_Comm       comm;
5458 
5459     PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
5460     PetscCall(DMGetCoordinateDim(dm, &cdim));
5461     PetscCall(PetscOptionsScalarArray("-dm_coord_map_params", "Parameters for the coordinate remapping", "", params, &Np, &flg));
5462     if (!flg) Np = 0;
5463     // TODO Allow user to pass a map function by name
5464     PetscCall(PetscOptionsEnum("-dm_coord_map", "Coordinate mapping for built-in mesh", "", DMPlexCoordMaps, (PetscEnum)map, (PetscEnum *)&map, &flg));
5465     if (flg) {
5466       switch (map) {
5467       case DM_COORD_MAP_NONE:
5468         mapFunc = coordMap_identity;
5469         break;
5470       case DM_COORD_MAP_SHEAR:
5471         mapFunc = coordMap_shear;
5472         if (!Np) {
5473           Np        = cdim + 1;
5474           params[0] = 0;
5475           for (PetscInt d = 1; d <= cdim; ++d) params[d] = 1.0;
5476         }
5477         PetscCheck(Np == cdim + 1, comm, PETSC_ERR_ARG_WRONG, "The shear coordinate map must have cdim + 1 = %" PetscInt_FMT " parameters, not %" PetscInt_FMT, cdim + 1, Np);
5478         break;
5479       case DM_COORD_MAP_FLARE:
5480         mapFunc = coordMap_flare;
5481         if (!Np) {
5482           Np        = cdim + 1;
5483           params[0] = 0;
5484           for (PetscInt d = 1; d <= cdim; ++d) params[d] = 1.0;
5485         }
5486         PetscCheck(Np == cdim + 1, comm, PETSC_ERR_ARG_WRONG, "The flare coordinate map must have cdim + 1 = %" PetscInt_FMT " parameters, not %" PetscInt_FMT, cdim + 1, Np);
5487         break;
5488       case DM_COORD_MAP_ANNULUS:
5489         mapFunc = coordMap_annulus;
5490         if (!Np) {
5491           Np        = 2;
5492           params[0] = 1.;
5493           params[1] = 2.;
5494         }
5495         PetscCheck(Np == 2, comm, PETSC_ERR_ARG_WRONG, "The annulus coordinate map must have 2 parameters, not %" PetscInt_FMT, Np);
5496         break;
5497       case DM_COORD_MAP_SHELL:
5498         mapFunc = coordMap_shell;
5499         if (!Np) {
5500           Np        = 2;
5501           params[0] = 1.;
5502           params[1] = 2.;
5503         }
5504         PetscCheck(Np == 2, comm, PETSC_ERR_ARG_WRONG, "The spherical shell coordinate map must have 2 parameters, not %" PetscInt_FMT, Np);
5505         break;
5506       case DM_COORD_MAP_SINUSOID:
5507         mapFunc = coordMap_sinusoid;
5508         if (!Np) {
5509           Np        = 3;
5510           params[0] = 1.;
5511           params[1] = 1.;
5512           params[2] = 1.;
5513         }
5514         PetscCheck(Np == 3, comm, PETSC_ERR_ARG_WRONG, "The sinusoidal coordinate map must have 3 parameters, not %" PetscInt_FMT, Np);
5515         break;
5516       default:
5517         mapFunc = coordMap_identity;
5518       }
5519     }
5520     if (Np) {
5521       DM      cdm;
5522       PetscDS cds;
5523 
5524       PetscCall(DMGetCoordinateDM(dm, &cdm));
5525       PetscCall(DMGetDS(cdm, &cds));
5526       PetscCall(PetscDSSetConstants(cds, Np, params));
5527     }
5528     PetscCall(DMPlexRemapGeometry(dm, 0.0, mapFunc));
5529   }
5530   /* Handle ghost cells */
5531   PetscCall(PetscOptionsBool("-dm_plex_create_fv_ghost_cells", "Flag to create finite volume ghost cells on the boundary", "DMCreate", ghostCells, &ghostCells, NULL));
5532   if (ghostCells) {
5533     DM   gdm;
5534     char lname[PETSC_MAX_PATH_LEN];
5535 
5536     lname[0] = '\0';
5537     PetscCall(PetscOptionsString("-dm_plex_fv_ghost_cells_label", "Label name for ghost cells boundary", "DMCreate", lname, lname, sizeof(lname), &flg));
5538     PetscCall(DMPlexConstructGhostCells(dm, flg ? lname : NULL, NULL, &gdm));
5539     PetscCall(DMPlexReplace_Internal(dm, &gdm));
5540   }
5541   /* Handle 1D order */
5542   if (reorder != DM_REORDER_DEFAULT_FALSE && dim == 1) {
5543     DM           cdm, rdm;
5544     PetscDS      cds;
5545     PetscObject  obj;
5546     PetscClassId id = PETSC_OBJECT_CLASSID;
5547     IS           perm;
5548     PetscInt     Nf;
5549     PetscBool    distributed;
5550 
5551     PetscCall(DMPlexIsDistributed(dm, &distributed));
5552     PetscCall(DMGetCoordinateDM(dm, &cdm));
5553     PetscCall(DMGetDS(cdm, &cds));
5554     PetscCall(PetscDSGetNumFields(cds, &Nf));
5555     if (Nf) {
5556       PetscCall(PetscDSGetDiscretization(cds, 0, &obj));
5557       PetscCall(PetscObjectGetClassId(obj, &id));
5558     }
5559     if (!distributed && id != PETSCFE_CLASSID) {
5560       PetscCall(DMPlexGetOrdering1D(dm, &perm));
5561       PetscCall(DMPlexPermute(dm, perm, &rdm));
5562       PetscCall(DMPlexReplace_Internal(dm, &rdm));
5563       PetscCall(ISDestroy(&perm));
5564     }
5565   }
5566 /* Handle */
5567 non_refine:
5568   PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
5569   char    *phases[16];
5570   PetscInt Nphases = 16;
5571   PetscCall(PetscOptionsStringArray("-dm_plex_option_phases", "Option phase prefixes", "DMSetFromOptions", phases, &Nphases, &flg));
5572   PetscOptionsHeadEnd();
5573 
5574   // Phases
5575   if (flg) {
5576     DM          cdm;
5577     char       *oldPrefix, *oldCoordPrefix;
5578     const char *tmp;
5579 
5580     PetscCall(DMGetCoordinateDM(dm, &cdm));
5581     PetscCall(PetscObjectGetOptionsPrefix((PetscObject)dm, &tmp));
5582     PetscCall(PetscStrallocpy(tmp, &oldPrefix));
5583     PetscCall(PetscObjectGetOptionsPrefix((PetscObject)cdm, &tmp));
5584     PetscCall(PetscStrallocpy(tmp, &oldCoordPrefix));
5585     for (PetscInt ph = 0; ph < Nphases; ++ph) {
5586       PetscCall(PetscObjectAppendOptionsPrefix((PetscObject)dm, phases[ph]));
5587       PetscCall(PetscObjectAppendOptionsPrefix((PetscObject)cdm, phases[ph]));
5588       PetscCall(PetscInfo(dm, "Options phase %s for DM %s\n", phases[ph], dm->hdr.name));
5589       PetscCall(DMSetFromOptions(dm));
5590       PetscCall(PetscObjectSetOptionsPrefix((PetscObject)dm, oldPrefix));
5591       PetscCall(DMGetCoordinateDM(dm, &cdm));
5592       PetscCall(PetscObjectSetOptionsPrefix((PetscObject)cdm, oldCoordPrefix));
5593       PetscCall(PetscFree(phases[ph]));
5594     }
5595     PetscCall(PetscFree(oldPrefix));
5596     PetscCall(PetscFree(oldCoordPrefix));
5597   }
5598   PetscFunctionReturn(PETSC_SUCCESS);
5599 }
5600 
5601 static PetscErrorCode DMCreateGlobalVector_Plex(DM dm, Vec *vec)
5602 {
5603   PetscFunctionBegin;
5604   PetscCall(DMCreateGlobalVector_Section_Private(dm, vec));
5605   /* PetscCall(VecSetOperation(*vec, VECOP_DUPLICATE, (void(*)(void)) VecDuplicate_MPI_DM)); */
5606   PetscCall(VecSetOperation(*vec, VECOP_VIEW, (void (*)(void))VecView_Plex));
5607   PetscCall(VecSetOperation(*vec, VECOP_VIEWNATIVE, (void (*)(void))VecView_Plex_Native));
5608   PetscCall(VecSetOperation(*vec, VECOP_LOAD, (void (*)(void))VecLoad_Plex));
5609   PetscCall(VecSetOperation(*vec, VECOP_LOADNATIVE, (void (*)(void))VecLoad_Plex_Native));
5610   PetscFunctionReturn(PETSC_SUCCESS);
5611 }
5612 
5613 static PetscErrorCode DMCreateLocalVector_Plex(DM dm, Vec *vec)
5614 {
5615   PetscFunctionBegin;
5616   PetscCall(DMCreateLocalVector_Section_Private(dm, vec));
5617   PetscCall(VecSetOperation(*vec, VECOP_VIEW, (void (*)(void))VecView_Plex_Local));
5618   PetscCall(VecSetOperation(*vec, VECOP_LOAD, (void (*)(void))VecLoad_Plex_Local));
5619   PetscFunctionReturn(PETSC_SUCCESS);
5620 }
5621 
5622 static PetscErrorCode DMGetDimPoints_Plex(DM dm, PetscInt dim, PetscInt *pStart, PetscInt *pEnd)
5623 {
5624   PetscInt depth, d;
5625 
5626   PetscFunctionBegin;
5627   PetscCall(DMPlexGetDepth(dm, &depth));
5628   if (depth == 1) {
5629     PetscCall(DMGetDimension(dm, &d));
5630     if (dim == 0) PetscCall(DMPlexGetDepthStratum(dm, dim, pStart, pEnd));
5631     else if (dim == d) PetscCall(DMPlexGetDepthStratum(dm, 1, pStart, pEnd));
5632     else {
5633       *pStart = 0;
5634       *pEnd   = 0;
5635     }
5636   } else {
5637     PetscCall(DMPlexGetDepthStratum(dm, dim, pStart, pEnd));
5638   }
5639   PetscFunctionReturn(PETSC_SUCCESS);
5640 }
5641 
5642 static PetscErrorCode DMGetNeighbors_Plex(DM dm, PetscInt *nranks, const PetscMPIInt *ranks[])
5643 {
5644   PetscSF            sf;
5645   PetscMPIInt        niranks, njranks;
5646   PetscInt           n;
5647   const PetscMPIInt *iranks, *jranks;
5648   DM_Plex           *data = (DM_Plex *)dm->data;
5649 
5650   PetscFunctionBegin;
5651   PetscCall(DMGetPointSF(dm, &sf));
5652   if (!data->neighbors) {
5653     PetscCall(PetscSFSetUp(sf));
5654     PetscCall(PetscSFGetRootRanks(sf, &njranks, &jranks, NULL, NULL, NULL));
5655     PetscCall(PetscSFGetLeafRanks(sf, &niranks, &iranks, NULL, NULL));
5656     PetscCall(PetscMalloc1(njranks + niranks + 1, &data->neighbors));
5657     PetscCall(PetscArraycpy(data->neighbors + 1, jranks, njranks));
5658     PetscCall(PetscArraycpy(data->neighbors + njranks + 1, iranks, niranks));
5659     n = njranks + niranks;
5660     PetscCall(PetscSortRemoveDupsMPIInt(&n, data->neighbors + 1));
5661     /* The following cast should never fail: can't have more neighbors than PETSC_MPI_INT_MAX */
5662     PetscCall(PetscMPIIntCast(n, data->neighbors));
5663   }
5664   if (nranks) *nranks = data->neighbors[0];
5665   if (ranks) {
5666     if (data->neighbors[0]) *ranks = data->neighbors + 1;
5667     else *ranks = NULL;
5668   }
5669   PetscFunctionReturn(PETSC_SUCCESS);
5670 }
5671 
5672 PETSC_INTERN PetscErrorCode DMInterpolateSolution_Plex(DM, DM, Mat, Vec, Vec);
5673 
5674 static PetscErrorCode DMInitialize_Plex(DM dm)
5675 {
5676   PetscFunctionBegin;
5677   dm->ops->view                      = DMView_Plex;
5678   dm->ops->load                      = DMLoad_Plex;
5679   dm->ops->setfromoptions            = DMSetFromOptions_Plex;
5680   dm->ops->clone                     = DMClone_Plex;
5681   dm->ops->setup                     = DMSetUp_Plex;
5682   dm->ops->createlocalsection        = DMCreateLocalSection_Plex;
5683   dm->ops->createsectionpermutation  = DMCreateSectionPermutation_Plex;
5684   dm->ops->createdefaultconstraints  = DMCreateDefaultConstraints_Plex;
5685   dm->ops->createglobalvector        = DMCreateGlobalVector_Plex;
5686   dm->ops->createlocalvector         = DMCreateLocalVector_Plex;
5687   dm->ops->getlocaltoglobalmapping   = NULL;
5688   dm->ops->createfieldis             = NULL;
5689   dm->ops->createcoordinatedm        = DMCreateCoordinateDM_Plex;
5690   dm->ops->createcoordinatefield     = DMCreateCoordinateField_Plex;
5691   dm->ops->getcoloring               = NULL;
5692   dm->ops->creatematrix              = DMCreateMatrix_Plex;
5693   dm->ops->createinterpolation       = DMCreateInterpolation_Plex;
5694   dm->ops->createmassmatrix          = DMCreateMassMatrix_Plex;
5695   dm->ops->createmassmatrixlumped    = DMCreateMassMatrixLumped_Plex;
5696   dm->ops->createinjection           = DMCreateInjection_Plex;
5697   dm->ops->refine                    = DMRefine_Plex;
5698   dm->ops->coarsen                   = DMCoarsen_Plex;
5699   dm->ops->refinehierarchy           = DMRefineHierarchy_Plex;
5700   dm->ops->coarsenhierarchy          = DMCoarsenHierarchy_Plex;
5701   dm->ops->extrude                   = DMExtrude_Plex;
5702   dm->ops->globaltolocalbegin        = NULL;
5703   dm->ops->globaltolocalend          = NULL;
5704   dm->ops->localtoglobalbegin        = NULL;
5705   dm->ops->localtoglobalend          = NULL;
5706   dm->ops->destroy                   = DMDestroy_Plex;
5707   dm->ops->createsubdm               = DMCreateSubDM_Plex;
5708   dm->ops->createsuperdm             = DMCreateSuperDM_Plex;
5709   dm->ops->getdimpoints              = DMGetDimPoints_Plex;
5710   dm->ops->locatepoints              = DMLocatePoints_Plex;
5711   dm->ops->projectfunctionlocal      = DMProjectFunctionLocal_Plex;
5712   dm->ops->projectfunctionlabellocal = DMProjectFunctionLabelLocal_Plex;
5713   dm->ops->projectfieldlocal         = DMProjectFieldLocal_Plex;
5714   dm->ops->projectfieldlabellocal    = DMProjectFieldLabelLocal_Plex;
5715   dm->ops->projectbdfieldlabellocal  = DMProjectBdFieldLabelLocal_Plex;
5716   dm->ops->computel2diff             = DMComputeL2Diff_Plex;
5717   dm->ops->computel2gradientdiff     = DMComputeL2GradientDiff_Plex;
5718   dm->ops->computel2fielddiff        = DMComputeL2FieldDiff_Plex;
5719   dm->ops->getneighbors              = DMGetNeighbors_Plex;
5720   dm->ops->getlocalboundingbox       = DMGetLocalBoundingBox_Coordinates;
5721   dm->ops->createdomaindecomposition = DMCreateDomainDecomposition_Plex;
5722   dm->ops->createddscatters          = DMCreateDomainDecompositionScatters_Plex;
5723   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexInsertBoundaryValues_C", DMPlexInsertBoundaryValues_Plex));
5724   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexInsertTimeDerivativeBoundaryValues_C", DMPlexInsertTimeDerivativeBoundaryValues_Plex));
5725   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMSetUpGLVisViewer_C", DMSetUpGLVisViewer_Plex));
5726   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMCreateNeumannOverlap_C", DMCreateNeumannOverlap_Plex));
5727   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexDistributeGetDefault_C", DMPlexDistributeGetDefault_Plex));
5728   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexDistributeSetDefault_C", DMPlexDistributeSetDefault_Plex));
5729   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexReorderGetDefault_C", DMPlexReorderGetDefault_Plex));
5730   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexReorderSetDefault_C", DMPlexReorderSetDefault_Plex));
5731   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMReorderSectionGetDefault_C", DMReorderSectionGetDefault_Plex));
5732   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMReorderSectionSetDefault_C", DMReorderSectionSetDefault_Plex));
5733   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMReorderSectionGetType_C", DMReorderSectionGetType_Plex));
5734   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMReorderSectionSetType_C", DMReorderSectionSetType_Plex));
5735   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMInterpolateSolution_C", DMInterpolateSolution_Plex));
5736   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexGetOverlap_C", DMPlexGetOverlap_Plex));
5737   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexSetOverlap_C", DMPlexSetOverlap_Plex));
5738   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexGetUseCeed_C", DMPlexGetUseCeed_Plex));
5739   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexSetUseCeed_C", DMPlexSetUseCeed_Plex));
5740   PetscFunctionReturn(PETSC_SUCCESS);
5741 }
5742 
5743 PETSC_INTERN PetscErrorCode DMClone_Plex(DM dm, DM *newdm)
5744 {
5745   DM_Plex       *mesh = (DM_Plex *)dm->data;
5746   const PetscSF *face_sfs;
5747   PetscInt       num_face_sfs;
5748 
5749   PetscFunctionBegin;
5750   mesh->refct++;
5751   (*newdm)->data = mesh;
5752   PetscCall(DMPlexGetIsoperiodicFaceSF(dm, &num_face_sfs, &face_sfs));
5753   PetscCall(DMPlexSetIsoperiodicFaceSF(*newdm, num_face_sfs, (PetscSF *)face_sfs));
5754   PetscCall(PetscObjectChangeTypeName((PetscObject)*newdm, DMPLEX));
5755   PetscCall(DMInitialize_Plex(*newdm));
5756   PetscFunctionReturn(PETSC_SUCCESS);
5757 }
5758 
5759 /*MC
5760   DMPLEX = "plex" - A `DM` object that encapsulates an unstructured mesh (or grid), or CW Complex {cite}`cwcomplex`,
5761            which can be expressed using a Hasse Diagram {cite}`hassediagram`.
5762            In the local representation, `Vec`s contain all unknowns in the interior and shared boundary. This is
5763            specified by a `PetscSection` object. Ownership in the global representation is determined by
5764            ownership of the underlying `DMPLEX` points. This is specified by another `PetscSection` object.
5765 
5766   Options Database Keys:
5767 + -dm_refine_pre                     - Refine mesh before distribution
5768 + -dm_refine_uniform_pre             - Choose uniform or generator-based refinement
5769 + -dm_refine_volume_limit_pre        - Cell volume limit after pre-refinement using generator
5770 . -dm_distribute                     - Distribute mesh across processes
5771 . -dm_distribute_overlap             - Number of cells to overlap for distribution
5772 . -dm_refine                         - Refine mesh after distribution
5773 . -dm_localize <bool>                - Whether to localize coordinates for periodic meshes
5774 . -dm_sparse_localize <bool>         - Whether to only localize cells on the periodic boundary
5775 . -dm_plex_hash_location             - Use grid hashing for point location
5776 . -dm_plex_hash_box_faces <n,m,p>    - The number of divisions in each direction of the grid hash
5777 . -dm_plex_partition_balance         - Attempt to evenly divide points on partition boundary between processes
5778 . -dm_plex_remesh_bd                 - Allow changes to the boundary on remeshing
5779 . -dm_plex_max_projection_height     - Maximum mesh point height used to project locally
5780 . -dm_plex_regular_refinement        - Use special nested projection algorithm for regular refinement
5781 . -dm_plex_reorder_section           - Use specialized blocking if available
5782 . -dm_plex_check_all                 - Perform all checks below
5783 . -dm_plex_check_symmetry            - Check that the adjacency information in the mesh is symmetric
5784 . -dm_plex_check_skeleton <celltype> - Check that each cell has the correct number of vertices
5785 . -dm_plex_check_faces <celltype>    - Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type
5786 . -dm_plex_check_geometry            - Check that cells have positive volume
5787 . -dm_view :mesh.tex:ascii_latex     - View the mesh in LaTeX/TikZ
5788 . -dm_plex_view_scale <num>          - Scale the TikZ
5789 . -dm_plex_print_fem <num>           - View FEM assembly information, such as element vectors and matrices
5790 - -dm_plex_print_fvm <num>           - View FVM assembly information, such as flux updates
5791 
5792   Level: intermediate
5793 
5794 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMType`, `DMPlexCreate()`, `DMCreate()`, `DMSetType()`, `PetscSection`
5795 M*/
5796 
5797 PETSC_EXTERN PetscErrorCode DMCreate_Plex(DM dm)
5798 {
5799   DM_Plex *mesh;
5800   PetscInt unit;
5801 
5802   PetscFunctionBegin;
5803   PetscCall(PetscCitationsRegister(PlexCitation, &Plexcite));
5804   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5805   PetscCall(PetscNew(&mesh));
5806   dm->reorderSection = DM_REORDER_DEFAULT_NOTSET;
5807   dm->data           = mesh;
5808 
5809   mesh->refct = 1;
5810   PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->coneSection));
5811   PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->supportSection));
5812   mesh->refinementUniform       = PETSC_TRUE;
5813   mesh->refinementLimit         = -1.0;
5814   mesh->distDefault             = PETSC_TRUE;
5815   mesh->reorderDefault          = DM_REORDER_DEFAULT_NOTSET;
5816   mesh->distributionName        = NULL;
5817   mesh->interpolated            = DMPLEX_INTERPOLATED_INVALID;
5818   mesh->interpolatedCollective  = DMPLEX_INTERPOLATED_INVALID;
5819   mesh->interpolatePreferTensor = PETSC_TRUE;
5820 
5821   PetscCall(PetscPartitionerCreate(PetscObjectComm((PetscObject)dm), &mesh->partitioner));
5822   mesh->remeshBd = PETSC_FALSE;
5823 
5824   for (unit = 0; unit < NUM_PETSC_UNITS; ++unit) mesh->scale[unit] = 1.0;
5825 
5826   mesh->depthState    = -1;
5827   mesh->celltypeState = -1;
5828   mesh->printTol      = 1.0e-10;
5829   mesh->nonempty_comm = MPI_COMM_SELF;
5830 
5831   PetscCall(DMInitialize_Plex(dm));
5832   PetscFunctionReturn(PETSC_SUCCESS);
5833 }
5834 
5835 /*@
5836   DMPlexCreate - Creates a `DMPLEX` object, which encapsulates an unstructured mesh, or CW complex, which can be expressed using a Hasse Diagram.
5837 
5838   Collective
5839 
5840   Input Parameter:
5841 . comm - The communicator for the `DMPLEX` object
5842 
5843   Output Parameter:
5844 . mesh - The `DMPLEX` object
5845 
5846   Level: beginner
5847 
5848 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMType`, `DMCreate()`, `DMSetType()`
5849 @*/
5850 PetscErrorCode DMPlexCreate(MPI_Comm comm, DM *mesh)
5851 {
5852   PetscFunctionBegin;
5853   PetscAssertPointer(mesh, 2);
5854   PetscCall(DMCreate(comm, mesh));
5855   PetscCall(DMSetType(*mesh, DMPLEX));
5856   PetscFunctionReturn(PETSC_SUCCESS);
5857 }
5858 
5859 /*@C
5860   DMPlexBuildFromCellListParallel - Build distributed `DMPLEX` topology from a list of vertices for each cell (common mesh generator output) where all cells have the same celltype
5861 
5862   Collective; No Fortran Support
5863 
5864   Input Parameters:
5865 + dm          - The `DM`
5866 . numCells    - The number of cells owned by this process
5867 . numVertices - The number of vertices to be owned by this process, or `PETSC_DECIDE`
5868 . NVertices   - The global number of vertices, or `PETSC_DETERMINE`
5869 . numCorners  - The number of vertices for each cell
5870 - cells       - An array of numCells*numCorners numbers, the global vertex numbers for each cell
5871 
5872   Output Parameters:
5873 + vertexSF         - (Optional) `PetscSF` describing complete vertex ownership
5874 - verticesAdjSaved - (Optional) vertex adjacency array
5875 
5876   Level: advanced
5877 
5878   Notes:
5879   Two triangles sharing a face
5880 .vb
5881 
5882         2
5883       / | \
5884      /  |  \
5885     /   |   \
5886    0  0 | 1  3
5887     \   |   /
5888      \  |  /
5889       \ | /
5890         1
5891 .ve
5892   would have input
5893 .vb
5894   numCells = 2, numVertices = 4
5895   cells = [0 1 2  1 3 2]
5896 .ve
5897   which would result in the `DMPLEX`
5898 .vb
5899 
5900         4
5901       / | \
5902      /  |  \
5903     /   |   \
5904    2  0 | 1  5
5905     \   |   /
5906      \  |  /
5907       \ | /
5908         3
5909 .ve
5910 
5911   Vertices are implicitly numbered consecutively 0,...,NVertices.
5912   Each rank owns a chunk of numVertices consecutive vertices.
5913   If numVertices is `PETSC_DECIDE`, PETSc will distribute them as evenly as possible using PetscLayout.
5914   If NVertices is `PETSC_DETERMINE` and numVertices is PETSC_DECIDE, NVertices is computed by PETSc as the maximum vertex index in cells + 1.
5915   If only NVertices is `PETSC_DETERMINE`, it is computed as the sum of numVertices over all ranks.
5916 
5917   The cell distribution is arbitrary non-overlapping, independent of the vertex distribution.
5918 
5919 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexBuildFromCellList()`, `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildCoordinatesFromCellListParallel()`,
5920           `PetscSF`
5921 @*/
5922 PetscErrorCode DMPlexBuildFromCellListParallel(DM dm, PetscInt numCells, PetscInt numVertices, PetscInt NVertices, PetscInt numCorners, const PetscInt cells[], PeOp PetscSF *vertexSF, PeOp PetscInt *verticesAdjSaved[])
5923 {
5924   PetscSF     sfPoint;
5925   PetscLayout layout;
5926   PetscInt    numVerticesAdj, *verticesAdj, *cones, c, p;
5927 
5928   PetscFunctionBegin;
5929   PetscValidLogicalCollectiveInt(dm, NVertices, 4);
5930   PetscCall(PetscLogEventBegin(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
5931   /* Get/check global number of vertices */
5932   {
5933     PetscInt       NVerticesInCells, i;
5934     const PetscInt len = numCells * numCorners;
5935 
5936     /* NVerticesInCells = max(cells) + 1 */
5937     NVerticesInCells = PETSC_INT_MIN;
5938     for (i = 0; i < len; i++)
5939       if (cells[i] > NVerticesInCells) NVerticesInCells = cells[i];
5940     ++NVerticesInCells;
5941     PetscCallMPI(MPIU_Allreduce(MPI_IN_PLACE, &NVerticesInCells, 1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject)dm)));
5942 
5943     if (numVertices == PETSC_DECIDE && NVertices == PETSC_DECIDE) NVertices = NVerticesInCells;
5944     else
5945       PetscCheck(NVertices == PETSC_DECIDE || NVertices >= NVerticesInCells, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Specified global number of vertices %" PetscInt_FMT " must be greater than or equal to the number of vertices in cells %" PetscInt_FMT, NVertices, NVerticesInCells);
5946   }
5947   /* Count locally unique vertices */
5948   {
5949     PetscHSetI vhash;
5950     PetscInt   off = 0;
5951 
5952     PetscCall(PetscHSetICreate(&vhash));
5953     for (c = 0; c < numCells; ++c) {
5954       for (p = 0; p < numCorners; ++p) PetscCall(PetscHSetIAdd(vhash, cells[c * numCorners + p]));
5955     }
5956     PetscCall(PetscHSetIGetSize(vhash, &numVerticesAdj));
5957     if (!verticesAdjSaved) PetscCall(PetscMalloc1(numVerticesAdj, &verticesAdj));
5958     else verticesAdj = *verticesAdjSaved;
5959     PetscCall(PetscHSetIGetElems(vhash, &off, verticesAdj));
5960     PetscCall(PetscHSetIDestroy(&vhash));
5961     PetscCheck(off == numVerticesAdj, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid number of local vertices %" PetscInt_FMT " should be %" PetscInt_FMT, off, numVerticesAdj);
5962   }
5963   PetscCall(PetscSortInt(numVerticesAdj, verticesAdj));
5964   /* Create cones */
5965   PetscCall(DMPlexSetChart(dm, 0, numCells + numVerticesAdj));
5966   for (c = 0; c < numCells; ++c) PetscCall(DMPlexSetConeSize(dm, c, numCorners));
5967   PetscCall(DMSetUp(dm));
5968   PetscCall(DMPlexGetCones(dm, &cones));
5969   for (c = 0; c < numCells; ++c) {
5970     for (p = 0; p < numCorners; ++p) {
5971       const PetscInt gv = cells[c * numCorners + p];
5972       PetscInt       lv;
5973 
5974       /* Positions within verticesAdj form 0-based local vertex numbering;
5975          we need to shift it by numCells to get correct DAG points (cells go first) */
5976       PetscCall(PetscFindInt(gv, numVerticesAdj, verticesAdj, &lv));
5977       PetscCheck(lv >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Could not find global vertex %" PetscInt_FMT " in local connectivity", gv);
5978       cones[c * numCorners + p] = lv + numCells;
5979     }
5980   }
5981   /* Build point sf */
5982   PetscCall(PetscLayoutCreate(PetscObjectComm((PetscObject)dm), &layout));
5983   PetscCall(PetscLayoutSetSize(layout, NVertices));
5984   PetscCall(PetscLayoutSetLocalSize(layout, numVertices));
5985   PetscCall(PetscLayoutSetBlockSize(layout, 1));
5986   PetscCall(PetscSFCreateByMatchingIndices(layout, numVerticesAdj, verticesAdj, NULL, numCells, numVerticesAdj, verticesAdj, NULL, numCells, vertexSF, &sfPoint));
5987   PetscCall(PetscLayoutDestroy(&layout));
5988   if (!verticesAdjSaved) PetscCall(PetscFree(verticesAdj));
5989   PetscCall(PetscObjectSetName((PetscObject)sfPoint, "point SF"));
5990   if (dm->sf) {
5991     const char *prefix;
5992 
5993     PetscCall(PetscObjectGetOptionsPrefix((PetscObject)dm->sf, &prefix));
5994     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)sfPoint, prefix));
5995   }
5996   PetscCall(DMSetPointSF(dm, sfPoint));
5997   PetscCall(PetscSFDestroy(&sfPoint));
5998   if (vertexSF) PetscCall(PetscObjectSetName((PetscObject)*vertexSF, "Vertex Ownership SF"));
5999   /* Fill in the rest of the topology structure */
6000   PetscCall(DMPlexSymmetrize(dm));
6001   PetscCall(DMPlexStratify(dm));
6002   PetscCall(PetscLogEventEnd(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
6003   PetscFunctionReturn(PETSC_SUCCESS);
6004 }
6005 
6006 /*@C
6007   DMPlexBuildFromCellSectionParallel - Build distributed `DMPLEX` topology from a list of vertices for each cell (common mesh generator output) allowing multiple celltypes
6008 
6009   Collective; No Fortran Support
6010 
6011   Input Parameters:
6012 + dm          - The `DM`
6013 . numCells    - The number of cells owned by this process
6014 . numVertices - The number of vertices to be owned by this process, or `PETSC_DECIDE`
6015 . NVertices   - The global number of vertices, or `PETSC_DETERMINE`
6016 . cellSection - The `PetscSection` giving the number of vertices for each cell (layout of cells)
6017 - cells       - An array of the global vertex numbers for each cell
6018 
6019   Output Parameters:
6020 + vertexSF         - (Optional) `PetscSF` describing complete vertex ownership
6021 - verticesAdjSaved - (Optional) vertex adjacency array
6022 
6023   Level: advanced
6024 
6025   Notes:
6026   A triangle and quadrilateral sharing a face
6027 .vb
6028         2----------3
6029       / |          |
6030      /  |          |
6031     /   |          |
6032    0  0 |     1    |
6033     \   |          |
6034      \  |          |
6035       \ |          |
6036         1----------4
6037 .ve
6038   would have input
6039 .vb
6040   numCells = 2, numVertices = 5
6041   cells = [0 1 2  1 4 3 2]
6042 .ve
6043   which would result in the `DMPLEX`
6044 .vb
6045         4----------5
6046       / |          |
6047      /  |          |
6048     /   |          |
6049    2  0 |     1    |
6050     \   |          |
6051      \  |          |
6052       \ |          |
6053         3----------6
6054 .ve
6055 
6056   Vertices are implicitly numbered consecutively 0,...,NVertices.
6057   Each rank owns a chunk of numVertices consecutive vertices.
6058   If numVertices is `PETSC_DECIDE`, PETSc will distribute them as evenly as possible using PetscLayout.
6059   If NVertices is `PETSC_DETERMINE` and numVertices is PETSC_DECIDE, NVertices is computed by PETSc as the maximum vertex index in cells + 1.
6060   If only NVertices is `PETSC_DETERMINE`, it is computed as the sum of numVertices over all ranks.
6061 
6062   The cell distribution is arbitrary non-overlapping, independent of the vertex distribution.
6063 
6064 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexBuildFromCellListParallel()`, `DMPlexCreateFromCellSectionParallel()`, `DMPlexBuildCoordinatesFromCellListParallel()`,
6065           `PetscSF`
6066 @*/
6067 PetscErrorCode DMPlexBuildFromCellSectionParallel(DM dm, PetscInt numCells, PetscInt numVertices, PetscInt NVertices, PetscSection cellSection, const PetscInt cells[], PeOp PetscSF *vertexSF, PeOp PetscInt *verticesAdjSaved[])
6068 {
6069   PetscSF     sfPoint;
6070   PetscLayout layout;
6071   PetscInt    numVerticesAdj, *verticesAdj, *cones, cStart, cEnd, len;
6072 
6073   PetscFunctionBegin;
6074   PetscValidLogicalCollectiveInt(dm, NVertices, 4);
6075   PetscCall(PetscLogEventBegin(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
6076   PetscCall(PetscSectionGetChart(cellSection, &cStart, &cEnd));
6077   PetscCall(PetscSectionGetStorageSize(cellSection, &len));
6078   PetscCheck(cStart == 0 && cEnd == numCells, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Section chart [%" PetscInt_FMT ", %" PetscInt_FMT ") should be [0, %" PetscInt_FMT ")", cStart, cEnd, numCells);
6079   /* Get/check global number of vertices */
6080   {
6081     PetscInt NVerticesInCells;
6082 
6083     /* NVerticesInCells = max(cells) + 1 */
6084     NVerticesInCells = PETSC_MIN_INT;
6085     for (PetscInt i = 0; i < len; i++)
6086       if (cells[i] > NVerticesInCells) NVerticesInCells = cells[i];
6087     ++NVerticesInCells;
6088     PetscCallMPI(MPIU_Allreduce(MPI_IN_PLACE, &NVerticesInCells, 1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject)dm)));
6089 
6090     if (numVertices == PETSC_DECIDE && NVertices == PETSC_DECIDE) NVertices = NVerticesInCells;
6091     else
6092       PetscCheck(NVertices == PETSC_DECIDE || NVertices >= NVerticesInCells, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Specified global number of vertices %" PetscInt_FMT " must be greater than or equal to the number of vertices in cells %" PetscInt_FMT, NVertices, NVerticesInCells);
6093   }
6094   /* Count locally unique vertices */
6095   {
6096     PetscHSetI vhash;
6097     PetscInt   off = 0;
6098 
6099     PetscCall(PetscHSetICreate(&vhash));
6100     for (PetscInt i = 0; i < len; i++) PetscCall(PetscHSetIAdd(vhash, cells[i]));
6101     PetscCall(PetscHSetIGetSize(vhash, &numVerticesAdj));
6102     if (!verticesAdjSaved) PetscCall(PetscMalloc1(numVerticesAdj, &verticesAdj));
6103     else verticesAdj = *verticesAdjSaved;
6104     PetscCall(PetscHSetIGetElems(vhash, &off, verticesAdj));
6105     PetscCall(PetscHSetIDestroy(&vhash));
6106     PetscCheck(off == numVerticesAdj, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid number of local vertices %" PetscInt_FMT " should be %" PetscInt_FMT, off, numVerticesAdj);
6107   }
6108   PetscCall(PetscSortInt(numVerticesAdj, verticesAdj));
6109   /* Create cones */
6110   PetscCall(DMPlexSetChart(dm, 0, numCells + numVerticesAdj));
6111   for (PetscInt c = 0; c < numCells; ++c) {
6112     PetscInt dof;
6113 
6114     PetscCall(PetscSectionGetDof(cellSection, c, &dof));
6115     PetscCall(DMPlexSetConeSize(dm, c, dof));
6116   }
6117   PetscCall(DMSetUp(dm));
6118   PetscCall(DMPlexGetCones(dm, &cones));
6119   for (PetscInt c = 0; c < numCells; ++c) {
6120     PetscInt dof, off;
6121 
6122     PetscCall(PetscSectionGetDof(cellSection, c, &dof));
6123     PetscCall(PetscSectionGetOffset(cellSection, c, &off));
6124     for (PetscInt p = off; p < off + dof; ++p) {
6125       const PetscInt gv = cells[p];
6126       PetscInt       lv;
6127 
6128       /* Positions within verticesAdj form 0-based local vertex numbering;
6129          we need to shift it by numCells to get correct DAG points (cells go first) */
6130       PetscCall(PetscFindInt(gv, numVerticesAdj, verticesAdj, &lv));
6131       PetscCheck(lv >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Could not find global vertex %" PetscInt_FMT " in local connectivity", gv);
6132       cones[p] = lv + numCells;
6133     }
6134   }
6135   /* Build point sf */
6136   PetscCall(PetscLayoutCreate(PetscObjectComm((PetscObject)dm), &layout));
6137   PetscCall(PetscLayoutSetSize(layout, NVertices));
6138   PetscCall(PetscLayoutSetLocalSize(layout, numVertices));
6139   PetscCall(PetscLayoutSetBlockSize(layout, 1));
6140   PetscCall(PetscSFCreateByMatchingIndices(layout, numVerticesAdj, verticesAdj, NULL, numCells, numVerticesAdj, verticesAdj, NULL, numCells, vertexSF, &sfPoint));
6141   PetscCall(PetscLayoutDestroy(&layout));
6142   if (!verticesAdjSaved) PetscCall(PetscFree(verticesAdj));
6143   PetscCall(PetscObjectSetName((PetscObject)sfPoint, "point SF"));
6144   if (dm->sf) {
6145     const char *prefix;
6146 
6147     PetscCall(PetscObjectGetOptionsPrefix((PetscObject)dm->sf, &prefix));
6148     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)sfPoint, prefix));
6149   }
6150   PetscCall(DMSetPointSF(dm, sfPoint));
6151   PetscCall(PetscSFDestroy(&sfPoint));
6152   if (vertexSF) PetscCall(PetscObjectSetName((PetscObject)*vertexSF, "Vertex Ownership SF"));
6153   /* Fill in the rest of the topology structure */
6154   PetscCall(DMPlexSymmetrize(dm));
6155   PetscCall(DMPlexStratify(dm));
6156   PetscCall(PetscLogEventEnd(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
6157   PetscFunctionReturn(PETSC_SUCCESS);
6158 }
6159 
6160 /*@
6161   DMPlexBuildCoordinatesFromCellListParallel - Build `DM` coordinates from a list of coordinates for each owned vertex (common mesh generator output)
6162 
6163   Collective; No Fortran Support
6164 
6165   Input Parameters:
6166 + dm           - The `DM`
6167 . spaceDim     - The spatial dimension used for coordinates
6168 . sfVert       - `PetscSF` describing complete vertex ownership
6169 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
6170 
6171   Level: advanced
6172 
6173 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildFromCellListParallel()`
6174 @*/
6175 PetscErrorCode DMPlexBuildCoordinatesFromCellListParallel(DM dm, PetscInt spaceDim, PetscSF sfVert, const PetscReal vertexCoords[])
6176 {
6177   PetscSection coordSection;
6178   Vec          coordinates;
6179   PetscScalar *coords;
6180   PetscInt     numVertices, numVerticesAdj, coordSize, v, vStart, vEnd;
6181   PetscMPIInt  spaceDimi;
6182 
6183   PetscFunctionBegin;
6184   PetscCall(PetscLogEventBegin(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
6185   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
6186   PetscCheck(vStart >= 0 && vEnd >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM is not set up properly. DMPlexBuildFromCellList() should be called first.");
6187   PetscCall(DMSetCoordinateDim(dm, spaceDim));
6188   PetscCall(PetscSFGetGraph(sfVert, &numVertices, &numVerticesAdj, NULL, NULL));
6189   PetscCheck(vEnd - vStart == numVerticesAdj, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Supplied sfVert has wrong number of leaves = %" PetscInt_FMT " != %" PetscInt_FMT " = vEnd - vStart", numVerticesAdj, vEnd - vStart);
6190   PetscCall(DMGetCoordinateSection(dm, &coordSection));
6191   PetscCall(PetscSectionSetNumFields(coordSection, 1));
6192   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, spaceDim));
6193   PetscCall(PetscSectionSetChart(coordSection, vStart, vEnd));
6194   for (v = vStart; v < vEnd; ++v) {
6195     PetscCall(PetscSectionSetDof(coordSection, v, spaceDim));
6196     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, spaceDim));
6197   }
6198   PetscCall(PetscSectionSetUp(coordSection));
6199   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
6200   PetscCall(VecCreate(PetscObjectComm((PetscObject)dm), &coordinates));
6201   PetscCall(VecSetBlockSize(coordinates, spaceDim));
6202   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
6203   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
6204   PetscCall(VecSetType(coordinates, VECSTANDARD));
6205   PetscCall(VecGetArray(coordinates, &coords));
6206   {
6207     MPI_Datatype coordtype;
6208 
6209     /* Need a temp buffer for coords if we have complex/single */
6210     PetscCall(PetscMPIIntCast(spaceDim, &spaceDimi));
6211     PetscCallMPI(MPI_Type_contiguous(spaceDimi, MPIU_SCALAR, &coordtype));
6212     PetscCallMPI(MPI_Type_commit(&coordtype));
6213 #if defined(PETSC_USE_COMPLEX)
6214     {
6215       PetscScalar *svertexCoords;
6216       PetscInt     i;
6217       PetscCall(PetscMalloc1(numVertices * spaceDim, &svertexCoords));
6218       for (i = 0; i < numVertices * spaceDim; i++) svertexCoords[i] = vertexCoords[i];
6219       PetscCall(PetscSFBcastBegin(sfVert, coordtype, svertexCoords, coords, MPI_REPLACE));
6220       PetscCall(PetscSFBcastEnd(sfVert, coordtype, svertexCoords, coords, MPI_REPLACE));
6221       PetscCall(PetscFree(svertexCoords));
6222     }
6223 #else
6224     PetscCall(PetscSFBcastBegin(sfVert, coordtype, vertexCoords, coords, MPI_REPLACE));
6225     PetscCall(PetscSFBcastEnd(sfVert, coordtype, vertexCoords, coords, MPI_REPLACE));
6226 #endif
6227     PetscCallMPI(MPI_Type_free(&coordtype));
6228   }
6229   PetscCall(VecRestoreArray(coordinates, &coords));
6230   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
6231   PetscCall(VecDestroy(&coordinates));
6232   PetscCall(PetscLogEventEnd(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
6233   PetscFunctionReturn(PETSC_SUCCESS);
6234 }
6235 
6236 /*@
6237   DMPlexCreateFromCellListParallelPetsc - Create distributed `DMPLEX` from a list of vertices for each cell (common mesh generator output) where all cells have the same celltype
6238 
6239   Collective
6240 
6241   Input Parameters:
6242 + comm         - The communicator
6243 . dim          - The topological dimension of the mesh
6244 . numCells     - The number of cells owned by this process
6245 . numVertices  - The number of vertices owned by this process, or `PETSC_DECIDE`
6246 . NVertices    - The global number of vertices, or `PETSC_DECIDE`
6247 . numCorners   - The number of vertices for each cell
6248 . interpolate  - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
6249 . cells        - An array of numCells*numCorners numbers, the global vertex numbers for each cell
6250 . spaceDim     - The spatial dimension used for coordinates
6251 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
6252 
6253   Output Parameters:
6254 + dm          - The `DM`
6255 . vertexSF    - (Optional) `PetscSF` describing complete vertex ownership
6256 - verticesAdj - (Optional) vertex adjacency array
6257 
6258   Level: intermediate
6259 
6260   Notes:
6261   This function is just a convenient sequence of `DMCreate()`, `DMSetType()`, `DMSetDimension()`,
6262   `DMPlexBuildFromCellListParallel()`, `DMPlexInterpolate()`, `DMPlexBuildCoordinatesFromCellListParallel()`
6263 
6264   See `DMPlexBuildFromCellListParallel()` for an example and details about the topology-related parameters.
6265 
6266   See `DMPlexBuildCoordinatesFromCellListParallel()` for details about the geometry-related parameters.
6267 
6268 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromCellListPetsc()`, `DMPlexBuildFromCellListParallel()`, `DMPlexBuildCoordinatesFromCellListParallel()`, `DMPlexCreateFromDAG()`, `DMPlexCreate()`
6269 @*/
6270 PetscErrorCode DMPlexCreateFromCellListParallelPetsc(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt NVertices, PetscInt numCorners, PetscBool interpolate, const PetscInt cells[], PetscInt spaceDim, const PetscReal vertexCoords[], PeOp PetscSF *vertexSF, PeOp PetscInt *verticesAdj[], DM *dm)
6271 {
6272   PetscSF sfVert;
6273 
6274   PetscFunctionBegin;
6275   PetscCall(DMCreate(comm, dm));
6276   PetscCall(DMSetType(*dm, DMPLEX));
6277   PetscValidLogicalCollectiveInt(*dm, dim, 2);
6278   PetscValidLogicalCollectiveInt(*dm, spaceDim, 9);
6279   PetscCall(DMSetDimension(*dm, dim));
6280   PetscCall(DMPlexBuildFromCellListParallel(*dm, numCells, numVertices, NVertices, numCorners, cells, &sfVert, verticesAdj));
6281   if (interpolate) {
6282     DM idm;
6283 
6284     PetscCall(DMPlexInterpolate(*dm, &idm));
6285     PetscCall(DMDestroy(dm));
6286     *dm = idm;
6287   }
6288   PetscCall(DMPlexBuildCoordinatesFromCellListParallel(*dm, spaceDim, sfVert, vertexCoords));
6289   if (vertexSF) *vertexSF = sfVert;
6290   else PetscCall(PetscSFDestroy(&sfVert));
6291   PetscFunctionReturn(PETSC_SUCCESS);
6292 }
6293 
6294 /*@
6295   DMPlexCreateFromCellSectionParallel - Create distributed `DMPLEX` from a list of vertices for each cell (common mesh generator output) and supports multiple celltypes
6296 
6297   Collective
6298 
6299   Input Parameters:
6300 + comm         - The communicator
6301 . dim          - The topological dimension of the mesh
6302 . numCells     - The number of cells owned by this process
6303 . numVertices  - The number of vertices owned by this process, or `PETSC_DECIDE`
6304 . NVertices    - The global number of vertices, or `PETSC_DECIDE`
6305 . cellSection  - The `PetscSection` giving the number of vertices for each cell (layout of cells)
6306 . interpolate  - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
6307 . cells        - An array of the global vertex numbers for each cell
6308 . spaceDim     - The spatial dimension used for coordinates
6309 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
6310 
6311   Output Parameters:
6312 + dm          - The `DM`
6313 . vertexSF    - (Optional) `PetscSF` describing complete vertex ownership
6314 - verticesAdj - (Optional) vertex adjacency array
6315 
6316   Level: intermediate
6317 
6318   Notes:
6319   This function is just a convenient sequence of `DMCreate()`, `DMSetType()`, `DMSetDimension()`,
6320   `DMPlexBuildFromCellSectionParallel()`, `DMPlexInterpolate()`, `DMPlexBuildCoordinatesFromCellListParallel()`
6321 
6322   See `DMPlexBuildFromCellSectionParallel()` for an example and details about the topology-related parameters.
6323 
6324   See `DMPlexBuildCoordinatesFromCellListParallel()` for details about the geometry-related parameters.
6325 
6326 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromCellListPetsc()`, `DMPlexBuildFromCellListParallel()`, `DMPlexBuildCoordinatesFromCellListParallel()`, `DMPlexCreateFromDAG()`, `DMPlexCreate()`
6327 @*/
6328 PetscErrorCode DMPlexCreateFromCellSectionParallel(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt NVertices, PetscSection cellSection, PetscBool interpolate, const PetscInt cells[], PetscInt spaceDim, const PetscReal vertexCoords[], PeOp PetscSF *vertexSF, PeOp PetscInt *verticesAdj[], DM *dm)
6329 {
6330   PetscSF sfVert;
6331 
6332   PetscFunctionBegin;
6333   PetscCall(DMCreate(comm, dm));
6334   PetscCall(DMSetType(*dm, DMPLEX));
6335   PetscValidLogicalCollectiveInt(*dm, dim, 2);
6336   PetscValidLogicalCollectiveInt(*dm, spaceDim, 9);
6337   PetscCall(DMSetDimension(*dm, dim));
6338   PetscCall(DMPlexBuildFromCellSectionParallel(*dm, numCells, numVertices, NVertices, cellSection, cells, &sfVert, verticesAdj));
6339   if (interpolate) {
6340     DM idm;
6341 
6342     PetscCall(DMPlexInterpolate(*dm, &idm));
6343     PetscCall(DMDestroy(dm));
6344     *dm = idm;
6345   }
6346   PetscCall(DMPlexBuildCoordinatesFromCellListParallel(*dm, spaceDim, sfVert, vertexCoords));
6347   if (vertexSF) *vertexSF = sfVert;
6348   else PetscCall(PetscSFDestroy(&sfVert));
6349   PetscFunctionReturn(PETSC_SUCCESS);
6350 }
6351 
6352 /*@
6353   DMPlexBuildFromCellList - Build `DMPLEX` topology from a list of vertices for each cell (common mesh generator output)
6354 
6355   Collective; No Fortran Support
6356 
6357   Input Parameters:
6358 + dm          - The `DM`
6359 . numCells    - The number of cells owned by this process
6360 . numVertices - The number of vertices owned by this process, or `PETSC_DETERMINE`
6361 . numCorners  - The number of vertices for each cell
6362 - cells       - An array of `numCells` x `numCorners` numbers, the global vertex numbers for each cell
6363 
6364   Level: advanced
6365 
6366   Notes:
6367   Two triangles sharing a face
6368 .vb
6369 
6370         2
6371       / | \
6372      /  |  \
6373     /   |   \
6374    0  0 | 1  3
6375     \   |   /
6376      \  |  /
6377       \ | /
6378         1
6379 .ve
6380   would have input
6381 .vb
6382   numCells = 2, numVertices = 4
6383   cells = [0 1 2  1 3 2]
6384 .ve
6385   which would result in the `DMPLEX`
6386 .vb
6387 
6388         4
6389       / | \
6390      /  |  \
6391     /   |   \
6392    2  0 | 1  5
6393     \   |   /
6394      \  |  /
6395       \ | /
6396         3
6397 .ve
6398 
6399   If numVertices is `PETSC_DETERMINE`, it is computed by PETSc as the maximum vertex index in cells + 1.
6400 
6401 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexBuildFromCellListParallel()`, `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromCellListPetsc()`
6402 @*/
6403 PetscErrorCode DMPlexBuildFromCellList(DM dm, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, const PetscInt cells[])
6404 {
6405   PetscInt *cones, c, p, dim;
6406 
6407   PetscFunctionBegin;
6408   PetscCall(PetscLogEventBegin(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
6409   PetscCall(DMGetDimension(dm, &dim));
6410   /* Get/check global number of vertices */
6411   {
6412     PetscInt       NVerticesInCells, i;
6413     const PetscInt len = numCells * numCorners;
6414 
6415     /* NVerticesInCells = max(cells) + 1 */
6416     NVerticesInCells = PETSC_INT_MIN;
6417     for (i = 0; i < len; i++)
6418       if (cells[i] > NVerticesInCells) NVerticesInCells = cells[i];
6419     ++NVerticesInCells;
6420 
6421     if (numVertices == PETSC_DECIDE) numVertices = NVerticesInCells;
6422     else
6423       PetscCheck(numVertices >= NVerticesInCells, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Specified number of vertices %" PetscInt_FMT " must be greater than or equal to the number of vertices in cells %" PetscInt_FMT, numVertices, NVerticesInCells);
6424   }
6425   PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
6426   for (c = 0; c < numCells; ++c) PetscCall(DMPlexSetConeSize(dm, c, numCorners));
6427   PetscCall(DMSetUp(dm));
6428   PetscCall(DMPlexGetCones(dm, &cones));
6429   for (c = 0; c < numCells; ++c) {
6430     for (p = 0; p < numCorners; ++p) cones[c * numCorners + p] = cells[c * numCorners + p] + numCells;
6431   }
6432   PetscCall(DMPlexSymmetrize(dm));
6433   PetscCall(DMPlexStratify(dm));
6434   PetscCall(PetscLogEventEnd(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
6435   PetscFunctionReturn(PETSC_SUCCESS);
6436 }
6437 
6438 /*@
6439   DMPlexBuildCoordinatesFromCellList - Build `DM` coordinates from a list of coordinates for each owned vertex (common mesh generator output)
6440 
6441   Collective
6442 
6443   Input Parameters:
6444 + dm           - The `DM`
6445 . spaceDim     - The spatial dimension used for coordinates
6446 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
6447 
6448   Level: advanced
6449 
6450 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexBuildCoordinatesFromCellListParallel()`, `DMPlexCreateFromCellListPetsc()`, `DMPlexBuildFromCellList()`
6451 @*/
6452 PetscErrorCode DMPlexBuildCoordinatesFromCellList(DM dm, PetscInt spaceDim, const PetscReal vertexCoords[])
6453 {
6454   PetscSection coordSection;
6455   Vec          coordinates;
6456   DM           cdm;
6457   PetscScalar *coords;
6458   PetscInt     v, vStart, vEnd, d;
6459 
6460   PetscFunctionBegin;
6461   PetscCall(PetscLogEventBegin(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
6462   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
6463   PetscCheck(vStart >= 0 && vEnd >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM is not set up properly. DMPlexBuildFromCellList() should be called first.");
6464   PetscCall(DMSetCoordinateDim(dm, spaceDim));
6465   PetscCall(DMGetCoordinateSection(dm, &coordSection));
6466   PetscCall(PetscSectionSetNumFields(coordSection, 1));
6467   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, spaceDim));
6468   PetscCall(PetscSectionSetChart(coordSection, vStart, vEnd));
6469   for (v = vStart; v < vEnd; ++v) {
6470     PetscCall(PetscSectionSetDof(coordSection, v, spaceDim));
6471     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, spaceDim));
6472   }
6473   PetscCall(PetscSectionSetUp(coordSection));
6474 
6475   PetscCall(DMGetCoordinateDM(dm, &cdm));
6476   PetscCall(DMCreateLocalVector(cdm, &coordinates));
6477   PetscCall(VecSetBlockSize(coordinates, spaceDim));
6478   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
6479   PetscCall(VecGetArrayWrite(coordinates, &coords));
6480   for (v = 0; v < vEnd - vStart; ++v) {
6481     for (d = 0; d < spaceDim; ++d) coords[v * spaceDim + d] = vertexCoords[v * spaceDim + d];
6482   }
6483   PetscCall(VecRestoreArrayWrite(coordinates, &coords));
6484   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
6485   PetscCall(VecDestroy(&coordinates));
6486   PetscCall(PetscLogEventEnd(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
6487   PetscFunctionReturn(PETSC_SUCCESS);
6488 }
6489 
6490 /*@
6491   DMPlexCreateFromCellListPetsc - Create `DMPLEX` from a list of vertices for each cell (common mesh generator output), but only process 0 takes in the input
6492 
6493   Collective
6494 
6495   Input Parameters:
6496 + comm         - The communicator
6497 . dim          - The topological dimension of the mesh
6498 . numCells     - The number of cells, only on process 0
6499 . numVertices  - The number of vertices owned by this process, or `PETSC_DECIDE`, only on process 0
6500 . numCorners   - The number of vertices for each cell, only on process 0
6501 . interpolate  - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
6502 . cells        - An array of $ numCells \times numCorners$ numbers, the vertices for each cell, only on process 0
6503 . spaceDim     - The spatial dimension used for coordinates
6504 - vertexCoords - An array of $ numVertices \times spaceDim$ numbers, the coordinates of each vertex, only on process 0
6505 
6506   Output Parameter:
6507 . dm - The `DM`, which only has points on process 0
6508 
6509   Level: intermediate
6510 
6511   Notes:
6512   This function is just a convenient sequence of `DMCreate()`, `DMSetType()`, `DMSetDimension()`, `DMPlexBuildFromCellList()`,
6513   `DMPlexInterpolate()`, `DMPlexBuildCoordinatesFromCellList()`
6514 
6515   See `DMPlexBuildFromCellList()` for an example and details about the topology-related parameters.
6516   See `DMPlexBuildCoordinatesFromCellList()` for details about the geometry-related parameters.
6517   See `DMPlexCreateFromCellListParallelPetsc()` for parallel input
6518 
6519 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildFromCellList()`, `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromDAG()`, `DMPlexCreate()`
6520 @*/
6521 PetscErrorCode DMPlexCreateFromCellListPetsc(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, PetscBool interpolate, const PetscInt cells[], PetscInt spaceDim, const PetscReal vertexCoords[], DM *dm)
6522 {
6523   PetscMPIInt rank;
6524 
6525   PetscFunctionBegin;
6526   PetscCheck(dim, comm, PETSC_ERR_ARG_OUTOFRANGE, "This is not appropriate for 0-dimensional meshes. Consider either creating the DM using DMPlexCreateFromDAG(), by hand, or using DMSwarm.");
6527   PetscCallMPI(MPI_Comm_rank(comm, &rank));
6528   PetscCall(DMCreate(comm, dm));
6529   PetscCall(DMSetType(*dm, DMPLEX));
6530   PetscCall(DMSetDimension(*dm, dim));
6531   if (rank == 0) PetscCall(DMPlexBuildFromCellList(*dm, numCells, numVertices, numCorners, cells));
6532   else PetscCall(DMPlexBuildFromCellList(*dm, 0, 0, 0, NULL));
6533   if (interpolate) {
6534     DM idm;
6535 
6536     PetscCall(DMPlexInterpolate(*dm, &idm));
6537     PetscCall(DMDestroy(dm));
6538     *dm = idm;
6539   }
6540   if (rank == 0) PetscCall(DMPlexBuildCoordinatesFromCellList(*dm, spaceDim, vertexCoords));
6541   else PetscCall(DMPlexBuildCoordinatesFromCellList(*dm, spaceDim, NULL));
6542   PetscFunctionReturn(PETSC_SUCCESS);
6543 }
6544 
6545 /*@
6546   DMPlexCreateFromDAG - This takes as input the adjacency-list representation of the Directed Acyclic Graph (Hasse Diagram) encoding a mesh, and produces a `DM`
6547 
6548   Input Parameters:
6549 + dm               - The empty `DM` object, usually from `DMCreate()` and `DMSetDimension()`
6550 . depth            - The depth of the DAG
6551 . numPoints        - Array of size $ depth + 1 $ containing the number of points at each `depth`
6552 . coneSize         - The cone size of each point
6553 . cones            - The concatenation of the cone points for each point, the cone list must be oriented correctly for each point
6554 . coneOrientations - The orientation of each cone point
6555 - vertexCoords     - An array of $ numPoints[0] \times spacedim $ numbers representing the coordinates of each vertex, with `spacedim` the value set via `DMSetCoordinateDim()`
6556 
6557   Output Parameter:
6558 . dm - The `DM`
6559 
6560   Level: advanced
6561 
6562   Note:
6563   Two triangles sharing a face would have input
6564 .vb
6565   depth = 1, numPoints = [4 2], coneSize = [3 3 0 0 0 0]
6566   cones = [2 3 4  3 5 4], coneOrientations = [0 0 0  0 0 0]
6567  vertexCoords = [-1.0 0.0  0.0 -1.0  0.0 1.0  1.0 0.0]
6568 .ve
6569   which would result in the DMPlex
6570 .vb
6571         4
6572       / | \
6573      /  |  \
6574     /   |   \
6575    2  0 | 1  5
6576     \   |   /
6577      \  |  /
6578       \ | /
6579         3
6580 .ve
6581   Notice that all points are numbered consecutively, unlike `DMPlexCreateFromCellListPetsc()`
6582 
6583   Developer Note:
6584   This does not create anything so should not have create in the name.
6585 
6586 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromCellListPetsc()`, `DMPlexCreate()`
6587 @*/
6588 PetscErrorCode DMPlexCreateFromDAG(DM dm, PetscInt depth, const PetscInt numPoints[], const PetscInt coneSize[], const PetscInt cones[], const PetscInt coneOrientations[], const PetscScalar vertexCoords[])
6589 {
6590   Vec          coordinates;
6591   PetscSection coordSection;
6592   PetscScalar *coords;
6593   PetscInt     coordSize, firstVertex = -1, pStart = 0, pEnd = 0, p, v, dim, dimEmbed, d, off;
6594 
6595   PetscFunctionBegin;
6596   PetscCall(DMGetDimension(dm, &dim));
6597   PetscCall(DMGetCoordinateDim(dm, &dimEmbed));
6598   PetscCheck(dimEmbed >= dim, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Embedding dimension %" PetscInt_FMT " cannot be less than intrinsic dimension %" PetscInt_FMT, dimEmbed, dim);
6599   for (d = 0; d <= depth; ++d) pEnd += numPoints[d];
6600   PetscCall(DMPlexSetChart(dm, pStart, pEnd));
6601   for (p = pStart; p < pEnd; ++p) {
6602     PetscCall(DMPlexSetConeSize(dm, p, coneSize[p - pStart]));
6603     if (firstVertex < 0 && !coneSize[p - pStart]) firstVertex = p - pStart;
6604   }
6605   PetscCheck(firstVertex >= 0 || !numPoints[0], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Expected %" PetscInt_FMT " vertices but could not find any", numPoints[0]);
6606   PetscCall(DMSetUp(dm)); /* Allocate space for cones */
6607   for (p = pStart, off = 0; p < pEnd; off += coneSize[p - pStart], ++p) {
6608     PetscCall(DMPlexSetCone(dm, p, &cones[off]));
6609     PetscCall(DMPlexSetConeOrientation(dm, p, &coneOrientations[off]));
6610   }
6611   PetscCall(DMPlexSymmetrize(dm));
6612   PetscCall(DMPlexStratify(dm));
6613   /* Build coordinates */
6614   PetscCall(DMGetCoordinateSection(dm, &coordSection));
6615   PetscCall(PetscSectionSetNumFields(coordSection, 1));
6616   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dimEmbed));
6617   PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numPoints[0]));
6618   for (v = firstVertex; v < firstVertex + numPoints[0]; ++v) {
6619     PetscCall(PetscSectionSetDof(coordSection, v, dimEmbed));
6620     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dimEmbed));
6621   }
6622   PetscCall(PetscSectionSetUp(coordSection));
6623   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
6624   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
6625   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
6626   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
6627   PetscCall(VecSetBlockSize(coordinates, PetscMax(dimEmbed, 1)));
6628   PetscCall(VecSetType(coordinates, VECSTANDARD));
6629   if (vertexCoords) {
6630     PetscCall(VecGetArray(coordinates, &coords));
6631     for (v = 0; v < numPoints[0]; ++v) {
6632       PetscInt off;
6633 
6634       PetscCall(PetscSectionGetOffset(coordSection, v + firstVertex, &off));
6635       for (d = 0; d < dimEmbed; ++d) coords[off + d] = vertexCoords[v * dimEmbed + d];
6636     }
6637   }
6638   PetscCall(VecRestoreArray(coordinates, &coords));
6639   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
6640   PetscCall(VecDestroy(&coordinates));
6641   PetscFunctionReturn(PETSC_SUCCESS);
6642 }
6643 
6644 /*
6645   DMPlexCreateCellVertexFromFile - Create a `DMPLEX` mesh from a simple cell-vertex file.
6646 
6647   Collective
6648 
6649 + comm        - The MPI communicator
6650 . filename    - Name of the .dat file
6651 - interpolate - Create faces and edges in the mesh
6652 
6653   Output Parameter:
6654 . dm  - The `DM` object representing the mesh
6655 
6656   Level: beginner
6657 
6658   Note:
6659   The format is the simplest possible:
6660 .vb
6661   dim Ne Nv Nc Nl
6662   v_1 v_2 ... v_Nc
6663   ...
6664   x y z marker_1 ... marker_Nl
6665 .ve
6666 
6667   Developer Note:
6668   Should use a `PetscViewer` not a filename
6669 
6670 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromFile()`, `DMPlexCreateGmsh()`, `DMPlexCreate()`
6671 */
6672 static PetscErrorCode DMPlexCreateCellVertexFromFile(MPI_Comm comm, const char filename[], PetscBool interpolate, DM *dm)
6673 {
6674   DMLabel      marker;
6675   PetscViewer  viewer;
6676   Vec          coordinates;
6677   PetscSection coordSection;
6678   PetscScalar *coords;
6679   char         line[PETSC_MAX_PATH_LEN];
6680   PetscInt     cdim, coordSize, v, c, d;
6681   PetscMPIInt  rank;
6682   int          snum, dim, Nv, Nc, Ncn, Nl;
6683 
6684   PetscFunctionBegin;
6685   PetscCallMPI(MPI_Comm_rank(comm, &rank));
6686   PetscCall(PetscViewerCreate(comm, &viewer));
6687   PetscCall(PetscViewerSetType(viewer, PETSCVIEWERASCII));
6688   PetscCall(PetscViewerFileSetMode(viewer, FILE_MODE_READ));
6689   PetscCall(PetscViewerFileSetName(viewer, filename));
6690   if (rank == 0) {
6691     PetscCall(PetscViewerRead(viewer, line, 5, NULL, PETSC_STRING));
6692     snum = sscanf(line, "%d %d %d %d %d", &dim, &Nc, &Nv, &Ncn, &Nl);
6693     PetscCheck(snum == 5, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
6694   } else {
6695     Nc = Nv = Ncn = Nl = 0;
6696   }
6697   PetscCallMPI(MPI_Bcast(&dim, 1, MPI_INT, 0, comm));
6698   cdim = dim;
6699   PetscCall(DMCreate(comm, dm));
6700   PetscCall(DMSetType(*dm, DMPLEX));
6701   PetscCall(DMPlexSetChart(*dm, 0, Nc + Nv));
6702   PetscCall(DMSetDimension(*dm, dim));
6703   PetscCall(DMSetCoordinateDim(*dm, cdim));
6704   /* Read topology */
6705   if (rank == 0) {
6706     char     format[PETSC_MAX_PATH_LEN];
6707     PetscInt cone[8];
6708     int      vbuf[8], v;
6709 
6710     for (c = 0; c < Ncn; ++c) {
6711       format[c * 3 + 0] = '%';
6712       format[c * 3 + 1] = 'd';
6713       format[c * 3 + 2] = ' ';
6714     }
6715     format[Ncn * 3 - 1] = '\0';
6716     for (c = 0; c < Nc; ++c) PetscCall(DMPlexSetConeSize(*dm, c, Ncn));
6717     PetscCall(DMSetUp(*dm));
6718     for (c = 0; c < Nc; ++c) {
6719       PetscCall(PetscViewerRead(viewer, line, Ncn, NULL, PETSC_STRING));
6720       switch (Ncn) {
6721       case 2:
6722         snum = sscanf(line, format, &vbuf[0], &vbuf[1]);
6723         break;
6724       case 3:
6725         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2]);
6726         break;
6727       case 4:
6728         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3]);
6729         break;
6730       case 6:
6731         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3], &vbuf[4], &vbuf[5]);
6732         break;
6733       case 8:
6734         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3], &vbuf[4], &vbuf[5], &vbuf[6], &vbuf[7]);
6735         break;
6736       default:
6737         SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "No cell shape with %d vertices", Ncn);
6738       }
6739       PetscCheck(snum == Ncn, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
6740       for (v = 0; v < Ncn; ++v) cone[v] = vbuf[v] + Nc;
6741       /* Hexahedra are inverted */
6742       if (Ncn == 8) {
6743         PetscInt tmp = cone[1];
6744         cone[1]      = cone[3];
6745         cone[3]      = tmp;
6746       }
6747       PetscCall(DMPlexSetCone(*dm, c, cone));
6748     }
6749   }
6750   PetscCall(DMPlexSymmetrize(*dm));
6751   PetscCall(DMPlexStratify(*dm));
6752   /* Read coordinates */
6753   PetscCall(DMGetCoordinateSection(*dm, &coordSection));
6754   PetscCall(PetscSectionSetNumFields(coordSection, 1));
6755   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, cdim));
6756   PetscCall(PetscSectionSetChart(coordSection, Nc, Nc + Nv));
6757   for (v = Nc; v < Nc + Nv; ++v) {
6758     PetscCall(PetscSectionSetDof(coordSection, v, cdim));
6759     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, cdim));
6760   }
6761   PetscCall(PetscSectionSetUp(coordSection));
6762   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
6763   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
6764   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
6765   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
6766   PetscCall(VecSetBlockSize(coordinates, cdim));
6767   PetscCall(VecSetType(coordinates, VECSTANDARD));
6768   PetscCall(VecGetArray(coordinates, &coords));
6769   if (rank == 0) {
6770     char   format[PETSC_MAX_PATH_LEN];
6771     double x[3];
6772     int    l, val[3];
6773 
6774     if (Nl) {
6775       for (l = 0; l < Nl; ++l) {
6776         format[l * 3 + 0] = '%';
6777         format[l * 3 + 1] = 'd';
6778         format[l * 3 + 2] = ' ';
6779       }
6780       format[Nl * 3 - 1] = '\0';
6781       PetscCall(DMCreateLabel(*dm, "marker"));
6782       PetscCall(DMGetLabel(*dm, "marker", &marker));
6783     }
6784     for (v = 0; v < Nv; ++v) {
6785       PetscCall(PetscViewerRead(viewer, line, 3 + Nl, NULL, PETSC_STRING));
6786       snum = sscanf(line, "%lg %lg %lg", &x[0], &x[1], &x[2]);
6787       PetscCheck(snum == 3, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
6788       switch (Nl) {
6789       case 0:
6790         snum = 0;
6791         break;
6792       case 1:
6793         snum = sscanf(line, format, &val[0]);
6794         break;
6795       case 2:
6796         snum = sscanf(line, format, &val[0], &val[1]);
6797         break;
6798       case 3:
6799         snum = sscanf(line, format, &val[0], &val[1], &val[2]);
6800         break;
6801       default:
6802         SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Request support for %d labels", Nl);
6803       }
6804       PetscCheck(snum == Nl, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
6805       for (d = 0; d < cdim; ++d) coords[v * cdim + d] = x[d];
6806       for (l = 0; l < Nl; ++l) PetscCall(DMLabelSetValue(marker, v + Nc, val[l]));
6807     }
6808   }
6809   PetscCall(VecRestoreArray(coordinates, &coords));
6810   PetscCall(DMSetCoordinatesLocal(*dm, coordinates));
6811   PetscCall(VecDestroy(&coordinates));
6812   PetscCall(PetscViewerDestroy(&viewer));
6813   if (interpolate) {
6814     DM      idm;
6815     DMLabel bdlabel;
6816 
6817     PetscCall(DMPlexInterpolate(*dm, &idm));
6818     PetscCall(DMDestroy(dm));
6819     *dm = idm;
6820 
6821     if (!Nl) {
6822       PetscCall(DMCreateLabel(*dm, "marker"));
6823       PetscCall(DMGetLabel(*dm, "marker", &bdlabel));
6824       PetscCall(DMPlexMarkBoundaryFaces(*dm, PETSC_DETERMINE, bdlabel));
6825       PetscCall(DMPlexLabelComplete(*dm, bdlabel));
6826     }
6827   }
6828   PetscFunctionReturn(PETSC_SUCCESS);
6829 }
6830 
6831 /*@
6832   DMPlexCreateFromFile - This takes a filename and produces a `DM`
6833 
6834   Collective
6835 
6836   Input Parameters:
6837 + comm        - The communicator
6838 . filename    - A file name
6839 . plexname    - The object name of the resulting `DM`, also used for intra-datafile lookup by some formats
6840 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
6841 
6842   Output Parameter:
6843 . dm - The `DM`
6844 
6845   Options Database Key:
6846 . -dm_plex_create_from_hdf5_xdmf - use the `PETSC_VIEWER_HDF5_XDMF` format for reading HDF5
6847 
6848   Use `-dm_plex_create_ prefix` to pass options to the internal `PetscViewer`, e.g.
6849 $ -dm_plex_create_viewer_hdf5_collective
6850 
6851   Level: beginner
6852 
6853   Notes:
6854   Using `PETSCVIEWERHDF5` type with `PETSC_VIEWER_HDF5_PETSC` format, one can save multiple `DMPLEX`
6855   meshes in a single HDF5 file. This in turn requires one to name the `DMPLEX` object with `PetscObjectSetName()`
6856   before saving it with `DMView()` and before loading it with `DMLoad()` for identification of the mesh object.
6857   The input parameter name is thus used to name the `DMPLEX` object when `DMPlexCreateFromFile()` internally
6858   calls `DMLoad()`. Currently, name is ignored for other viewer types and/or formats.
6859 
6860 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromDAG()`, `DMPlexCreateFromCellListPetsc()`, `DMPlexCreate()`, `PetscObjectSetName()`, `DMView()`, `DMLoad()`
6861 @*/
6862 PetscErrorCode DMPlexCreateFromFile(MPI_Comm comm, const char filename[], const char plexname[], PetscBool interpolate, DM *dm)
6863 {
6864   const char  extGmsh[]      = ".msh";
6865   const char  extGmsh2[]     = ".msh2";
6866   const char  extGmsh4[]     = ".msh4";
6867   const char  extCGNS[]      = ".cgns";
6868   const char  extExodus[]    = ".exo";
6869   const char  extExodus_e[]  = ".e";
6870   const char  extGenesis[]   = ".gen";
6871   const char  extFluent[]    = ".cas";
6872   const char  extHDF5[]      = ".h5";
6873   const char  extXDMFHDF5[]  = ".xdmf.h5";
6874   const char  extPLY[]       = ".ply";
6875   const char  extEGADSlite[] = ".egadslite";
6876   const char  extEGADS[]     = ".egads";
6877   const char  extIGES[]      = ".igs";
6878   const char  extIGES2[]     = ".iges";
6879   const char  extSTEP[]      = ".stp";
6880   const char  extSTEP2[]     = ".step";
6881   const char  extBREP[]      = ".brep";
6882   const char  extCV[]        = ".dat";
6883   size_t      len;
6884   PetscBool   isGmsh, isGmsh2, isGmsh4, isCGNS, isExodus, isGenesis, isFluent, isHDF5, isPLY, isEGADSlite, isEGADS, isIGES, isIGES2, isSTEP, isSTEP2, isBREP, isCV, isXDMFHDF5;
6885   PetscMPIInt rank;
6886 
6887   PetscFunctionBegin;
6888   PetscAssertPointer(filename, 2);
6889   if (plexname) PetscAssertPointer(plexname, 3);
6890   PetscAssertPointer(dm, 5);
6891   PetscCall(DMInitializePackage());
6892   PetscCall(PetscLogEventBegin(DMPLEX_CreateFromFile, 0, 0, 0, 0));
6893   PetscCallMPI(MPI_Comm_rank(comm, &rank));
6894   PetscCall(PetscStrlen(filename, &len));
6895   PetscCheck(len, comm, PETSC_ERR_ARG_WRONG, "Filename must be a valid path");
6896 
6897 #define CheckExtension(extension__, is_extension__) \
6898   do { \
6899     PetscAssert(sizeof(extension__), comm, PETSC_ERR_PLIB, "Zero-size extension: %s", extension__); \
6900     /* don't count the null-terminator at the end */ \
6901     const size_t ext_len = sizeof(extension__) - 1; \
6902     if (len < ext_len) { \
6903       is_extension__ = PETSC_FALSE; \
6904     } else { \
6905       PetscCall(PetscStrncmp(filename + len - ext_len, extension__, ext_len, &is_extension__)); \
6906     } \
6907   } while (0)
6908 
6909   CheckExtension(extGmsh, isGmsh);
6910   CheckExtension(extGmsh2, isGmsh2);
6911   CheckExtension(extGmsh4, isGmsh4);
6912   CheckExtension(extCGNS, isCGNS);
6913   CheckExtension(extExodus, isExodus);
6914   if (!isExodus) CheckExtension(extExodus_e, isExodus);
6915   CheckExtension(extGenesis, isGenesis);
6916   CheckExtension(extFluent, isFluent);
6917   CheckExtension(extHDF5, isHDF5);
6918   CheckExtension(extPLY, isPLY);
6919   CheckExtension(extEGADSlite, isEGADSlite);
6920   CheckExtension(extEGADS, isEGADS);
6921   CheckExtension(extIGES, isIGES);
6922   CheckExtension(extIGES2, isIGES2);
6923   CheckExtension(extSTEP, isSTEP);
6924   CheckExtension(extSTEP2, isSTEP2);
6925   CheckExtension(extBREP, isBREP);
6926   CheckExtension(extCV, isCV);
6927   CheckExtension(extXDMFHDF5, isXDMFHDF5);
6928 
6929 #undef CheckExtension
6930 
6931   if (isGmsh || isGmsh2 || isGmsh4) {
6932     PetscCall(DMPlexCreateGmshFromFile(comm, filename, interpolate, dm));
6933   } else if (isCGNS) {
6934     PetscCall(DMPlexCreateCGNSFromFile(comm, filename, interpolate, dm));
6935   } else if (isExodus || isGenesis) {
6936     PetscCall(DMPlexCreateExodusFromFile(comm, filename, interpolate, dm));
6937   } else if (isFluent) {
6938     PetscCall(DMPlexCreateFluentFromFile(comm, filename, interpolate, dm));
6939   } else if (isHDF5) {
6940     PetscViewer viewer;
6941 
6942     /* PETSC_VIEWER_HDF5_XDMF is used if the filename ends with .xdmf.h5, or if -dm_plex_create_from_hdf5_xdmf option is present */
6943     PetscCall(PetscOptionsGetBool(NULL, NULL, "-dm_plex_create_from_hdf5_xdmf", &isXDMFHDF5, NULL));
6944     PetscCall(PetscViewerCreate(comm, &viewer));
6945     PetscCall(PetscViewerSetType(viewer, PETSCVIEWERHDF5));
6946     PetscCall(PetscViewerSetOptionsPrefix(viewer, "dm_plex_create_"));
6947     PetscCall(PetscViewerSetFromOptions(viewer));
6948     PetscCall(PetscViewerFileSetMode(viewer, FILE_MODE_READ));
6949     PetscCall(PetscViewerFileSetName(viewer, filename));
6950 
6951     PetscCall(DMCreate(comm, dm));
6952     PetscCall(PetscObjectSetName((PetscObject)*dm, plexname));
6953     PetscCall(DMSetType(*dm, DMPLEX));
6954     if (isXDMFHDF5) PetscCall(PetscViewerPushFormat(viewer, PETSC_VIEWER_HDF5_XDMF));
6955     PetscCall(DMLoad(*dm, viewer));
6956     if (isXDMFHDF5) PetscCall(PetscViewerPopFormat(viewer));
6957     PetscCall(PetscViewerDestroy(&viewer));
6958 
6959     if (interpolate) {
6960       DM idm;
6961 
6962       PetscCall(DMPlexInterpolate(*dm, &idm));
6963       PetscCall(DMDestroy(dm));
6964       *dm = idm;
6965     }
6966   } else if (isPLY) {
6967     PetscCall(DMPlexCreatePLYFromFile(comm, filename, interpolate, dm));
6968   } else if (isEGADSlite || isEGADS || isIGES || isIGES2 || isSTEP || isSTEP2 || isBREP) {
6969     PetscCall(DMPlexCreateGeomFromFile(comm, filename, dm, isEGADSlite));
6970 
6971     if (!interpolate) {
6972       DM udm;
6973 
6974       PetscCall(DMPlexUninterpolate(*dm, &udm));
6975       PetscCall(DMDestroy(dm));
6976       *dm = udm;
6977     }
6978   } else if (isCV) {
6979     PetscCall(DMPlexCreateCellVertexFromFile(comm, filename, interpolate, dm));
6980   } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cannot load file %s: unrecognized extension", filename);
6981   PetscCall(PetscStrlen(plexname, &len));
6982   if (len) PetscCall(PetscObjectSetName((PetscObject)*dm, plexname));
6983   PetscCall(PetscLogEventEnd(DMPLEX_CreateFromFile, 0, 0, 0, 0));
6984   PetscFunctionReturn(PETSC_SUCCESS);
6985 }
6986 
6987 /*@
6988   DMPlexCreateEphemeral - This takes a `DMPlexTransform` and a base `DMPlex` and produces an ephemeral `DM`, meaning one that is created on the fly in response to queries.
6989 
6990   Input Parameters:
6991 + tr     - The `DMPlexTransform`
6992 - prefix - An options prefix, or NULL
6993 
6994   Output Parameter:
6995 . dm - The `DM`
6996 
6997   Level: beginner
6998 
6999   Notes:
7000   An emphemeral mesh is one that is not stored concretely, as in the default `DMPLEX` implementation, but rather is produced on the fly in response to queries, using information from the transform and the base mesh.
7001 
7002 .seealso: `DMPlexCreateFromFile`, `DMPlexCreateFromDAG()`, `DMPlexCreateFromCellListPetsc()`, `DMPlexCreate()`
7003 @*/
7004 PetscErrorCode DMPlexCreateEphemeral(DMPlexTransform tr, const char prefix[], DM *dm)
7005 {
7006   DM           bdm, bcdm, cdm;
7007   Vec          coordinates, coordinatesNew;
7008   PetscSection cs;
7009   PetscInt     cdim, Nl;
7010 
7011   PetscFunctionBegin;
7012   PetscCall(DMCreate(PetscObjectComm((PetscObject)tr), dm));
7013   PetscCall(DMSetType(*dm, DMPLEX));
7014   ((DM_Plex *)(*dm)->data)->interpolated = DMPLEX_INTERPOLATED_FULL;
7015   // Handle coordinates
7016   PetscCall(DMPlexTransformGetDM(tr, &bdm));
7017   PetscCall(DMPlexTransformSetDimensions(tr, bdm, *dm));
7018   PetscCall(DMGetCoordinateDim(*dm, &cdim));
7019   PetscCall(DMGetCoordinateDM(bdm, &bcdm));
7020   PetscCall(DMGetCoordinateDM(*dm, &cdm));
7021   PetscCall(DMCopyDisc(bcdm, cdm));
7022   PetscCall(DMGetLocalSection(cdm, &cs));
7023   PetscCall(PetscSectionSetNumFields(cs, 1));
7024   PetscCall(PetscSectionSetFieldComponents(cs, 0, cdim));
7025   PetscCall(DMGetCoordinatesLocal(bdm, &coordinates));
7026   PetscCall(VecDuplicate(coordinates, &coordinatesNew));
7027   PetscCall(VecCopy(coordinates, coordinatesNew));
7028   PetscCall(DMSetCoordinatesLocal(*dm, coordinatesNew));
7029   PetscCall(VecDestroy(&coordinatesNew));
7030 
7031   PetscCall(PetscObjectReference((PetscObject)tr));
7032   PetscCall(DMPlexTransformDestroy(&((DM_Plex *)(*dm)->data)->tr));
7033   ((DM_Plex *)(*dm)->data)->tr = tr;
7034   PetscCall(DMPlexDistributeSetDefault(*dm, PETSC_FALSE));
7035   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)*dm, prefix));
7036   PetscCall(DMSetFromOptions(*dm));
7037 
7038   PetscCall(DMGetNumLabels(bdm, &Nl));
7039   for (PetscInt l = 0; l < Nl; ++l) {
7040     DMLabel     label, labelNew;
7041     const char *lname;
7042     PetscBool   isDepth, isCellType;
7043 
7044     PetscCall(DMGetLabelName(bdm, l, &lname));
7045     PetscCall(PetscStrcmp(lname, "depth", &isDepth));
7046     if (isDepth) continue;
7047     PetscCall(PetscStrcmp(lname, "celltype", &isCellType));
7048     if (isCellType) continue;
7049     PetscCall(DMCreateLabel(*dm, lname));
7050     PetscCall(DMGetLabel(bdm, lname, &label));
7051     PetscCall(DMGetLabel(*dm, lname, &labelNew));
7052     PetscCall(DMLabelSetType(labelNew, DMLABELEPHEMERAL));
7053     PetscCall(DMLabelEphemeralSetLabel(labelNew, label));
7054     PetscCall(DMLabelEphemeralSetTransform(labelNew, tr));
7055     PetscCall(DMLabelSetUp(labelNew));
7056   }
7057   PetscFunctionReturn(PETSC_SUCCESS);
7058 }
7059