xref: /petsc/src/dm/impls/plex/plexcreate.c (revision f8b9f887efa5534f423807e37a1fe2e87d779bac)
1 #define PETSCDM_DLL
2 #include <petsc/private/dmpleximpl.h> /*I   "petscdmplex.h"   I*/
3 #include <petsc/private/hashseti.h>
4 #include <petscsf.h>
5 #include <petscdmplextransform.h> /*I   "petscdmplextransform.h"   I*/
6 #include <petscdmlabelephemeral.h>
7 #include <petsc/private/kernels/blockmatmult.h>
8 #include <petsc/private/kernels/blockinvert.h>
9 
10 #ifdef PETSC_HAVE_UNISTD_H
11   #include <unistd.h>
12 #endif
13 #include <errno.h>
14 
15 PetscLogEvent DMPLEX_CreateFromFile, DMPLEX_CreateFromOptions, DMPLEX_BuildFromCellList, DMPLEX_BuildCoordinatesFromCellList;
16 
17 /* External function declarations here */
18 static PetscErrorCode DMInitialize_Plex(DM dm);
19 
20 PETSC_EXTERN PetscErrorCode DMPlexCheckEGADS_Private(DM dm)
21 {
22   PetscObject modelObj;
23 
24   PetscFunctionBegin;
25   PetscCall(PetscObjectQuery((PetscObject)dm, "EGADS Model", &modelObj));
26   PetscCheck(modelObj, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Input DM must have attached EGADS Geometry Model");
27   PetscFunctionReturn(PETSC_SUCCESS);
28 }
29 
30 static PetscErrorCode DMPlexCopyContext_Private(DM dmin, const char name[], DM dmout)
31 {
32   PetscObject obj;
33 
34   PetscFunctionBegin;
35   PetscCall(PetscObjectQuery((PetscObject)dmin, name, &obj));
36   if (obj) PetscCall(PetscObjectCompose((PetscObject)dmout, name, obj));
37   PetscFunctionReturn(PETSC_SUCCESS);
38 }
39 
40 static PetscErrorCode DMPlexSwapContext_Private(DM dmA, const char name[], DM dmB)
41 {
42   PetscObject objA, objB;
43 
44   PetscFunctionBegin;
45   PetscCall(PetscObjectQuery((PetscObject)dmA, name, &objA));
46   PetscCall(PetscObjectQuery((PetscObject)dmB, name, &objB));
47   PetscCall(PetscObjectReference(objA));
48   PetscCall(PetscObjectReference(objB));
49   PetscCall(PetscObjectCompose((PetscObject)dmA, name, objB));
50   PetscCall(PetscObjectCompose((PetscObject)dmB, name, objA));
51   PetscCall(PetscObjectDereference(objA));
52   PetscCall(PetscObjectDereference(objB));
53   PetscFunctionReturn(PETSC_SUCCESS);
54 }
55 
56 PetscErrorCode DMPlexCopyEGADSInfo_Internal(DM dmin, DM dmout)
57 {
58   PetscFunctionBegin;
59   PetscCall(DMPlexCopyContext_Private(dmin, "EGADS Model", dmout));
60   PetscCall(DMPlexCopyContext_Private(dmin, "EGADS Context", dmout));
61   PetscCall(DMPlexCopyContext_Private(dmin, "EGADSlite Model", dmout));
62   PetscCall(DMPlexCopyContext_Private(dmin, "EGADSlite Context", dmout));
63   PetscFunctionReturn(PETSC_SUCCESS);
64 }
65 
66 static PetscErrorCode DMPlexSwapEGADSInfo_Private(DM dmA, DM dmB)
67 {
68   PetscFunctionBegin;
69   PetscCall(DMPlexSwapContext_Private(dmA, "EGADS Model", dmB));
70   PetscCall(DMPlexSwapContext_Private(dmA, "EGADS Context", dmB));
71   PetscCall(DMPlexSwapContext_Private(dmA, "EGADSlite Model", dmB));
72   PetscCall(DMPlexSwapContext_Private(dmA, "EGADSlite Context", dmB));
73   PetscFunctionReturn(PETSC_SUCCESS);
74 }
75 
76 /* This copies internal things in the Plex structure that we generally want when making a new, related Plex */
77 PetscErrorCode DMPlexCopy_Internal(DM dmin, PetscBool copyPeriodicity, PetscBool copyOverlap, DM dmout)
78 {
79   const PetscReal     *maxCell, *Lstart, *L;
80   VecType              vecType;
81   MatType              matType;
82   PetscBool            dist, useCeed, balance_partition;
83   DMReorderDefaultFlag reorder;
84 
85   PetscFunctionBegin;
86   if (dmin == dmout) PetscFunctionReturn(PETSC_SUCCESS);
87   PetscCall(DMGetVecType(dmin, &vecType));
88   PetscCall(DMSetVecType(dmout, vecType));
89   PetscCall(DMGetMatType(dmin, &matType));
90   PetscCall(DMSetMatType(dmout, matType));
91   if (copyPeriodicity) {
92     PetscCall(DMGetPeriodicity(dmin, &maxCell, &Lstart, &L));
93     PetscCall(DMSetPeriodicity(dmout, maxCell, Lstart, L));
94     PetscCall(DMLocalizeCoordinates(dmout));
95   }
96   PetscCall(DMPlexDistributeGetDefault(dmin, &dist));
97   PetscCall(DMPlexDistributeSetDefault(dmout, dist));
98   PetscCall(DMPlexReorderGetDefault(dmin, &reorder));
99   PetscCall(DMPlexReorderSetDefault(dmout, reorder));
100   PetscCall(DMPlexGetUseCeed(dmin, &useCeed));
101   PetscCall(DMPlexSetUseCeed(dmout, useCeed));
102   PetscCall(DMPlexGetPartitionBalance(dmin, &balance_partition));
103   PetscCall(DMPlexSetPartitionBalance(dmout, balance_partition));
104   ((DM_Plex *)dmout->data)->useHashLocation = ((DM_Plex *)dmin->data)->useHashLocation;
105   ((DM_Plex *)dmout->data)->printSetValues  = ((DM_Plex *)dmin->data)->printSetValues;
106   ((DM_Plex *)dmout->data)->printFEM        = ((DM_Plex *)dmin->data)->printFEM;
107   ((DM_Plex *)dmout->data)->printFVM        = ((DM_Plex *)dmin->data)->printFVM;
108   ((DM_Plex *)dmout->data)->printL2         = ((DM_Plex *)dmin->data)->printL2;
109   ((DM_Plex *)dmout->data)->printLocate     = ((DM_Plex *)dmin->data)->printLocate;
110   ((DM_Plex *)dmout->data)->printProject    = ((DM_Plex *)dmin->data)->printProject;
111   ((DM_Plex *)dmout->data)->printTol        = ((DM_Plex *)dmin->data)->printTol;
112   if (copyOverlap) PetscCall(DMPlexSetOverlap_Plex(dmout, dmin, 0));
113   PetscCall(DMPlexCopyEGADSInfo_Internal(dmin, dmout));
114   PetscFunctionReturn(PETSC_SUCCESS);
115 }
116 
117 /* Replace dm with the contents of ndm, and then destroy ndm
118    - Share the DM_Plex structure
119    - Share the coordinates
120    - Share the SF
121 */
122 PetscErrorCode DMPlexReplace_Internal(DM dm, DM *ndm)
123 {
124   PetscSF          sf;
125   DM               dmNew = *ndm, coordDM, coarseDM;
126   Vec              coords;
127   const PetscReal *maxCell, *Lstart, *L;
128   PetscInt         dim, cdim;
129   PetscBool        use_natural;
130 
131   PetscFunctionBegin;
132   if (dm == dmNew) {
133     PetscCall(DMDestroy(ndm));
134     PetscFunctionReturn(PETSC_SUCCESS);
135   }
136   dm->setupcalled = dmNew->setupcalled;
137   if (!dm->hdr.name) {
138     const char *name;
139 
140     PetscCall(PetscObjectGetName((PetscObject)*ndm, &name));
141     PetscCall(PetscObjectSetName((PetscObject)dm, name));
142   }
143   PetscCall(DMGetDimension(dmNew, &dim));
144   PetscCall(DMSetDimension(dm, dim));
145   PetscCall(DMGetCoordinateDim(dmNew, &cdim));
146   PetscCall(DMSetCoordinateDim(dm, cdim));
147   PetscCall(DMGetPointSF(dmNew, &sf));
148   PetscCall(DMSetPointSF(dm, sf));
149   PetscCall(DMGetCoordinateDM(dmNew, &coordDM));
150   PetscCall(DMGetCoordinatesLocal(dmNew, &coords));
151   PetscCall(DMSetCoordinateDM(dm, coordDM));
152   PetscCall(DMSetCoordinatesLocal(dm, coords));
153   PetscCall(DMGetCellCoordinateDM(dmNew, &coordDM));
154   PetscCall(DMGetCellCoordinatesLocal(dmNew, &coords));
155   PetscCall(DMSetCellCoordinateDM(dm, coordDM));
156   PetscCall(DMSetCellCoordinatesLocal(dm, coords));
157   /* Do not want to create the coordinate field if it does not already exist, so do not call DMGetCoordinateField() */
158   PetscCall(DMFieldDestroy(&dm->coordinates[0].field));
159   dm->coordinates[0].field            = dmNew->coordinates[0].field;
160   ((DM_Plex *)dmNew->data)->coordFunc = ((DM_Plex *)dm->data)->coordFunc;
161   PetscCall(DMGetPeriodicity(dmNew, &maxCell, &Lstart, &L));
162   PetscCall(DMSetPeriodicity(dm, maxCell, Lstart, L));
163   PetscCall(DMGetNaturalSF(dmNew, &sf));
164   PetscCall(DMSetNaturalSF(dm, sf));
165   PetscCall(DMGetUseNatural(dmNew, &use_natural));
166   PetscCall(DMSetUseNatural(dm, use_natural));
167   PetscCall(DMDestroy_Plex(dm));
168   PetscCall(DMInitialize_Plex(dm));
169   dm->data = dmNew->data;
170   ((DM_Plex *)dmNew->data)->refct++;
171   {
172     PetscInt       num_face_sfs;
173     const PetscSF *sfs;
174     PetscCall(DMPlexGetIsoperiodicFaceSF(dm, &num_face_sfs, &sfs));
175     PetscCall(DMPlexSetIsoperiodicFaceSF(dm, num_face_sfs, (PetscSF *)sfs)); // for the compose function effect on dm
176   }
177   PetscCall(DMDestroyLabelLinkList_Internal(dm));
178   PetscCall(DMCopyLabels(dmNew, dm, PETSC_OWN_POINTER, PETSC_TRUE, DM_COPY_LABELS_FAIL));
179   PetscCall(DMGetCoarseDM(dmNew, &coarseDM));
180   PetscCall(DMSetCoarseDM(dm, coarseDM));
181   PetscCall(DMPlexCopyEGADSInfo_Internal(dmNew, dm));
182   PetscCall(DMDestroy(ndm));
183   PetscFunctionReturn(PETSC_SUCCESS);
184 }
185 
186 /* Swap dm with the contents of dmNew
187    - Swap the DM_Plex structure
188    - Swap the coordinates
189    - Swap the point PetscSF
190 */
191 static PetscErrorCode DMPlexSwap_Static(DM dmA, DM dmB)
192 {
193   DM          coordDMA, coordDMB;
194   Vec         coordsA, coordsB;
195   PetscSF     sfA, sfB;
196   DMField     fieldTmp;
197   void       *tmp;
198   DMLabelLink listTmp;
199   DMLabel     depthTmp;
200   PetscInt    tmpI;
201 
202   PetscFunctionBegin;
203   if (dmA == dmB) PetscFunctionReturn(PETSC_SUCCESS);
204   PetscCall(DMGetPointSF(dmA, &sfA));
205   PetscCall(DMGetPointSF(dmB, &sfB));
206   PetscCall(PetscObjectReference((PetscObject)sfA));
207   PetscCall(DMSetPointSF(dmA, sfB));
208   PetscCall(DMSetPointSF(dmB, sfA));
209   PetscCall(PetscObjectDereference((PetscObject)sfA));
210 
211   PetscCall(DMGetCoordinateDM(dmA, &coordDMA));
212   PetscCall(DMGetCoordinateDM(dmB, &coordDMB));
213   PetscCall(PetscObjectReference((PetscObject)coordDMA));
214   PetscCall(DMSetCoordinateDM(dmA, coordDMB));
215   PetscCall(DMSetCoordinateDM(dmB, coordDMA));
216   PetscCall(PetscObjectDereference((PetscObject)coordDMA));
217 
218   PetscCall(DMGetCoordinatesLocal(dmA, &coordsA));
219   PetscCall(DMGetCoordinatesLocal(dmB, &coordsB));
220   PetscCall(PetscObjectReference((PetscObject)coordsA));
221   PetscCall(DMSetCoordinatesLocal(dmA, coordsB));
222   PetscCall(DMSetCoordinatesLocal(dmB, coordsA));
223   PetscCall(PetscObjectDereference((PetscObject)coordsA));
224 
225   PetscCall(DMGetCellCoordinateDM(dmA, &coordDMA));
226   PetscCall(DMGetCellCoordinateDM(dmB, &coordDMB));
227   PetscCall(PetscObjectReference((PetscObject)coordDMA));
228   PetscCall(DMSetCellCoordinateDM(dmA, coordDMB));
229   PetscCall(DMSetCellCoordinateDM(dmB, coordDMA));
230   PetscCall(PetscObjectDereference((PetscObject)coordDMA));
231 
232   PetscCall(DMGetCellCoordinatesLocal(dmA, &coordsA));
233   PetscCall(DMGetCellCoordinatesLocal(dmB, &coordsB));
234   PetscCall(PetscObjectReference((PetscObject)coordsA));
235   PetscCall(DMSetCellCoordinatesLocal(dmA, coordsB));
236   PetscCall(DMSetCellCoordinatesLocal(dmB, coordsA));
237   PetscCall(PetscObjectDereference((PetscObject)coordsA));
238 
239   PetscCall(DMPlexSwapEGADSInfo_Private(dmA, dmB));
240 
241   fieldTmp                  = dmA->coordinates[0].field;
242   dmA->coordinates[0].field = dmB->coordinates[0].field;
243   dmB->coordinates[0].field = fieldTmp;
244   fieldTmp                  = dmA->coordinates[1].field;
245   dmA->coordinates[1].field = dmB->coordinates[1].field;
246   dmB->coordinates[1].field = fieldTmp;
247   tmp                       = dmA->data;
248   dmA->data                 = dmB->data;
249   dmB->data                 = tmp;
250   listTmp                   = dmA->labels;
251   dmA->labels               = dmB->labels;
252   dmB->labels               = listTmp;
253   depthTmp                  = dmA->depthLabel;
254   dmA->depthLabel           = dmB->depthLabel;
255   dmB->depthLabel           = depthTmp;
256   depthTmp                  = dmA->celltypeLabel;
257   dmA->celltypeLabel        = dmB->celltypeLabel;
258   dmB->celltypeLabel        = depthTmp;
259   tmpI                      = dmA->levelup;
260   dmA->levelup              = dmB->levelup;
261   dmB->levelup              = tmpI;
262   PetscFunctionReturn(PETSC_SUCCESS);
263 }
264 
265 PetscErrorCode DMPlexInterpolateInPlace_Internal(DM dm)
266 {
267   DM idm;
268 
269   PetscFunctionBegin;
270   PetscCall(DMPlexInterpolate(dm, &idm));
271   PetscCall(DMPlexCopyCoordinates(dm, idm));
272   PetscCall(DMPlexReplace_Internal(dm, &idm));
273   PetscFunctionReturn(PETSC_SUCCESS);
274 }
275 
276 /*@C
277   DMPlexCreateCoordinateSpace - Creates a finite element space for the coordinates
278 
279   Collective
280 
281   Input Parameters:
282 + dm        - The `DMPLEX`
283 . degree    - The degree of the finite element or `PETSC_DECIDE`
284 . project   - Flag to project current coordinates into the space
285 - coordFunc - An optional function to map new points from refinement to the surface
286 
287   Level: advanced
288 
289 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `PetscPointFunc`, `PetscFECreateLagrange()`, `DMGetCoordinateDM()`
290 @*/
291 PetscErrorCode DMPlexCreateCoordinateSpace(DM dm, PetscInt degree, PetscBool project, PetscPointFunc coordFunc)
292 {
293   DM_Plex *mesh = (DM_Plex *)dm->data;
294   PetscFE  fe   = NULL;
295   DM       cdm;
296   PetscInt dim, cdim, dE, qorder, height;
297 
298   PetscFunctionBegin;
299   PetscCall(DMGetDimension(dm, &dim));
300   cdim = dim;
301   PetscCall(DMGetCoordinateDim(dm, &dE));
302   qorder = degree;
303   PetscCall(DMGetCoordinateDM(dm, &cdm));
304   PetscObjectOptionsBegin((PetscObject)cdm);
305   PetscCall(PetscOptionsBoundedInt("-default_quadrature_order", "Quadrature order is one less than quadrature points per edge", "DMPlexCreateCoordinateSpace", qorder, &qorder, NULL, 0));
306   PetscCall(PetscOptionsBoundedInt("-dm_plex_coordinate_dim", "Set the coordinate dimension", "DMPlexCreateCoordinateSpace", cdim, &cdim, NULL, dim));
307   PetscOptionsEnd();
308   PetscCall(DMPlexGetVTKCellHeight(dm, &height));
309   if (cdim > dim) {
310     DM           cdm;
311     PetscSection cs, csNew;
312     Vec          coordinates, coordinatesNew;
313     VecType      vectype;
314     IS           idx;
315     PetscInt    *indices;
316     PetscInt     bs, n;
317 
318     // Recreate coordinate section
319     {
320       const char *fieldName = NULL, *compName = NULL;
321       PetscInt    Nc, pStart, pEnd;
322 
323       PetscCall(DMGetCoordinateDM(dm, &cdm));
324       PetscCall(DMGetLocalSection(cdm, &cs));
325       PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)cs), &csNew));
326       PetscCall(PetscSectionSetNumFields(csNew, 1));
327       PetscCall(PetscSectionGetFieldName(cs, 0, &fieldName));
328       PetscCall(PetscSectionSetFieldName(csNew, 0, fieldName));
329       PetscCall(PetscSectionGetFieldComponents(cs, 0, &Nc));
330       PetscCall(PetscSectionSetFieldComponents(csNew, 0, cdim));
331       for (PetscInt c = 0; c < Nc; ++c) {
332         PetscCall(PetscSectionGetComponentName(cs, 0, c, &compName));
333         PetscCall(PetscSectionSetComponentName(csNew, 0, c, compName));
334       }
335       PetscCall(PetscSectionGetChart(cs, &pStart, &pEnd));
336       PetscCall(PetscSectionSetChart(csNew, pStart, pEnd));
337       for (PetscInt p = pStart; p < pEnd; ++p) {
338         PetscInt dof;
339 
340         PetscCall(PetscSectionGetDof(cs, p, &dof));
341         if (dof) {
342           PetscCall(PetscSectionSetDof(csNew, p, cdim));
343           PetscCall(PetscSectionSetFieldDof(csNew, p, 0, cdim));
344         }
345       }
346       PetscCall(PetscSectionSetUp(csNew));
347     }
348     PetscCall(DMSetLocalSection(cdm, csNew));
349     PetscCall(PetscSectionDestroy(&csNew));
350     // Reset coordinate dimension for coordinate DM
351     PetscCall(DMSetCoordinateDim(cdm, cdim));
352     PetscCall(DMSetCoordinateField(cdm, NULL));
353     // Inject coordinates into higher dimension
354     PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
355     PetscCall(VecGetBlockSize(coordinates, &bs));
356     PetscCheck(bs == dim, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "We can only inject simple coordinates into a higher dimension");
357     PetscCall(VecCreate(PetscObjectComm((PetscObject)coordinates), &coordinatesNew));
358     PetscCall(VecGetType(coordinates, &vectype));
359     PetscCall(VecSetType(coordinatesNew, vectype));
360     PetscCall(VecGetLocalSize(coordinates, &n));
361     PetscCheck(!(n % bs), PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "We can only inject simple coordinates into a higher dimension");
362     n /= bs;
363     PetscCall(VecSetSizes(coordinatesNew, n * cdim, PETSC_DETERMINE));
364     PetscCall(VecSetUp(coordinatesNew));
365     PetscCall(PetscMalloc1(n * bs, &indices));
366     for (PetscInt i = 0; i < n; ++i)
367       for (PetscInt b = 0; b < bs; ++b) indices[i * bs + b] = i * cdim + b;
368     PetscCall(ISCreateGeneral(PETSC_COMM_SELF, n * bs, indices, PETSC_OWN_POINTER, &idx));
369     PetscCall(VecISCopy(coordinatesNew, idx, SCATTER_FORWARD, coordinates));
370     PetscCall(ISDestroy(&idx));
371     PetscCall(DMSetCoordinatesLocal(dm, coordinatesNew));
372     PetscCall(VecDestroy(&coordinatesNew));
373     PetscCall(DMSetCoordinateDim(dm, cdim));
374     {
375       PetscInt gn;
376 
377       PetscCall(DMGetCoordinates(dm, &coordinatesNew));
378       PetscCall(VecGetLocalSize(coordinatesNew, &gn));
379       PetscCheck(gn == n * cdim, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Global coordinate size %" PetscInt_FMT " != %" PetscInt_FMT "local coordinate size", gn, n * cdim);
380     }
381     dE      = cdim;
382     project = PETSC_FALSE;
383   }
384   if (degree >= 0) {
385     DMPolytopeType ct = DM_POLYTOPE_UNKNOWN;
386     PetscInt       cStart, cEnd, gct;
387 
388     PetscCall(DMPlexGetHeightStratum(dm, height, &cStart, &cEnd));
389     if (cEnd > cStart) PetscCall(DMPlexGetCellType(dm, cStart, &ct));
390     gct = (PetscInt)ct;
391     PetscCallMPI(MPIU_Allreduce(MPI_IN_PLACE, &gct, 1, MPIU_INT, MPI_MIN, PetscObjectComm((PetscObject)dm)));
392     ct = (DMPolytopeType)gct;
393     // Work around current bug in PetscDualSpaceSetUp_Lagrange()
394     //   Can be seen in plex_tutorials-ex10_1
395     if (ct != DM_POLYTOPE_SEG_PRISM_TENSOR && ct != DM_POLYTOPE_TRI_PRISM_TENSOR && ct != DM_POLYTOPE_QUAD_PRISM_TENSOR) PetscCall(PetscFECreateLagrangeByCell(PETSC_COMM_SELF, dim, dE, ct, degree, qorder, &fe));
396   }
397   PetscCall(DMSetCoordinateDisc(dm, fe, project));
398   PetscCall(PetscFEDestroy(&fe));
399   mesh->coordFunc = coordFunc;
400   PetscFunctionReturn(PETSC_SUCCESS);
401 }
402 
403 /*@
404   DMPlexCreateDoublet - Creates a mesh of two cells of the specified type, optionally with later refinement.
405 
406   Collective
407 
408   Input Parameters:
409 + comm            - The communicator for the `DM` object
410 . dim             - The spatial dimension
411 . simplex         - Flag for simplicial cells, otherwise they are tensor product cells
412 . interpolate     - Flag to create intermediate mesh pieces (edges, faces)
413 - refinementLimit - A nonzero number indicates the largest admissible volume for a refined cell
414 
415   Output Parameter:
416 . newdm - The `DM` object
417 
418   Level: beginner
419 
420 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMSetType()`, `DMCreate()`
421 @*/
422 PetscErrorCode DMPlexCreateDoublet(MPI_Comm comm, PetscInt dim, PetscBool simplex, PetscBool interpolate, PetscReal refinementLimit, DM *newdm)
423 {
424   DM          dm;
425   PetscMPIInt rank;
426 
427   PetscFunctionBegin;
428   PetscCall(DMCreate(comm, &dm));
429   PetscCall(DMSetType(dm, DMPLEX));
430   PetscCall(DMSetDimension(dm, dim));
431   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
432   PetscCallMPI(MPI_Comm_rank(comm, &rank));
433   switch (dim) {
434   case 2:
435     if (simplex) PetscCall(PetscObjectSetName((PetscObject)dm, "triangular"));
436     else PetscCall(PetscObjectSetName((PetscObject)dm, "quadrilateral"));
437     break;
438   case 3:
439     if (simplex) PetscCall(PetscObjectSetName((PetscObject)dm, "tetrahedral"));
440     else PetscCall(PetscObjectSetName((PetscObject)dm, "hexahedral"));
441     break;
442   default:
443     SETERRQ(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %" PetscInt_FMT, dim);
444   }
445   if (rank) {
446     PetscInt numPoints[2] = {0, 0};
447     PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, NULL, NULL, NULL, NULL));
448   } else {
449     switch (dim) {
450     case 2:
451       if (simplex) {
452         PetscInt    numPoints[2]        = {4, 2};
453         PetscInt    coneSize[6]         = {3, 3, 0, 0, 0, 0};
454         PetscInt    cones[6]            = {2, 3, 4, 5, 4, 3};
455         PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
456         PetscScalar vertexCoords[8]     = {-0.5, 0.5, 0.0, 0.0, 0.0, 1.0, 0.5, 0.5};
457 
458         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
459       } else {
460         PetscInt    numPoints[2]        = {6, 2};
461         PetscInt    coneSize[8]         = {4, 4, 0, 0, 0, 0, 0, 0};
462         PetscInt    cones[8]            = {2, 3, 4, 5, 3, 6, 7, 4};
463         PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
464         PetscScalar vertexCoords[12]    = {-1.0, -0.5, 0.0, -0.5, 0.0, 0.5, -1.0, 0.5, 1.0, -0.5, 1.0, 0.5};
465 
466         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
467       }
468       break;
469     case 3:
470       if (simplex) {
471         PetscInt    numPoints[2]        = {5, 2};
472         PetscInt    coneSize[7]         = {4, 4, 0, 0, 0, 0, 0};
473         PetscInt    cones[8]            = {4, 3, 5, 2, 5, 3, 4, 6};
474         PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
475         PetscScalar vertexCoords[15]    = {-1.0, 0.0, 0.0, 0.0, -1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 1.0, 0.0, 1.0, 0.0, 0.0};
476 
477         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
478       } else {
479         PetscInt    numPoints[2]         = {12, 2};
480         PetscInt    coneSize[14]         = {8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
481         PetscInt    cones[16]            = {2, 3, 4, 5, 6, 7, 8, 9, 5, 4, 10, 11, 7, 12, 13, 8};
482         PetscInt    coneOrientations[16] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
483         PetscScalar vertexCoords[36]     = {-1.0, -0.5, -0.5, -1.0, 0.5, -0.5, 0.0, 0.5, -0.5, 0.0, -0.5, -0.5, -1.0, -0.5, 0.5, 0.0, -0.5, 0.5, 0.0, 0.5, 0.5, -1.0, 0.5, 0.5, 1.0, 0.5, -0.5, 1.0, -0.5, -0.5, 1.0, -0.5, 0.5, 1.0, 0.5, 0.5};
484 
485         PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
486       }
487       break;
488     default:
489       SETERRQ(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cannot make meshes for dimension %" PetscInt_FMT, dim);
490     }
491   }
492   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
493   *newdm = dm;
494   if (refinementLimit > 0.0) {
495     DM          rdm;
496     const char *name;
497 
498     PetscCall(DMPlexSetRefinementUniform(*newdm, PETSC_FALSE));
499     PetscCall(DMPlexSetRefinementLimit(*newdm, refinementLimit));
500     PetscCall(DMRefine(*newdm, comm, &rdm));
501     PetscCall(PetscObjectGetName((PetscObject)*newdm, &name));
502     PetscCall(PetscObjectSetName((PetscObject)rdm, name));
503     PetscCall(DMDestroy(newdm));
504     *newdm = rdm;
505   }
506   if (interpolate) {
507     DM idm;
508 
509     PetscCall(DMPlexInterpolate(*newdm, &idm));
510     PetscCall(DMDestroy(newdm));
511     *newdm = idm;
512   }
513   PetscFunctionReturn(PETSC_SUCCESS);
514 }
515 
516 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_1D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[])
517 {
518   const PetscInt numVertices    = 2;
519   PetscInt       markerRight    = 1;
520   PetscInt       markerLeft     = 1;
521   PetscBool      markerSeparate = PETSC_FALSE;
522   Vec            coordinates;
523   PetscSection   coordSection;
524   PetscScalar   *coords;
525   PetscInt       coordSize;
526   PetscMPIInt    rank;
527   PetscInt       cdim = 1, v;
528 
529   PetscFunctionBegin;
530   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
531   if (markerSeparate) {
532     markerRight = 2;
533     markerLeft  = 1;
534   }
535   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
536   if (rank == 0) {
537     PetscCall(DMPlexSetChart(dm, 0, numVertices));
538     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
539     PetscCall(DMSetLabelValue(dm, "marker", 0, markerLeft));
540     PetscCall(DMSetLabelValue(dm, "marker", 1, markerRight));
541   }
542   PetscCall(DMPlexSymmetrize(dm));
543   PetscCall(DMPlexStratify(dm));
544   /* Build coordinates */
545   PetscCall(DMSetCoordinateDim(dm, cdim));
546   PetscCall(DMGetCoordinateSection(dm, &coordSection));
547   PetscCall(PetscSectionSetNumFields(coordSection, 1));
548   PetscCall(PetscSectionSetChart(coordSection, 0, numVertices));
549   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, cdim));
550   for (v = 0; v < numVertices; ++v) {
551     PetscCall(PetscSectionSetDof(coordSection, v, cdim));
552     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, cdim));
553   }
554   PetscCall(PetscSectionSetUp(coordSection));
555   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
556   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
557   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
558   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
559   PetscCall(VecSetBlockSize(coordinates, cdim));
560   PetscCall(VecSetType(coordinates, VECSTANDARD));
561   PetscCall(VecGetArray(coordinates, &coords));
562   coords[0] = lower[0];
563   coords[1] = upper[0];
564   PetscCall(VecRestoreArray(coordinates, &coords));
565   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
566   PetscCall(VecDestroy(&coordinates));
567   PetscFunctionReturn(PETSC_SUCCESS);
568 }
569 
570 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_2D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[])
571 {
572   const PetscInt numVertices    = (edges[0] + 1) * (edges[1] + 1);
573   const PetscInt numEdges       = edges[0] * (edges[1] + 1) + (edges[0] + 1) * edges[1];
574   PetscInt       markerTop      = 1;
575   PetscInt       markerBottom   = 1;
576   PetscInt       markerRight    = 1;
577   PetscInt       markerLeft     = 1;
578   PetscBool      markerSeparate = PETSC_FALSE;
579   Vec            coordinates;
580   PetscSection   coordSection;
581   PetscScalar   *coords;
582   PetscInt       coordSize;
583   PetscMPIInt    rank;
584   PetscInt       v, vx, vy;
585 
586   PetscFunctionBegin;
587   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
588   if (markerSeparate) {
589     markerTop    = 3;
590     markerBottom = 1;
591     markerRight  = 2;
592     markerLeft   = 4;
593   }
594   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
595   if (rank == 0) {
596     PetscInt e, ex, ey;
597 
598     PetscCall(DMPlexSetChart(dm, 0, numEdges + numVertices));
599     for (e = 0; e < numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
600     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
601     for (vx = 0; vx <= edges[0]; vx++) {
602       for (ey = 0; ey < edges[1]; ey++) {
603         PetscInt edge   = vx * edges[1] + ey + edges[0] * (edges[1] + 1);
604         PetscInt vertex = ey * (edges[0] + 1) + vx + numEdges;
605         PetscInt cone[2];
606 
607         cone[0] = vertex;
608         cone[1] = vertex + edges[0] + 1;
609         PetscCall(DMPlexSetCone(dm, edge, cone));
610         if (vx == edges[0]) {
611           PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
612           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
613           if (ey == edges[1] - 1) {
614             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
615             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerRight));
616           }
617         } else if (vx == 0) {
618           PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
619           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
620           if (ey == edges[1] - 1) {
621             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
622             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerLeft));
623           }
624         }
625       }
626     }
627     for (vy = 0; vy <= edges[1]; vy++) {
628       for (ex = 0; ex < edges[0]; ex++) {
629         PetscInt edge   = vy * edges[0] + ex;
630         PetscInt vertex = vy * (edges[0] + 1) + ex + numEdges;
631         PetscInt cone[2];
632 
633         cone[0] = vertex;
634         cone[1] = vertex + 1;
635         PetscCall(DMPlexSetCone(dm, edge, cone));
636         if (vy == edges[1]) {
637           PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
638           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
639           if (ex == edges[0] - 1) {
640             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
641             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerTop));
642           }
643         } else if (vy == 0) {
644           PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
645           PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
646           if (ex == edges[0] - 1) {
647             PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
648             PetscCall(DMSetLabelValue(dm, "Face Sets", cone[1], markerBottom));
649           }
650         }
651       }
652     }
653   }
654   PetscCall(DMPlexSymmetrize(dm));
655   PetscCall(DMPlexStratify(dm));
656   /* Build coordinates */
657   PetscCall(DMSetCoordinateDim(dm, 2));
658   PetscCall(DMGetCoordinateSection(dm, &coordSection));
659   PetscCall(PetscSectionSetNumFields(coordSection, 1));
660   PetscCall(PetscSectionSetChart(coordSection, numEdges, numEdges + numVertices));
661   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, 2));
662   for (v = numEdges; v < numEdges + numVertices; ++v) {
663     PetscCall(PetscSectionSetDof(coordSection, v, 2));
664     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, 2));
665   }
666   PetscCall(PetscSectionSetUp(coordSection));
667   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
668   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
669   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
670   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
671   PetscCall(VecSetBlockSize(coordinates, 2));
672   PetscCall(VecSetType(coordinates, VECSTANDARD));
673   PetscCall(VecGetArray(coordinates, &coords));
674   for (vy = 0; vy <= edges[1]; ++vy) {
675     for (vx = 0; vx <= edges[0]; ++vx) {
676       coords[(vy * (edges[0] + 1) + vx) * 2 + 0] = lower[0] + ((upper[0] - lower[0]) / edges[0]) * vx;
677       coords[(vy * (edges[0] + 1) + vx) * 2 + 1] = lower[1] + ((upper[1] - lower[1]) / edges[1]) * vy;
678     }
679   }
680   PetscCall(VecRestoreArray(coordinates, &coords));
681   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
682   PetscCall(VecDestroy(&coordinates));
683   PetscFunctionReturn(PETSC_SUCCESS);
684 }
685 
686 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Tensor_3D_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt faces[])
687 {
688   PetscInt     vertices[3], numVertices;
689   PetscInt     numFaces       = 2 * faces[0] * faces[1] + 2 * faces[1] * faces[2] + 2 * faces[0] * faces[2];
690   PetscInt     markerTop      = 1;
691   PetscInt     markerBottom   = 1;
692   PetscInt     markerFront    = 1;
693   PetscInt     markerBack     = 1;
694   PetscInt     markerRight    = 1;
695   PetscInt     markerLeft     = 1;
696   PetscBool    markerSeparate = PETSC_FALSE;
697   Vec          coordinates;
698   PetscSection coordSection;
699   PetscScalar *coords;
700   PetscInt     coordSize;
701   PetscMPIInt  rank;
702   PetscInt     v, vx, vy, vz;
703   PetscInt     voffset, iface = 0, cone[4];
704 
705   PetscFunctionBegin;
706   PetscCheck(faces[0] >= 1 && faces[1] >= 1 && faces[2] >= 1, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Must have at least 1 face per side");
707   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
708   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
709   if (markerSeparate) {
710     markerBottom = 1;
711     markerTop    = 2;
712     markerFront  = 3;
713     markerBack   = 4;
714     markerRight  = 5;
715     markerLeft   = 6;
716   }
717   vertices[0] = faces[0] + 1;
718   vertices[1] = faces[1] + 1;
719   vertices[2] = faces[2] + 1;
720   numVertices = vertices[0] * vertices[1] * vertices[2];
721   if (rank == 0) {
722     PetscInt f;
723 
724     PetscCall(DMPlexSetChart(dm, 0, numFaces + numVertices));
725     for (f = 0; f < numFaces; ++f) PetscCall(DMPlexSetConeSize(dm, f, 4));
726     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
727 
728     /* Side 0 (Top) */
729     for (vy = 0; vy < faces[1]; vy++) {
730       for (vx = 0; vx < faces[0]; vx++) {
731         voffset = numFaces + vertices[0] * vertices[1] * (vertices[2] - 1) + vy * vertices[0] + vx;
732         cone[0] = voffset;
733         cone[1] = voffset + 1;
734         cone[2] = voffset + vertices[0] + 1;
735         cone[3] = voffset + vertices[0];
736         PetscCall(DMPlexSetCone(dm, iface, cone));
737         PetscCall(DMSetLabelValue(dm, "marker", iface, markerTop));
738         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerTop));
739         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerTop));
740         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerTop));
741         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 1, markerTop));
742         iface++;
743       }
744     }
745 
746     /* Side 1 (Bottom) */
747     for (vy = 0; vy < faces[1]; vy++) {
748       for (vx = 0; vx < faces[0]; vx++) {
749         voffset = numFaces + vy * (faces[0] + 1) + vx;
750         cone[0] = voffset + 1;
751         cone[1] = voffset;
752         cone[2] = voffset + vertices[0];
753         cone[3] = voffset + vertices[0] + 1;
754         PetscCall(DMPlexSetCone(dm, iface, cone));
755         PetscCall(DMSetLabelValue(dm, "marker", iface, markerBottom));
756         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerBottom));
757         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerBottom));
758         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerBottom));
759         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 1, markerBottom));
760         iface++;
761       }
762     }
763 
764     /* Side 2 (Front) */
765     for (vz = 0; vz < faces[2]; vz++) {
766       for (vx = 0; vx < faces[0]; vx++) {
767         voffset = numFaces + vz * vertices[0] * vertices[1] + vx;
768         cone[0] = voffset;
769         cone[1] = voffset + 1;
770         cone[2] = voffset + vertices[0] * vertices[1] + 1;
771         cone[3] = voffset + vertices[0] * vertices[1];
772         PetscCall(DMPlexSetCone(dm, iface, cone));
773         PetscCall(DMSetLabelValue(dm, "marker", iface, markerFront));
774         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerFront));
775         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerFront));
776         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 0, markerFront));
777         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 1, markerFront));
778         iface++;
779       }
780     }
781 
782     /* Side 3 (Back) */
783     for (vz = 0; vz < faces[2]; vz++) {
784       for (vx = 0; vx < faces[0]; vx++) {
785         voffset = numFaces + vz * vertices[0] * vertices[1] + vertices[0] * (vertices[1] - 1) + vx;
786         cone[0] = voffset + vertices[0] * vertices[1];
787         cone[1] = voffset + vertices[0] * vertices[1] + 1;
788         cone[2] = voffset + 1;
789         cone[3] = voffset;
790         PetscCall(DMPlexSetCone(dm, iface, cone));
791         PetscCall(DMSetLabelValue(dm, "marker", iface, markerBack));
792         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerBack));
793         PetscCall(DMSetLabelValue(dm, "marker", voffset + 1, markerBack));
794         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 0, markerBack));
795         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 1, markerBack));
796         iface++;
797       }
798     }
799 
800     /* Side 4 (Left) */
801     for (vz = 0; vz < faces[2]; vz++) {
802       for (vy = 0; vy < faces[1]; vy++) {
803         voffset = numFaces + vz * vertices[0] * vertices[1] + vy * vertices[0];
804         cone[0] = voffset;
805         cone[1] = voffset + vertices[0] * vertices[1];
806         cone[2] = voffset + vertices[0] * vertices[1] + vertices[0];
807         cone[3] = voffset + vertices[0];
808         PetscCall(DMPlexSetCone(dm, iface, cone));
809         PetscCall(DMSetLabelValue(dm, "marker", iface, markerLeft));
810         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerLeft));
811         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerLeft));
812         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[1] + 0, markerLeft));
813         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + vertices[0], markerLeft));
814         iface++;
815       }
816     }
817 
818     /* Side 5 (Right) */
819     for (vz = 0; vz < faces[2]; vz++) {
820       for (vy = 0; vy < faces[1]; vy++) {
821         voffset = numFaces + vz * vertices[0] * vertices[1] + vy * vertices[0] + faces[0];
822         cone[0] = voffset + vertices[0] * vertices[1];
823         cone[1] = voffset;
824         cone[2] = voffset + vertices[0];
825         cone[3] = voffset + vertices[0] * vertices[1] + vertices[0];
826         PetscCall(DMPlexSetCone(dm, iface, cone));
827         PetscCall(DMSetLabelValue(dm, "marker", iface, markerRight));
828         PetscCall(DMSetLabelValue(dm, "marker", voffset + 0, markerRight));
829         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] + 0, markerRight));
830         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + 0, markerRight));
831         PetscCall(DMSetLabelValue(dm, "marker", voffset + vertices[0] * vertices[1] + vertices[0], markerRight));
832         iface++;
833       }
834     }
835   }
836   PetscCall(DMPlexSymmetrize(dm));
837   PetscCall(DMPlexStratify(dm));
838   /* Build coordinates */
839   PetscCall(DMSetCoordinateDim(dm, 3));
840   PetscCall(DMGetCoordinateSection(dm, &coordSection));
841   PetscCall(PetscSectionSetNumFields(coordSection, 1));
842   PetscCall(PetscSectionSetChart(coordSection, numFaces, numFaces + numVertices));
843   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, 3));
844   for (v = numFaces; v < numFaces + numVertices; ++v) {
845     PetscCall(PetscSectionSetDof(coordSection, v, 3));
846     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, 3));
847   }
848   PetscCall(PetscSectionSetUp(coordSection));
849   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
850   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
851   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
852   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
853   PetscCall(VecSetBlockSize(coordinates, 3));
854   PetscCall(VecSetType(coordinates, VECSTANDARD));
855   PetscCall(VecGetArray(coordinates, &coords));
856   for (vz = 0; vz <= faces[2]; ++vz) {
857     for (vy = 0; vy <= faces[1]; ++vy) {
858       for (vx = 0; vx <= faces[0]; ++vx) {
859         coords[((vz * (faces[1] + 1) + vy) * (faces[0] + 1) + vx) * 3 + 0] = lower[0] + ((upper[0] - lower[0]) / faces[0]) * vx;
860         coords[((vz * (faces[1] + 1) + vy) * (faces[0] + 1) + vx) * 3 + 1] = lower[1] + ((upper[1] - lower[1]) / faces[1]) * vy;
861         coords[((vz * (faces[1] + 1) + vy) * (faces[0] + 1) + vx) * 3 + 2] = lower[2] + ((upper[2] - lower[2]) / faces[2]) * vz;
862       }
863     }
864   }
865   PetscCall(VecRestoreArray(coordinates, &coords));
866   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
867   PetscCall(VecDestroy(&coordinates));
868   PetscFunctionReturn(PETSC_SUCCESS);
869 }
870 
871 static PetscErrorCode DMPlexCreateBoxSurfaceMesh_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], PetscBool interpolate)
872 {
873   PetscFunctionBegin;
874   PetscValidLogicalCollectiveInt(dm, dim, 2);
875   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
876   PetscCall(DMSetDimension(dm, dim - 1));
877   PetscCall(DMSetCoordinateDim(dm, dim));
878   switch (dim) {
879   case 1:
880     PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_1D_Internal(dm, lower, upper, faces));
881     break;
882   case 2:
883     PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_2D_Internal(dm, lower, upper, faces));
884     break;
885   case 3:
886     PetscCall(DMPlexCreateBoxSurfaceMesh_Tensor_3D_Internal(dm, lower, upper, faces));
887     break;
888   default:
889     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Dimension not supported: %" PetscInt_FMT, dim);
890   }
891   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
892   if (interpolate) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
893   PetscFunctionReturn(PETSC_SUCCESS);
894 }
895 
896 /*@C
897   DMPlexCreateBoxSurfaceMesh - Creates a mesh on the surface of the tensor product of unit intervals (box) using tensor cells (hexahedra).
898 
899   Collective
900 
901   Input Parameters:
902 + comm        - The communicator for the `DM` object
903 . dim         - The spatial dimension of the box, so the resulting mesh is has dimension `dim`-1
904 . faces       - Number of faces per dimension, or `NULL` for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
905 . lower       - The lower left corner, or `NULL` for (0, 0, 0)
906 . upper       - The upper right corner, or `NULL` for (1, 1, 1)
907 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
908 
909   Output Parameter:
910 . dm - The `DM` object
911 
912   Level: beginner
913 
914 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMSetFromOptions()`, `DMPlexCreateBoxMesh()`, `DMPlexCreateFromFile()`, `DMSetType()`, `DMCreate()`
915 @*/
916 PetscErrorCode DMPlexCreateBoxSurfaceMesh(MPI_Comm comm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], PetscBool interpolate, DM *dm)
917 {
918   PetscInt  fac[3] = {1, 1, 1};
919   PetscReal low[3] = {0, 0, 0};
920   PetscReal upp[3] = {1, 1, 1};
921 
922   PetscFunctionBegin;
923   PetscCall(DMCreate(comm, dm));
924   PetscCall(DMSetType(*dm, DMPLEX));
925   PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(*dm, dim, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, interpolate));
926   PetscFunctionReturn(PETSC_SUCCESS);
927 }
928 
929 static PetscErrorCode DMPlexCreateLineMesh_Internal(DM dm, PetscInt segments, PetscReal lower, PetscReal upper, DMBoundaryType bd)
930 {
931   PetscInt     i, fStart, fEnd, numCells = 0, numVerts = 0;
932   PetscInt     numPoints[2], *coneSize, *cones, *coneOrientations;
933   PetscScalar *vertexCoords;
934   PetscReal    L, maxCell;
935   PetscBool    markerSeparate = PETSC_FALSE;
936   PetscInt     markerLeft = 1, faceMarkerLeft = 1;
937   PetscInt     markerRight = 1, faceMarkerRight = 2;
938   PetscBool    wrap = (bd == DM_BOUNDARY_PERIODIC || bd == DM_BOUNDARY_TWIST) ? PETSC_TRUE : PETSC_FALSE;
939   PetscMPIInt  rank;
940 
941   PetscFunctionBegin;
942   PetscAssertPointer(dm, 1);
943 
944   PetscCall(DMSetDimension(dm, 1));
945   PetscCall(DMCreateLabel(dm, "marker"));
946   PetscCall(DMCreateLabel(dm, "Face Sets"));
947 
948   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
949   if (rank == 0) numCells = segments;
950   if (rank == 0) numVerts = segments + (wrap ? 0 : 1);
951 
952   numPoints[0] = numVerts;
953   numPoints[1] = numCells;
954   PetscCall(PetscMalloc4(numCells + numVerts, &coneSize, numCells * 2, &cones, numCells + numVerts, &coneOrientations, numVerts, &vertexCoords));
955   PetscCall(PetscArrayzero(coneOrientations, numCells + numVerts));
956   for (i = 0; i < numCells; ++i) coneSize[i] = 2;
957   for (i = 0; i < numVerts; ++i) coneSize[numCells + i] = 0;
958   for (i = 0; i < numCells; ++i) {
959     cones[2 * i]     = numCells + i % numVerts;
960     cones[2 * i + 1] = numCells + (i + 1) % numVerts;
961   }
962   for (i = 0; i < numVerts; ++i) vertexCoords[i] = lower + (upper - lower) * ((PetscReal)i / (PetscReal)numCells);
963   PetscCall(DMPlexCreateFromDAG(dm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
964   PetscCall(PetscFree4(coneSize, cones, coneOrientations, vertexCoords));
965 
966   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
967   if (markerSeparate) {
968     markerLeft  = faceMarkerLeft;
969     markerRight = faceMarkerRight;
970   }
971   if (!wrap && rank == 0) {
972     PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
973     PetscCall(DMSetLabelValue(dm, "marker", fStart, markerLeft));
974     PetscCall(DMSetLabelValue(dm, "marker", fEnd - 1, markerRight));
975     PetscCall(DMSetLabelValue(dm, "Face Sets", fStart, faceMarkerLeft));
976     PetscCall(DMSetLabelValue(dm, "Face Sets", fEnd - 1, faceMarkerRight));
977   }
978   if (wrap) {
979     L       = upper - lower;
980     maxCell = (PetscReal)1.1 * (L / (PetscReal)PetscMax(1, segments));
981     PetscCall(DMSetPeriodicity(dm, &maxCell, &lower, &L));
982   }
983   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
984   PetscFunctionReturn(PETSC_SUCCESS);
985 }
986 
987 // Creates "Face Sets" label based on the standard box labeling conventions
988 static PetscErrorCode DMPlexSetBoxLabel_Internal(DM dm, const DMBoundaryType periodicity[])
989 {
990   DM              cdm;
991   PetscSection    csection;
992   Vec             coordinates;
993   DMLabel         label;
994   IS              faces_is;
995   PetscInt        dim, num_face = 0;
996   const PetscInt *faces;
997   PetscInt        faceMarkerBottom, faceMarkerTop, faceMarkerFront, faceMarkerBack, faceMarkerRight, faceMarkerLeft;
998 
999   PetscFunctionBeginUser;
1000   PetscCall(DMGetDimension(dm, &dim));
1001   PetscCheck((dim == 2) || (dim == 3), PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "DMPlex box labeling only supports 2D and 3D meshes, received DM of dimension %" PetscInt_FMT, dim);
1002   // Get Face Sets label
1003   PetscCall(DMGetLabel(dm, "Face Sets", &label));
1004   if (label) {
1005     PetscCall(DMLabelReset(label));
1006   } else {
1007     PetscCall(DMCreateLabel(dm, "Face Sets"));
1008     PetscCall(DMGetLabel(dm, "Face Sets", &label));
1009   }
1010   PetscCall(DMPlexMarkBoundaryFaces(dm, 1, label));
1011   PetscCall(DMGetStratumIS(dm, "Face Sets", 1, &faces_is));
1012 
1013   switch (dim) {
1014   case 2:
1015     faceMarkerTop    = 3;
1016     faceMarkerBottom = 1;
1017     faceMarkerRight  = 2;
1018     faceMarkerLeft   = 4;
1019     break;
1020   case 3:
1021     faceMarkerBottom = 1;
1022     faceMarkerTop    = 2;
1023     faceMarkerFront  = 3;
1024     faceMarkerBack   = 4;
1025     faceMarkerRight  = 5;
1026     faceMarkerLeft   = 6;
1027     break;
1028   default:
1029     SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Dimension %" PetscInt_FMT " not supported", dim);
1030   }
1031 
1032   if (faces_is) PetscCall(ISGetLocalSize(faces_is, &num_face));
1033   if (faces_is) PetscCall(ISGetIndices(faces_is, &faces));
1034   PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
1035   PetscCall(DMGetCoordinateDM(dm, &cdm));
1036   PetscCall(DMGetLocalSection(cdm, &csection));
1037   for (PetscInt f = 0; f < num_face; ++f) {
1038     PetscScalar *coords = NULL;
1039     PetscInt     face = faces[f], flip = 1, label_value = -1, coords_size;
1040 
1041     { // Determine if orientation of face is flipped
1042       PetscInt        num_cells_support, num_faces, start = -1;
1043       const PetscInt *orients, *cell_faces, *cells;
1044 
1045       PetscCall(DMPlexGetSupport(dm, face, &cells));
1046       PetscCall(DMPlexGetSupportSize(dm, face, &num_cells_support));
1047       PetscCheck(num_cells_support == 1, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "Expected one cell in support of exterior face, but got %" PetscInt_FMT " cells", num_cells_support);
1048       PetscCall(DMPlexGetCone(dm, cells[0], &cell_faces));
1049       PetscCall(DMPlexGetConeSize(dm, cells[0], &num_faces));
1050       for (PetscInt i = 0; i < num_faces; i++) {
1051         if (cell_faces[i] == face) start = i;
1052       }
1053       PetscCheck(start >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_CORRUPT, "Could not find face %" PetscInt_FMT " in cone of its support", face);
1054       PetscCall(DMPlexGetConeOrientation(dm, cells[0], &orients));
1055       if (orients[start] < 0) flip = -1;
1056     }
1057 
1058     // Cannot use DMPlexComputeCellGeometryFVM() for high-order geometry, so must calculate normal vectors manually
1059     // Use the vertices (depth 0) of coordinate DM to calculate normal vector
1060     PetscCall(DMPlexVecGetClosureAtDepth_Internal(cdm, csection, coordinates, face, 0, &coords_size, &coords));
1061     switch (dim) {
1062     case 2: {
1063       PetscScalar vec[2];
1064 
1065       for (PetscInt d = 0; d < dim; ++d) vec[d] = flip * (PetscRealPart(coords[1 * dim + d]) - PetscRealPart(coords[0 * dim + d]));
1066       PetscScalar normal[] = {vec[1], -vec[0]};
1067       if (PetscAbsScalar(normal[0]) > PetscAbsScalar(normal[1])) {
1068         label_value = PetscRealPart(normal[0]) > 0 ? faceMarkerRight : faceMarkerLeft;
1069       } else {
1070         label_value = PetscRealPart(normal[1]) > 0 ? faceMarkerTop : faceMarkerBottom;
1071       }
1072     } break;
1073     case 3: {
1074       PetscScalar vec1[3], vec2[3], normal[3];
1075 
1076       for (PetscInt d = 0; d < dim; ++d) {
1077         vec1[d] = PetscRealPart(coords[1 * dim + d]) - PetscRealPart(coords[0 * dim + d]);
1078         vec2[d] = PetscRealPart(coords[2 * dim + d]) - PetscRealPart(coords[1 * dim + d]);
1079       }
1080 
1081       // Calculate normal vector via cross-product
1082       normal[0] = flip * ((vec1[1] * vec2[2]) - (vec1[2] * vec2[1]));
1083       normal[1] = flip * ((vec1[2] * vec2[0]) - (vec1[0] * vec2[2]));
1084       normal[2] = flip * ((vec1[0] * vec2[1]) - (vec1[1] * vec2[0]));
1085 
1086       if (PetscAbsScalar(normal[0]) > PetscAbsScalar(normal[1])) {
1087         if (PetscAbsScalar(normal[0]) > PetscAbsScalar(normal[2])) {
1088           label_value = PetscRealPart(normal[0]) > 0 ? faceMarkerRight : faceMarkerLeft;
1089         } else {
1090           label_value = PetscRealPart(normal[2]) > 0 ? faceMarkerTop : faceMarkerBottom;
1091         }
1092       } else {
1093         if (PetscAbsScalar(normal[1]) > PetscAbsScalar(normal[2])) {
1094           label_value = PetscRealPart(normal[1]) > 0 ? faceMarkerBack : faceMarkerFront;
1095         } else {
1096           label_value = PetscRealPart(normal[2]) > 0 ? faceMarkerTop : faceMarkerBottom;
1097         }
1098       }
1099     } break;
1100     }
1101 
1102     PetscInt previous_label_value; // always 1 due to DMPlexMarkBoundaryFaces call above
1103     PetscCall(DMGetLabelValue(dm, "Face Sets", face, &previous_label_value));
1104     PetscCall(DMClearLabelValue(dm, "Face Sets", face, previous_label_value));
1105     PetscCall(DMSetLabelValue(dm, "Face Sets", face, label_value));
1106     PetscCall(DMPlexVecRestoreClosure(cdm, csection, coordinates, face, &coords_size, &coords));
1107   }
1108   if (faces_is) PetscCall(ISRestoreIndices(faces_is, &faces));
1109   PetscCall(ISDestroy(&faces_is));
1110 
1111   // Create Isoperiodic SF from newly-created face labels
1112   PetscSF     periodicsfs[3];
1113   PetscInt    periodic_sf_index  = 0;
1114   PetscScalar transform[3][4][4] = {{{0.}}};
1115   for (PetscInt d = 0; d < dim; d++) {
1116     IS              donor_is, periodic_is;
1117     const PetscInt *donor_faces = NULL, *periodic_faces = NULL;
1118     PetscInt        num_donor = 0, num_periodic = 0;
1119     PetscSF         centroidsf;
1120     PetscReal       donor_to_periodic_distance;
1121     const PetscInt  face_pairings[2][3][2] = {
1122       // 2D face pairings, {donor, periodic}
1123       {{4, 2}, {1, 3}},
1124       // 3D face pairings
1125       {{5, 6}, {3, 4}, {1, 2}}
1126     };
1127 
1128     if (periodicity[d] != DM_BOUNDARY_PERIODIC) continue;
1129     {
1130       // Compute centroidsf, which is the mapping from donor faces to periodic faces
1131       // Matches the centroid of the faces together, ignoring the periodic direction component (which should not match between donor and periodic face)
1132       PetscInt     coords_size, centroid_comps = dim - 1;
1133       PetscScalar *coords = NULL;
1134       PetscReal   *donor_centroids, *periodic_centroids;
1135       PetscReal    loc_periodic[2] = {PETSC_MIN_REAL, PETSC_MIN_REAL}, loc_periodic_global[2]; // Location of donor (0) and periodic (1) faces in periodic direction
1136 
1137       PetscCall(DMGetStratumIS(dm, "Face Sets", face_pairings[dim - 2][d][0], &donor_is));
1138       PetscCall(DMGetStratumIS(dm, "Face Sets", face_pairings[dim - 2][d][1], &periodic_is));
1139       if (donor_is) {
1140         PetscCall(ISGetLocalSize(donor_is, &num_donor));
1141         PetscCall(ISGetIndices(donor_is, &donor_faces));
1142       }
1143       if (periodic_is) {
1144         PetscCall(ISGetLocalSize(periodic_is, &num_periodic));
1145         PetscCall(ISGetIndices(periodic_is, &periodic_faces));
1146       }
1147       PetscCall(PetscCalloc2(num_donor * centroid_comps, &donor_centroids, num_periodic * centroid_comps, &periodic_centroids));
1148       for (PetscInt f = 0; f < num_donor; f++) {
1149         PetscInt face = donor_faces[f], num_coords;
1150         PetscCall(DMPlexVecGetClosureAtDepth_Internal(cdm, csection, coordinates, face, 0, &coords_size, &coords));
1151         num_coords = coords_size / dim;
1152         for (PetscInt c = 0; c < num_coords; c++) {
1153           PetscInt comp_index = 0;
1154           loc_periodic[0]     = PetscRealPart(coords[c * dim + d]);
1155           for (PetscInt i = 0; i < dim; i++) {
1156             if (i == d) continue; // Periodic direction not used for centroid calculation
1157             donor_centroids[f * centroid_comps + comp_index] += PetscRealPart(coords[c * dim + i]) / num_coords;
1158             comp_index++;
1159           }
1160         }
1161         PetscCall(DMPlexVecRestoreClosure(cdm, csection, coordinates, face, &coords_size, &coords));
1162       }
1163 
1164       for (PetscInt f = 0; f < num_periodic; f++) {
1165         PetscInt face = periodic_faces[f], num_coords;
1166         PetscCall(DMPlexVecGetClosureAtDepth_Internal(cdm, csection, coordinates, face, 0, &coords_size, &coords));
1167         num_coords = coords_size / dim;
1168         for (PetscInt c = 0; c < num_coords; c++) {
1169           PetscInt comp_index = 0;
1170           loc_periodic[1]     = PetscRealPart(coords[c * dim + d]);
1171           for (PetscInt i = 0; i < dim; i++) {
1172             if (i == d) continue; // Periodic direction not used for centroid calculation
1173             periodic_centroids[f * centroid_comps + comp_index] += PetscRealPart(coords[c * dim + i]) / num_coords;
1174             comp_index++;
1175           }
1176         }
1177         PetscCall(DMPlexVecRestoreClosure(cdm, csection, coordinates, face, &coords_size, &coords));
1178       }
1179       PetscCallMPI(MPIU_Allreduce(loc_periodic, loc_periodic_global, 2, MPIU_REAL, MPIU_MAX, PetscObjectComm((PetscObject)dm)));
1180       donor_to_periodic_distance = loc_periodic_global[1] - loc_periodic_global[0];
1181 
1182       PetscCall(PetscSFCreate(PetscObjectComm((PetscObject)dm), &centroidsf));
1183       PetscCall(PetscSFSetGraphFromCoordinates(centroidsf, num_donor, num_periodic, centroid_comps, 1e-10, donor_centroids, periodic_centroids));
1184       PetscCall(PetscSFViewFromOptions(centroidsf, NULL, "-dm_plex_box_label_centroid_sf_view"));
1185       PetscCall(PetscFree2(donor_centroids, periodic_centroids));
1186     }
1187 
1188     { // Create Isoperiodic SF using centroidsSF
1189       PetscInt           pStart, pEnd;
1190       PetscInt          *leaf_faces;
1191       const PetscSFNode *firemote;
1192       PetscSFNode       *isoperiodic_leaves;
1193 
1194       PetscCall(PetscMalloc1(num_periodic, &leaf_faces));
1195       PetscCall(PetscSFBcastBegin(centroidsf, MPIU_INT, donor_faces, leaf_faces, MPI_REPLACE));
1196       PetscCall(PetscSFBcastEnd(centroidsf, MPIU_INT, donor_faces, leaf_faces, MPI_REPLACE));
1197 
1198       PetscCall(PetscMalloc1(num_periodic, &isoperiodic_leaves));
1199       PetscCall(PetscSFGetGraph(centroidsf, NULL, NULL, NULL, &firemote));
1200       for (PetscInt l = 0; l < num_periodic; ++l) {
1201         isoperiodic_leaves[l].index = leaf_faces[l];
1202         isoperiodic_leaves[l].rank  = firemote[l].rank;
1203       }
1204 
1205       PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
1206       PetscCall(PetscSFCreate(PetscObjectComm((PetscObject)dm), &periodicsfs[periodic_sf_index]));
1207       PetscCall(PetscSFSetGraph(periodicsfs[periodic_sf_index], pEnd - pStart, num_periodic, (PetscInt *)periodic_faces, PETSC_COPY_VALUES, isoperiodic_leaves, PETSC_OWN_POINTER));
1208       PetscCall(PetscSFViewFromOptions(periodicsfs[periodic_sf_index], NULL, "-dm_plex_box_label_periodic_sf_view"));
1209       PetscCall(PetscFree(leaf_faces));
1210     }
1211 
1212     transform[periodic_sf_index][0][0] = 1;
1213     transform[periodic_sf_index][1][1] = 1;
1214     transform[periodic_sf_index][2][2] = 1;
1215     transform[periodic_sf_index][3][3] = 1;
1216     transform[periodic_sf_index][d][3] = donor_to_periodic_distance;
1217 
1218     periodic_sf_index++;
1219     PetscCall(PetscSFDestroy(&centroidsf));
1220     if (donor_is) {
1221       PetscCall(ISRestoreIndices(donor_is, &donor_faces));
1222       PetscCall(ISDestroy(&donor_is));
1223     }
1224     if (periodic_is) {
1225       PetscCall(ISRestoreIndices(periodic_is, &periodic_faces));
1226       PetscCall(ISDestroy(&periodic_is));
1227     }
1228     PetscCall(DMClearLabelStratum(dm, "Face Sets", face_pairings[dim - 2][d][0]));
1229     PetscCall(DMClearLabelStratum(dm, "Face Sets", face_pairings[dim - 2][d][1]));
1230   }
1231   PetscCall(DMPlexSetIsoperiodicFaceSF(dm, periodic_sf_index, periodicsfs));
1232   PetscCall(DMPlexSetIsoperiodicFaceTransform(dm, periodic_sf_index, (const PetscScalar *)transform));
1233   for (PetscInt p = 0; p < periodic_sf_index; p++) PetscCall(PetscSFDestroy(&periodicsfs[p]));
1234 
1235   { // Update coordinate DM with new Face Sets label
1236     DM      cdm;
1237     DMLabel oldFaceSets, newFaceSets;
1238     PetscCall(DMGetCoordinateDM(dm, &cdm));
1239     PetscCall(DMGetLabel(cdm, "Face Sets", &oldFaceSets));
1240     if (oldFaceSets) PetscCall(DMRemoveLabelBySelf(cdm, &oldFaceSets, PETSC_FALSE));
1241     PetscCall(DMLabelDuplicate(label, &newFaceSets));
1242     PetscCall(DMAddLabel(cdm, newFaceSets));
1243     PetscCall(DMLabelDestroy(&newFaceSets));
1244   }
1245   PetscFunctionReturn(PETSC_SUCCESS);
1246 }
1247 
1248 static PetscErrorCode DMPlexCreateSquareMesh_Simplex_CrissCross(DM dm, const PetscInt edges[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType bd[])
1249 {
1250   PetscInt       markerTop = 1, faceMarkerTop = 3;
1251   PetscInt       markerBottom = 1, faceMarkerBottom = 1;
1252   PetscInt       markerRight = 1, faceMarkerRight = 2;
1253   PetscInt       markerLeft = 1, faceMarkerLeft = 4;
1254   PetscBool      markerSeparate = PETSC_FALSE;
1255   DMBoundaryType bdX = bd[0], bdY = bd[1];
1256   PetscMPIInt    rank;
1257 
1258   PetscFunctionBegin;
1259   PetscCheck(bdX == DM_BOUNDARY_NONE || bdX == DM_BOUNDARY_PERIODIC, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Not implemented for boundary type %s", DMBoundaryTypes[bdX]);
1260   PetscCheck(bdY == DM_BOUNDARY_NONE || bdY == DM_BOUNDARY_PERIODIC, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Not implemented for boundary type %s", DMBoundaryTypes[bdY]);
1261   PetscCall(DMSetDimension(dm, 2));
1262   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
1263   PetscCall(DMCreateLabel(dm, "marker"));
1264   PetscCall(DMCreateLabel(dm, "Face Sets"));
1265   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
1266   if (markerSeparate) {
1267     markerBottom = faceMarkerBottom;
1268     markerTop    = faceMarkerTop;
1269     markerRight  = faceMarkerRight;
1270     markerLeft   = faceMarkerLeft;
1271   }
1272   {
1273     const PetscInt numXEdges    = rank == 0 ? edges[0] : 0;
1274     const PetscInt numYEdges    = rank == 0 ? edges[1] : 0;
1275     const PetscInt numZEdges    = rank == 0 ? 4 * edges[0] * edges[1] : 0; /* Z-edges are the 4 internal edges per cell */
1276     const PetscInt numXVertices = rank == 0 ? (bdX == DM_BOUNDARY_PERIODIC ? edges[0] : edges[0] + 1) : 0;
1277     const PetscInt numYVertices = rank == 0 ? (bdY == DM_BOUNDARY_PERIODIC ? edges[1] : edges[1] + 1) : 0;
1278     const PetscInt numZVertices = rank == 0 ? edges[0] * edges[1] : 0;
1279     const PetscInt numCells     = 4 * numXEdges * numYEdges;
1280     const PetscInt numTotXEdges = numXEdges * numYVertices;
1281     const PetscInt numTotYEdges = numYEdges * numXVertices;
1282     const PetscInt numVertices  = numXVertices * numYVertices + numZVertices;
1283     const PetscInt numEdges     = numTotXEdges + numTotYEdges + numZEdges;
1284     const PetscInt firstVertex  = numCells;
1285     const PetscInt firstXEdge   = numCells + numVertices;
1286     const PetscInt firstYEdge   = firstXEdge + numTotXEdges;
1287     const PetscInt firstZEdge   = firstYEdge + numTotYEdges;
1288     Vec            coordinates;
1289     PetscSection   coordSection;
1290     PetscScalar   *coords;
1291     PetscInt       coordSize;
1292     PetscInt       v, vx, vy;
1293     PetscInt       c, e, ex, ey;
1294 
1295     PetscCall(DMPlexSetChart(dm, 0, numCells + numEdges + numVertices));
1296     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 3));
1297     for (e = firstXEdge; e < firstXEdge + numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
1298     PetscCall(DMSetUp(dm));
1299 
1300     /* Build cells and Z-edges */
1301     for (ey = 0; ey < numYEdges; ++ey) {
1302       for (ex = 0; ex < numXEdges; ++ex) {
1303         const PetscInt exp1 = (ex + 1) % numXVertices;
1304         const PetscInt eyp1 = (ey + 1) % numYVertices;
1305         const PetscInt ez   = firstZEdge + 4 * (ey * numXEdges + ex);
1306         const PetscInt vc   = firstVertex + numXVertices * numYVertices + ey * numXEdges + ex;
1307         const PetscInt v0   = firstVertex + ey * numXVertices + ex;
1308         const PetscInt v1   = firstVertex + ey * numXVertices + exp1;
1309         const PetscInt v2   = firstVertex + eyp1 * numXVertices + exp1;
1310         const PetscInt v3   = firstVertex + eyp1 * numXVertices + ex;
1311         const PetscInt e0   = firstXEdge + ey * numXEdges + ex;
1312         const PetscInt e1   = firstYEdge + exp1 * numYEdges + ey;
1313         const PetscInt e2   = firstXEdge + eyp1 * numXEdges + ex;
1314         const PetscInt e3   = firstYEdge + ex * numYEdges + ey;
1315 
1316         const PetscInt cones[] = {ez, e0, ez + 1, ez + 1, e1, ez + 2, ez + 2, e2, ez + 3, ez + 3, e3, ez};
1317         const PetscInt ornts[] = {-1, 0, 0, -1, 0, 0, -1, -1, 0, -1, -1, 0};
1318         const PetscInt verts[] = {v0, vc, v1, vc, v2, vc, v3, vc};
1319 
1320         for (c = 0; c < 4; c++) {
1321           PetscInt cell = 4 * (ey * numXEdges + ex) + c;
1322           PetscInt edge = ez + c;
1323 
1324           PetscCall(DMPlexSetCone(dm, cell, cones + 3 * c));
1325           PetscCall(DMPlexSetConeOrientation(dm, cell, ornts + 3 * c));
1326           PetscCall(DMPlexSetCone(dm, edge, verts + 2 * c));
1327         }
1328       }
1329     }
1330 
1331     /* Build Y edges*/
1332     for (vx = 0; vx < numXVertices; vx++) {
1333       for (ey = 0; ey < numYEdges; ey++) {
1334         const PetscInt edge   = firstYEdge + vx * numYEdges + ey;
1335         const PetscInt v0     = firstVertex + ey * numXVertices + vx;
1336         const PetscInt v1     = firstVertex + ((ey + 1) % numYVertices) * numXVertices + vx;
1337         const PetscInt cone[] = {v0, v1};
1338 
1339         PetscCall(DMPlexSetCone(dm, edge, cone));
1340         if ((bdX != DM_BOUNDARY_PERIODIC) && (bdX != DM_BOUNDARY_TWIST)) {
1341           if (vx == numXVertices - 1) {
1342             PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerRight));
1343             PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1344             PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1345             if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1346           } else if (vx == 0) {
1347             PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerLeft));
1348             PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1349             PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1350             if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1351           }
1352         }
1353       }
1354     }
1355 
1356     /* Build X edges*/
1357     for (vy = 0; vy < numYVertices; vy++) {
1358       for (ex = 0; ex < numXEdges; ex++) {
1359         const PetscInt edge   = firstXEdge + vy * numXEdges + ex;
1360         const PetscInt v0     = firstVertex + vy * numXVertices + ex;
1361         const PetscInt v1     = firstVertex + vy * numXVertices + (ex + 1) % numXVertices;
1362         const PetscInt cone[] = {v0, v1};
1363 
1364         PetscCall(DMPlexSetCone(dm, edge, cone));
1365         if ((bdY != DM_BOUNDARY_PERIODIC) && (bdY != DM_BOUNDARY_TWIST)) {
1366           if (vy == numYVertices - 1) {
1367             PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerTop));
1368             PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1369             PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1370             if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1371           } else if (vy == 0) {
1372             PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerBottom));
1373             PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1374             PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1375             if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1376           }
1377         }
1378       }
1379     }
1380 
1381     /* Compute support, stratify, and celltype label */
1382     PetscCall(DMPlexSymmetrize(dm));
1383     PetscCall(DMPlexStratify(dm));
1384     PetscCall(DMPlexComputeCellTypes(dm));
1385 
1386     /* Build coordinates */
1387     PetscCall(DMGetCoordinateSection(dm, &coordSection));
1388     PetscCall(PetscSectionSetNumFields(coordSection, 1));
1389     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, 2));
1390     PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numVertices));
1391     for (v = firstVertex; v < firstVertex + numVertices; ++v) {
1392       PetscCall(PetscSectionSetDof(coordSection, v, 2));
1393       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, 2));
1394     }
1395     PetscCall(PetscSectionSetUp(coordSection));
1396     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
1397     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
1398     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
1399     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
1400     PetscCall(VecSetBlockSize(coordinates, 2));
1401     PetscCall(VecSetType(coordinates, VECSTANDARD));
1402     PetscCall(VecGetArray(coordinates, &coords));
1403     for (vy = 0; vy < numYVertices; ++vy) {
1404       for (vx = 0; vx < numXVertices; ++vx) {
1405         coords[2 * (vy * numXVertices + vx) + 0] = lower[0] + ((upper[0] - lower[0]) / numXEdges) * vx;
1406         coords[2 * (vy * numXVertices + vx) + 1] = lower[1] + ((upper[1] - lower[1]) / numYEdges) * vy;
1407       }
1408     }
1409     for (ey = 0; ey < numYEdges; ++ey) {
1410       for (ex = 0; ex < numXEdges; ++ex) {
1411         const PetscInt c = ey * numXEdges + ex + numYVertices * numXVertices;
1412 
1413         coords[2 * c + 0] = lower[0] + ((upper[0] - lower[0]) / numXEdges) * (ex + 0.5);
1414         coords[2 * c + 1] = lower[1] + ((upper[1] - lower[1]) / numYEdges) * (ey + 0.5);
1415       }
1416     }
1417     PetscCall(VecRestoreArray(coordinates, &coords));
1418     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
1419     PetscCall(VecDestroy(&coordinates));
1420 
1421     /* handle periodic BC */
1422     if (bdX == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_PERIODIC) {
1423       PetscReal L[2]       = {-1., -1.};
1424       PetscReal maxCell[2] = {-1., -1.};
1425 
1426       for (PetscInt d = 0; d < 2; ++d) {
1427         if (bd[d] != DM_BOUNDARY_NONE) {
1428           L[d]       = upper[d] - lower[d];
1429           maxCell[d] = 1.1 * (L[d] / PetscMax(1, edges[d]));
1430         }
1431       }
1432       PetscCall(DMSetPeriodicity(dm, maxCell, lower, L));
1433     }
1434   }
1435   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
1436   PetscFunctionReturn(PETSC_SUCCESS);
1437 }
1438 
1439 static PetscErrorCode DMPlexCreateBoxMesh_Simplex_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate)
1440 {
1441   DM        boundary, vol;
1442   DMLabel   bdlabel;
1443   PetscBool crisscross = PETSC_FALSE;
1444 
1445   PetscFunctionBegin;
1446   PetscAssertPointer(dm, 1);
1447   if (dim == 2) PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_box_crisscross", &crisscross, NULL));
1448   if (crisscross) {
1449     PetscCall(DMPlexCreateSquareMesh_Simplex_CrissCross(dm, faces, lower, upper, periodicity));
1450   } else {
1451     for (PetscInt i = 0; i < dim; ++i) PetscCheck(periodicity[i] == DM_BOUNDARY_NONE, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Periodicity is not supported for simplex meshes");
1452     PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &boundary));
1453     PetscCall(DMSetType(boundary, DMPLEX));
1454     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)boundary, ((PetscObject)dm)->prefix));
1455     PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(boundary, dim, faces, lower, upper, PETSC_FALSE));
1456     PetscCall(DMPlexGenerate(boundary, NULL, interpolate, &vol));
1457     PetscCall(DMGetLabel(vol, "marker", &bdlabel));
1458     if (bdlabel) PetscCall(DMPlexLabelComplete(vol, bdlabel));
1459     PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_FALSE, vol));
1460     PetscCall(DMPlexReplace_Internal(dm, &vol));
1461     PetscCall(DMDestroy(&boundary));
1462   }
1463   if (interpolate) {
1464     PetscCall(DMPlexInterpolateInPlace_Internal(dm));
1465     PetscCall(DMPlexSetBoxLabel_Internal(dm, periodicity));
1466   }
1467   PetscFunctionReturn(PETSC_SUCCESS);
1468 }
1469 
1470 static PetscErrorCode DMPlexCreateCubeMesh_Internal(DM dm, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[], DMBoundaryType bdX, DMBoundaryType bdY, DMBoundaryType bdZ)
1471 {
1472   DMLabel     cutLabel  = NULL;
1473   PetscInt    markerTop = 1, faceMarkerTop = 1;
1474   PetscInt    markerBottom = 1, faceMarkerBottom = 1;
1475   PetscInt    markerFront = 1, faceMarkerFront = 1;
1476   PetscInt    markerBack = 1, faceMarkerBack = 1;
1477   PetscInt    markerRight = 1, faceMarkerRight = 1;
1478   PetscInt    markerLeft = 1, faceMarkerLeft = 1;
1479   PetscInt    dim;
1480   PetscBool   markerSeparate = PETSC_FALSE, cutMarker = PETSC_FALSE;
1481   PetscMPIInt rank;
1482 
1483   PetscFunctionBegin;
1484   PetscCall(DMGetDimension(dm, &dim));
1485   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
1486   PetscCall(DMCreateLabel(dm, "marker"));
1487   PetscCall(DMCreateLabel(dm, "Face Sets"));
1488   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_periodic_cut", &cutMarker, NULL));
1489   if (bdX == DM_BOUNDARY_PERIODIC || bdX == DM_BOUNDARY_TWIST || bdY == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_TWIST || bdZ == DM_BOUNDARY_PERIODIC || bdZ == DM_BOUNDARY_TWIST) {
1490     if (cutMarker) {
1491       PetscCall(DMCreateLabel(dm, "periodic_cut"));
1492       PetscCall(DMGetLabel(dm, "periodic_cut", &cutLabel));
1493     }
1494   }
1495   switch (dim) {
1496   case 2:
1497     faceMarkerTop    = 3;
1498     faceMarkerBottom = 1;
1499     faceMarkerRight  = 2;
1500     faceMarkerLeft   = 4;
1501     break;
1502   case 3:
1503     faceMarkerBottom = 1;
1504     faceMarkerTop    = 2;
1505     faceMarkerFront  = 3;
1506     faceMarkerBack   = 4;
1507     faceMarkerRight  = 5;
1508     faceMarkerLeft   = 6;
1509     break;
1510   default:
1511     SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Dimension %" PetscInt_FMT " not supported", dim);
1512   }
1513   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_separate_marker", &markerSeparate, NULL));
1514   if (markerSeparate) {
1515     markerBottom = faceMarkerBottom;
1516     markerTop    = faceMarkerTop;
1517     markerFront  = faceMarkerFront;
1518     markerBack   = faceMarkerBack;
1519     markerRight  = faceMarkerRight;
1520     markerLeft   = faceMarkerLeft;
1521   }
1522   {
1523     const PetscInt numXEdges    = rank == 0 ? edges[0] : 0;
1524     const PetscInt numYEdges    = rank == 0 ? edges[1] : 0;
1525     const PetscInt numZEdges    = rank == 0 ? edges[2] : 0;
1526     const PetscInt numXVertices = rank == 0 ? (bdX == DM_BOUNDARY_PERIODIC || bdX == DM_BOUNDARY_TWIST ? edges[0] : edges[0] + 1) : 0;
1527     const PetscInt numYVertices = rank == 0 ? (bdY == DM_BOUNDARY_PERIODIC || bdY == DM_BOUNDARY_TWIST ? edges[1] : edges[1] + 1) : 0;
1528     const PetscInt numZVertices = rank == 0 ? (bdZ == DM_BOUNDARY_PERIODIC || bdZ == DM_BOUNDARY_TWIST ? edges[2] : edges[2] + 1) : 0;
1529     const PetscInt numCells     = numXEdges * numYEdges * numZEdges;
1530     const PetscInt numXFaces    = numYEdges * numZEdges;
1531     const PetscInt numYFaces    = numXEdges * numZEdges;
1532     const PetscInt numZFaces    = numXEdges * numYEdges;
1533     const PetscInt numTotXFaces = numXVertices * numXFaces;
1534     const PetscInt numTotYFaces = numYVertices * numYFaces;
1535     const PetscInt numTotZFaces = numZVertices * numZFaces;
1536     const PetscInt numFaces     = numTotXFaces + numTotYFaces + numTotZFaces;
1537     const PetscInt numTotXEdges = numXEdges * numYVertices * numZVertices;
1538     const PetscInt numTotYEdges = numYEdges * numXVertices * numZVertices;
1539     const PetscInt numTotZEdges = numZEdges * numXVertices * numYVertices;
1540     const PetscInt numVertices  = numXVertices * numYVertices * numZVertices;
1541     const PetscInt numEdges     = numTotXEdges + numTotYEdges + numTotZEdges;
1542     const PetscInt firstVertex  = (dim == 2) ? numFaces : numCells;
1543     const PetscInt firstXFace   = (dim == 2) ? 0 : numCells + numVertices;
1544     const PetscInt firstYFace   = firstXFace + numTotXFaces;
1545     const PetscInt firstZFace   = firstYFace + numTotYFaces;
1546     const PetscInt firstXEdge   = numCells + numFaces + numVertices;
1547     const PetscInt firstYEdge   = firstXEdge + numTotXEdges;
1548     const PetscInt firstZEdge   = firstYEdge + numTotYEdges;
1549     Vec            coordinates;
1550     PetscSection   coordSection;
1551     PetscScalar   *coords;
1552     PetscInt       coordSize;
1553     PetscInt       v, vx, vy, vz;
1554     PetscInt       c, f, fx, fy, fz, e, ex, ey, ez;
1555 
1556     PetscCall(DMPlexSetChart(dm, 0, numCells + numFaces + numEdges + numVertices));
1557     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 6));
1558     for (f = firstXFace; f < firstXFace + numFaces; ++f) PetscCall(DMPlexSetConeSize(dm, f, 4));
1559     for (e = firstXEdge; e < firstXEdge + numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
1560     PetscCall(DMSetUp(dm)); /* Allocate space for cones */
1561     /* Build cells */
1562     for (fz = 0; fz < numZEdges; ++fz) {
1563       for (fy = 0; fy < numYEdges; ++fy) {
1564         for (fx = 0; fx < numXEdges; ++fx) {
1565           PetscInt cell  = (fz * numYEdges + fy) * numXEdges + fx;
1566           PetscInt faceB = firstZFace + (fy * numXEdges + fx) * numZVertices + fz;
1567           PetscInt faceT = firstZFace + (fy * numXEdges + fx) * numZVertices + ((fz + 1) % numZVertices);
1568           PetscInt faceF = firstYFace + (fz * numXEdges + fx) * numYVertices + fy;
1569           PetscInt faceK = firstYFace + (fz * numXEdges + fx) * numYVertices + ((fy + 1) % numYVertices);
1570           PetscInt faceL = firstXFace + (fz * numYEdges + fy) * numXVertices + fx;
1571           PetscInt faceR = firstXFace + (fz * numYEdges + fy) * numXVertices + ((fx + 1) % numXVertices);
1572           /* B,  T,  F,  K,  R,  L */
1573           PetscInt ornt[6] = {-2, 0, 0, -3, 0, -2}; /* ??? */
1574           PetscInt cone[6];
1575 
1576           /* no boundary twisting in 3D */
1577           cone[0] = faceB;
1578           cone[1] = faceT;
1579           cone[2] = faceF;
1580           cone[3] = faceK;
1581           cone[4] = faceR;
1582           cone[5] = faceL;
1583           PetscCall(DMPlexSetCone(dm, cell, cone));
1584           PetscCall(DMPlexSetConeOrientation(dm, cell, ornt));
1585           if (bdX != DM_BOUNDARY_NONE && fx == numXEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
1586           if (bdY != DM_BOUNDARY_NONE && fy == numYEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
1587           if (bdZ != DM_BOUNDARY_NONE && fz == numZEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, cell, 2));
1588         }
1589       }
1590     }
1591     /* Build x faces */
1592     for (fz = 0; fz < numZEdges; ++fz) {
1593       for (fy = 0; fy < numYEdges; ++fy) {
1594         for (fx = 0; fx < numXVertices; ++fx) {
1595           PetscInt face    = firstXFace + (fz * numYEdges + fy) * numXVertices + fx;
1596           PetscInt edgeL   = firstZEdge + (fy * numXVertices + fx) * numZEdges + fz;
1597           PetscInt edgeR   = firstZEdge + (((fy + 1) % numYVertices) * numXVertices + fx) * numZEdges + fz;
1598           PetscInt edgeB   = firstYEdge + (fz * numXVertices + fx) * numYEdges + fy;
1599           PetscInt edgeT   = firstYEdge + (((fz + 1) % numZVertices) * numXVertices + fx) * numYEdges + fy;
1600           PetscInt ornt[4] = {0, 0, -1, -1};
1601           PetscInt cone[4];
1602 
1603           if (dim == 3) {
1604             /* markers */
1605             if (bdX != DM_BOUNDARY_PERIODIC) {
1606               if (fx == numXVertices - 1) {
1607                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerRight));
1608                 PetscCall(DMSetLabelValue(dm, "marker", face, markerRight));
1609               } else if (fx == 0) {
1610                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerLeft));
1611                 PetscCall(DMSetLabelValue(dm, "marker", face, markerLeft));
1612               }
1613             }
1614           }
1615           cone[0] = edgeB;
1616           cone[1] = edgeR;
1617           cone[2] = edgeT;
1618           cone[3] = edgeL;
1619           PetscCall(DMPlexSetCone(dm, face, cone));
1620           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
1621         }
1622       }
1623     }
1624     /* Build y faces */
1625     for (fz = 0; fz < numZEdges; ++fz) {
1626       for (fx = 0; fx < numXEdges; ++fx) {
1627         for (fy = 0; fy < numYVertices; ++fy) {
1628           PetscInt face    = firstYFace + (fz * numXEdges + fx) * numYVertices + fy;
1629           PetscInt edgeL   = firstZEdge + (fy * numXVertices + fx) * numZEdges + fz;
1630           PetscInt edgeR   = firstZEdge + (fy * numXVertices + ((fx + 1) % numXVertices)) * numZEdges + fz;
1631           PetscInt edgeB   = firstXEdge + (fz * numYVertices + fy) * numXEdges + fx;
1632           PetscInt edgeT   = firstXEdge + (((fz + 1) % numZVertices) * numYVertices + fy) * numXEdges + fx;
1633           PetscInt ornt[4] = {0, 0, -1, -1};
1634           PetscInt cone[4];
1635 
1636           if (dim == 3) {
1637             /* markers */
1638             if (bdY != DM_BOUNDARY_PERIODIC) {
1639               if (fy == numYVertices - 1) {
1640                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerBack));
1641                 PetscCall(DMSetLabelValue(dm, "marker", face, markerBack));
1642               } else if (fy == 0) {
1643                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerFront));
1644                 PetscCall(DMSetLabelValue(dm, "marker", face, markerFront));
1645               }
1646             }
1647           }
1648           cone[0] = edgeB;
1649           cone[1] = edgeR;
1650           cone[2] = edgeT;
1651           cone[3] = edgeL;
1652           PetscCall(DMPlexSetCone(dm, face, cone));
1653           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
1654         }
1655       }
1656     }
1657     /* Build z faces */
1658     for (fy = 0; fy < numYEdges; ++fy) {
1659       for (fx = 0; fx < numXEdges; ++fx) {
1660         for (fz = 0; fz < numZVertices; fz++) {
1661           PetscInt face    = firstZFace + (fy * numXEdges + fx) * numZVertices + fz;
1662           PetscInt edgeL   = firstYEdge + (fz * numXVertices + fx) * numYEdges + fy;
1663           PetscInt edgeR   = firstYEdge + (fz * numXVertices + ((fx + 1) % numXVertices)) * numYEdges + fy;
1664           PetscInt edgeB   = firstXEdge + (fz * numYVertices + fy) * numXEdges + fx;
1665           PetscInt edgeT   = firstXEdge + (fz * numYVertices + ((fy + 1) % numYVertices)) * numXEdges + fx;
1666           PetscInt ornt[4] = {0, 0, -1, -1};
1667           PetscInt cone[4];
1668 
1669           if (dim == 2) {
1670             if (bdX == DM_BOUNDARY_TWIST && fx == numXEdges - 1) {
1671               edgeR += numYEdges - 1 - 2 * fy;
1672               ornt[1] = -1;
1673             }
1674             if (bdY == DM_BOUNDARY_TWIST && fy == numYEdges - 1) {
1675               edgeT += numXEdges - 1 - 2 * fx;
1676               ornt[2] = 0;
1677             }
1678             if (bdX != DM_BOUNDARY_NONE && fx == numXEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, face, 2));
1679             if (bdY != DM_BOUNDARY_NONE && fy == numYEdges - 1 && cutLabel) PetscCall(DMLabelSetValue(cutLabel, face, 2));
1680           } else {
1681             /* markers */
1682             if (bdZ != DM_BOUNDARY_PERIODIC) {
1683               if (fz == numZVertices - 1) {
1684                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerTop));
1685                 PetscCall(DMSetLabelValue(dm, "marker", face, markerTop));
1686               } else if (fz == 0) {
1687                 PetscCall(DMSetLabelValue(dm, "Face Sets", face, faceMarkerBottom));
1688                 PetscCall(DMSetLabelValue(dm, "marker", face, markerBottom));
1689               }
1690             }
1691           }
1692           cone[0] = edgeB;
1693           cone[1] = edgeR;
1694           cone[2] = edgeT;
1695           cone[3] = edgeL;
1696           PetscCall(DMPlexSetCone(dm, face, cone));
1697           PetscCall(DMPlexSetConeOrientation(dm, face, ornt));
1698         }
1699       }
1700     }
1701     /* Build Z edges*/
1702     for (vy = 0; vy < numYVertices; vy++) {
1703       for (vx = 0; vx < numXVertices; vx++) {
1704         for (ez = 0; ez < numZEdges; ez++) {
1705           const PetscInt edge    = firstZEdge + (vy * numXVertices + vx) * numZEdges + ez;
1706           const PetscInt vertexB = firstVertex + (ez * numYVertices + vy) * numXVertices + vx;
1707           const PetscInt vertexT = firstVertex + (((ez + 1) % numZVertices) * numYVertices + vy) * numXVertices + vx;
1708           PetscInt       cone[2];
1709 
1710           cone[0] = vertexB;
1711           cone[1] = vertexT;
1712           PetscCall(DMPlexSetCone(dm, edge, cone));
1713           if (dim == 3) {
1714             if (bdX != DM_BOUNDARY_PERIODIC) {
1715               if (vx == numXVertices - 1) {
1716                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1717                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1718                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1719               } else if (vx == 0) {
1720                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1721                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1722                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1723               }
1724             }
1725             if (bdY != DM_BOUNDARY_PERIODIC) {
1726               if (vy == numYVertices - 1) {
1727                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBack));
1728                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBack));
1729                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBack));
1730               } else if (vy == 0) {
1731                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerFront));
1732                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerFront));
1733                 if (ez == numZEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerFront));
1734               }
1735             }
1736           }
1737         }
1738       }
1739     }
1740     /* Build Y edges*/
1741     for (vz = 0; vz < numZVertices; vz++) {
1742       for (vx = 0; vx < numXVertices; vx++) {
1743         for (ey = 0; ey < numYEdges; ey++) {
1744           const PetscInt nextv   = (dim == 2 && bdY == DM_BOUNDARY_TWIST && ey == numYEdges - 1) ? (numXVertices - vx - 1) : (vz * numYVertices + ((ey + 1) % numYVertices)) * numXVertices + vx;
1745           const PetscInt edge    = firstYEdge + (vz * numXVertices + vx) * numYEdges + ey;
1746           const PetscInt vertexF = firstVertex + (vz * numYVertices + ey) * numXVertices + vx;
1747           const PetscInt vertexK = firstVertex + nextv;
1748           PetscInt       cone[2];
1749 
1750           cone[0] = vertexF;
1751           cone[1] = vertexK;
1752           PetscCall(DMPlexSetCone(dm, edge, cone));
1753           if (dim == 2) {
1754             if ((bdX != DM_BOUNDARY_PERIODIC) && (bdX != DM_BOUNDARY_TWIST)) {
1755               if (vx == numXVertices - 1) {
1756                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerRight));
1757                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1758                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1759                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1760               } else if (vx == 0) {
1761                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerLeft));
1762                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1763                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1764                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1765               }
1766             } else {
1767               if (vx == 0 && cutLabel) {
1768                 PetscCall(DMLabelSetValue(cutLabel, edge, 1));
1769                 PetscCall(DMLabelSetValue(cutLabel, cone[0], 1));
1770                 if (ey == numYEdges - 1) PetscCall(DMLabelSetValue(cutLabel, cone[1], 1));
1771               }
1772             }
1773           } else {
1774             if (bdX != DM_BOUNDARY_PERIODIC) {
1775               if (vx == numXVertices - 1) {
1776                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerRight));
1777                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerRight));
1778                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerRight));
1779               } else if (vx == 0) {
1780                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerLeft));
1781                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerLeft));
1782                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerLeft));
1783               }
1784             }
1785             if (bdZ != DM_BOUNDARY_PERIODIC) {
1786               if (vz == numZVertices - 1) {
1787                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1788                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1789                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1790               } else if (vz == 0) {
1791                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1792                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1793                 if (ey == numYEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1794               }
1795             }
1796           }
1797         }
1798       }
1799     }
1800     /* Build X edges*/
1801     for (vz = 0; vz < numZVertices; vz++) {
1802       for (vy = 0; vy < numYVertices; vy++) {
1803         for (ex = 0; ex < numXEdges; ex++) {
1804           const PetscInt nextv   = (dim == 2 && bdX == DM_BOUNDARY_TWIST && ex == numXEdges - 1) ? (numYVertices - vy - 1) * numXVertices : (vz * numYVertices + vy) * numXVertices + (ex + 1) % numXVertices;
1805           const PetscInt edge    = firstXEdge + (vz * numYVertices + vy) * numXEdges + ex;
1806           const PetscInt vertexL = firstVertex + (vz * numYVertices + vy) * numXVertices + ex;
1807           const PetscInt vertexR = firstVertex + nextv;
1808           PetscInt       cone[2];
1809 
1810           cone[0] = vertexL;
1811           cone[1] = vertexR;
1812           PetscCall(DMPlexSetCone(dm, edge, cone));
1813           if (dim == 2) {
1814             if ((bdY != DM_BOUNDARY_PERIODIC) && (bdY != DM_BOUNDARY_TWIST)) {
1815               if (vy == numYVertices - 1) {
1816                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerTop));
1817                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1818                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1819                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1820               } else if (vy == 0) {
1821                 PetscCall(DMSetLabelValue(dm, "Face Sets", edge, faceMarkerBottom));
1822                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1823                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1824                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1825               }
1826             } else {
1827               if (vy == 0 && cutLabel) {
1828                 PetscCall(DMLabelSetValue(cutLabel, edge, 1));
1829                 PetscCall(DMLabelSetValue(cutLabel, cone[0], 1));
1830                 if (ex == numXEdges - 1) PetscCall(DMLabelSetValue(cutLabel, cone[1], 1));
1831               }
1832             }
1833           } else {
1834             if (bdY != DM_BOUNDARY_PERIODIC) {
1835               if (vy == numYVertices - 1) {
1836                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBack));
1837                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBack));
1838                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBack));
1839               } else if (vy == 0) {
1840                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerFront));
1841                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerFront));
1842                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerFront));
1843               }
1844             }
1845             if (bdZ != DM_BOUNDARY_PERIODIC) {
1846               if (vz == numZVertices - 1) {
1847                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerTop));
1848                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerTop));
1849                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerTop));
1850               } else if (vz == 0) {
1851                 PetscCall(DMSetLabelValue(dm, "marker", edge, markerBottom));
1852                 PetscCall(DMSetLabelValue(dm, "marker", cone[0], markerBottom));
1853                 if (ex == numXEdges - 1) PetscCall(DMSetLabelValue(dm, "marker", cone[1], markerBottom));
1854               }
1855             }
1856           }
1857         }
1858       }
1859     }
1860     PetscCall(DMPlexSymmetrize(dm));
1861     PetscCall(DMPlexStratify(dm));
1862     /* Build coordinates */
1863     PetscCall(DMGetCoordinateSection(dm, &coordSection));
1864     PetscCall(PetscSectionSetNumFields(coordSection, 1));
1865     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
1866     PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numVertices));
1867     for (v = firstVertex; v < firstVertex + numVertices; ++v) {
1868       PetscCall(PetscSectionSetDof(coordSection, v, dim));
1869       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
1870     }
1871     PetscCall(PetscSectionSetUp(coordSection));
1872     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
1873     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
1874     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
1875     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
1876     PetscCall(VecSetBlockSize(coordinates, dim));
1877     PetscCall(VecSetType(coordinates, VECSTANDARD));
1878     PetscCall(VecGetArray(coordinates, &coords));
1879     for (vz = 0; vz < numZVertices; ++vz) {
1880       for (vy = 0; vy < numYVertices; ++vy) {
1881         for (vx = 0; vx < numXVertices; ++vx) {
1882           coords[((vz * numYVertices + vy) * numXVertices + vx) * dim + 0] = lower[0] + ((upper[0] - lower[0]) / numXEdges) * vx;
1883           coords[((vz * numYVertices + vy) * numXVertices + vx) * dim + 1] = lower[1] + ((upper[1] - lower[1]) / numYEdges) * vy;
1884           if (dim == 3) coords[((vz * numYVertices + vy) * numXVertices + vx) * dim + 2] = lower[2] + ((upper[2] - lower[2]) / numZEdges) * vz;
1885         }
1886       }
1887     }
1888     PetscCall(VecRestoreArray(coordinates, &coords));
1889     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
1890     PetscCall(VecDestroy(&coordinates));
1891   }
1892   PetscFunctionReturn(PETSC_SUCCESS);
1893 }
1894 
1895 static PetscErrorCode DMPlexCreateBoxMesh_Tensor_Internal(DM dm, PetscInt dim, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[])
1896 {
1897   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
1898   PetscInt       fac[3] = {0, 0, 0}, d;
1899 
1900   PetscFunctionBegin;
1901   PetscAssertPointer(dm, 1);
1902   PetscValidLogicalCollectiveInt(dm, dim, 2);
1903   PetscCall(DMSetDimension(dm, dim));
1904   for (d = 0; d < dim; ++d) {
1905     fac[d] = faces[d];
1906     bdt[d] = periodicity[d];
1907   }
1908   PetscCall(DMPlexCreateCubeMesh_Internal(dm, lower, upper, fac, bdt[0], bdt[1], bdt[2]));
1909   if (periodicity[0] == DM_BOUNDARY_PERIODIC || periodicity[0] == DM_BOUNDARY_TWIST || periodicity[1] == DM_BOUNDARY_PERIODIC || periodicity[1] == DM_BOUNDARY_TWIST || (dim > 2 && (periodicity[2] == DM_BOUNDARY_PERIODIC || periodicity[2] == DM_BOUNDARY_TWIST))) {
1910     PetscReal L[3]       = {-1., -1., 0.};
1911     PetscReal maxCell[3] = {-1., -1., 0.};
1912 
1913     for (d = 0; d < dim; ++d) {
1914       if (periodicity[d] != DM_BOUNDARY_NONE) {
1915         L[d]       = upper[d] - lower[d];
1916         maxCell[d] = 1.1 * (L[d] / PetscMax(1, faces[d]));
1917       }
1918     }
1919     PetscCall(DMSetPeriodicity(dm, maxCell, lower, L));
1920   }
1921   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
1922   PetscFunctionReturn(PETSC_SUCCESS);
1923 }
1924 
1925 static PetscErrorCode DMPlexCreateBoxMesh_Internal(DM dm, DMPlexShape shape, PetscInt dim, PetscBool simplex, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate)
1926 {
1927   PetscFunctionBegin;
1928   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
1929   if (shape == DM_SHAPE_ZBOX) PetscCall(DMPlexCreateBoxMesh_Tensor_SFC_Internal(dm, dim, faces, lower, upper, periodicity, interpolate));
1930   else if (dim == 1) PetscCall(DMPlexCreateLineMesh_Internal(dm, faces[0], lower[0], upper[0], periodicity[0]));
1931   else if (simplex) PetscCall(DMPlexCreateBoxMesh_Simplex_Internal(dm, dim, faces, lower, upper, periodicity, interpolate));
1932   else PetscCall(DMPlexCreateBoxMesh_Tensor_Internal(dm, dim, faces, lower, upper, periodicity));
1933   if (!interpolate && dim > 1 && !simplex) {
1934     DM udm;
1935 
1936     PetscCall(DMPlexUninterpolate(dm, &udm));
1937     PetscCall(DMPlexCopyCoordinates(dm, udm));
1938     PetscCall(DMPlexReplace_Internal(dm, &udm));
1939   }
1940   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
1941   PetscFunctionReturn(PETSC_SUCCESS);
1942 }
1943 
1944 /*@
1945   DMPlexCreateBoxMesh - Creates a mesh on the tensor product of unit intervals (box) using simplices or tensor cells (hexahedra).
1946 
1947   Collective
1948 
1949   Input Parameters:
1950 + comm               - The communicator for the `DM` object
1951 . dim                - The spatial dimension
1952 . simplex            - `PETSC_TRUE` for simplices, `PETSC_FALSE` for tensor cells
1953 . faces              - Number of faces per dimension, or `NULL` for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
1954 . lower              - The lower left corner, or `NULL` for (0, 0, 0)
1955 . upper              - The upper right corner, or `NULL` for (1, 1, 1)
1956 . periodicity        - The boundary type for the X,Y,Z direction, or `NULL` for `DM_BOUNDARY_NONE`
1957 . interpolate        - Flag to create intermediate mesh pieces (edges, faces)
1958 . localizationHeight - Flag to localize edges and faces in addition to cells; only significant for periodic meshes
1959 - sparseLocalize     - Flag to localize coordinates only for cells near the periodic boundary; only significant for periodic meshes
1960 
1961   Output Parameter:
1962 . dm - The `DM` object
1963 
1964   Level: beginner
1965 
1966   Note:
1967   To customize this mesh using options, use
1968 .vb
1969   DMCreate(comm, &dm);
1970   DMSetType(dm, DMPLEX);
1971   DMSetFromOptions(dm);
1972 .ve
1973   and use the options in `DMSetFromOptions()`.
1974 
1975   Here is the numbering returned for 2 faces in each direction for tensor cells\:
1976 .vb
1977  10---17---11---18----12
1978   |         |         |
1979   |         |         |
1980  20    2   22    3    24
1981   |         |         |
1982   |         |         |
1983   7---15----8---16----9
1984   |         |         |
1985   |         |         |
1986  19    0   21    1   23
1987   |         |         |
1988   |         |         |
1989   4---13----5---14----6
1990 .ve
1991   and for simplicial cells
1992 .vb
1993  14----8---15----9----16
1994   |\     5  |\      7 |
1995   | \       | \       |
1996  13   2    14    3    15
1997   | 4   \   | 6   \   |
1998   |       \ |       \ |
1999  11----6---12----7----13
2000   |\        |\        |
2001   | \    1  | \     3 |
2002  10   0    11    1    12
2003   | 0   \   | 2   \   |
2004   |       \ |       \ |
2005   8----4----9----5----10
2006 .ve
2007 
2008 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMSetFromOptions()`, `DMPlexCreateFromFile()`, `DMPlexCreateHexCylinderMesh()`, `DMSetType()`, `DMCreate()`
2009 @*/
2010 PetscErrorCode DMPlexCreateBoxMesh(MPI_Comm comm, PetscInt dim, PetscBool simplex, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool interpolate, PetscInt localizationHeight, PetscBool sparseLocalize, DM *dm)
2011 {
2012   PetscInt       fac[3] = {1, 1, 1};
2013   PetscReal      low[3] = {0, 0, 0};
2014   PetscReal      upp[3] = {1, 1, 1};
2015   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
2016 
2017   PetscFunctionBegin;
2018   PetscCall(DMCreate(comm, dm));
2019   PetscCall(DMSetType(*dm, DMPLEX));
2020   PetscCall(DMPlexCreateBoxMesh_Internal(*dm, DM_SHAPE_BOX, dim, simplex, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, periodicity ? periodicity : bdt, interpolate));
2021   if (periodicity) {
2022     DM cdm;
2023 
2024     PetscCall(DMGetCoordinateDM(*dm, &cdm));
2025     PetscCall(DMPlexSetMaxProjectionHeight(cdm, localizationHeight));
2026     PetscCall(DMSetSparseLocalize(*dm, sparseLocalize));
2027     PetscCall(DMLocalizeCoordinates(*dm));
2028   }
2029   PetscFunctionReturn(PETSC_SUCCESS);
2030 }
2031 
2032 static PetscErrorCode DMPlexCreateWedgeBoxMesh_Internal(DM dm, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[])
2033 {
2034   DM       bdm, vol;
2035   PetscInt i;
2036 
2037   PetscFunctionBegin;
2038   // TODO Now we can support periodicity
2039   for (i = 0; i < 3; ++i) PetscCheck(periodicity[i] == DM_BOUNDARY_NONE, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Periodicity not yet supported");
2040   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &bdm));
2041   PetscCall(DMSetType(bdm, DMPLEX));
2042   PetscCall(DMSetDimension(bdm, 2));
2043   PetscCall(PetscLogEventBegin(DMPLEX_Generate, bdm, 0, 0, 0));
2044   PetscCall(DMPlexCreateBoxMesh_Simplex_Internal(bdm, 2, faces, lower, upper, periodicity, PETSC_TRUE));
2045   PetscCall(DMPlexExtrude(bdm, faces[2], upper[2] - lower[2], PETSC_TRUE, PETSC_FALSE, PETSC_FALSE, NULL, NULL, &vol));
2046   PetscCall(PetscLogEventEnd(DMPLEX_Generate, bdm, 0, 0, 0));
2047   PetscCall(DMDestroy(&bdm));
2048   PetscCall(DMPlexReplace_Internal(dm, &vol));
2049   if (lower[2] != 0.0) {
2050     Vec          v;
2051     PetscScalar *x;
2052     PetscInt     cDim, n;
2053 
2054     PetscCall(DMGetCoordinatesLocal(dm, &v));
2055     PetscCall(VecGetBlockSize(v, &cDim));
2056     PetscCall(VecGetLocalSize(v, &n));
2057     PetscCall(VecGetArray(v, &x));
2058     x += cDim;
2059     for (i = 0; i < n; i += cDim) x[i] += lower[2];
2060     PetscCall(VecRestoreArray(v, &x));
2061     PetscCall(DMSetCoordinatesLocal(dm, v));
2062   }
2063   PetscFunctionReturn(PETSC_SUCCESS);
2064 }
2065 
2066 /*@
2067   DMPlexCreateWedgeBoxMesh - Creates a 3-D mesh tessellating the (x,y) plane and extruding in the third direction using wedge cells.
2068 
2069   Collective
2070 
2071   Input Parameters:
2072 + comm        - The communicator for the `DM` object
2073 . faces       - Number of faces per dimension, or `NULL` for (1, 1, 1)
2074 . lower       - The lower left corner, or `NULL` for (0, 0, 0)
2075 . upper       - The upper right corner, or `NULL` for (1, 1, 1)
2076 . periodicity - The boundary type for the X,Y,Z direction, or `NULL` for `DM_BOUNDARY_NONE`
2077 . orderHeight - If `PETSC_TRUE`, orders the extruded cells in the height first. Otherwise, orders the cell on the layers first
2078 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
2079 
2080   Output Parameter:
2081 . dm - The `DM` object
2082 
2083   Level: beginner
2084 
2085 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateHexCylinderMesh()`, `DMPlexCreateWedgeCylinderMesh()`, `DMExtrude()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
2086 @*/
2087 PetscErrorCode DMPlexCreateWedgeBoxMesh(MPI_Comm comm, const PetscInt faces[], const PetscReal lower[], const PetscReal upper[], const DMBoundaryType periodicity[], PetscBool orderHeight, PetscBool interpolate, DM *dm)
2088 {
2089   PetscInt       fac[3] = {1, 1, 1};
2090   PetscReal      low[3] = {0, 0, 0};
2091   PetscReal      upp[3] = {1, 1, 1};
2092   DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
2093 
2094   PetscFunctionBegin;
2095   PetscCall(DMCreate(comm, dm));
2096   PetscCall(DMSetType(*dm, DMPLEX));
2097   PetscCall(DMPlexCreateWedgeBoxMesh_Internal(*dm, faces ? faces : fac, lower ? lower : low, upper ? upper : upp, periodicity ? periodicity : bdt));
2098   if (!interpolate) {
2099     DM udm;
2100 
2101     PetscCall(DMPlexUninterpolate(*dm, &udm));
2102     PetscCall(DMPlexReplace_Internal(*dm, &udm));
2103   }
2104   if (periodicity) PetscCall(DMLocalizeCoordinates(*dm));
2105   PetscFunctionReturn(PETSC_SUCCESS);
2106 }
2107 
2108 /*
2109   DMPlexTensorPointLexicographic_Private - Returns all tuples of size 'len' with nonnegative integers that are all less than or equal to 'max' for that dimension.
2110 
2111   Input Parameters:
2112 + len - The length of the tuple
2113 . max - The maximum for each dimension, so values are in [0, max)
2114 - tup - A tuple of length len+1: tup[len] > 0 indicates a stopping condition
2115 
2116   Output Parameter:
2117 . tup - A tuple of `len` integers whose entries are at most `max`
2118 
2119   Level: developer
2120 
2121   Note:
2122   Ordering is lexicographic with lowest index as least significant in ordering.
2123   e.g. for len == 2 and max == 2, this will return, in order, {0,0}, {1,0}, {2,0}, {0,1}, {1,1}, {2,1}, {0,2}, {1,2}, {2,2}.
2124 
2125 .seealso: PetscDualSpaceTensorPointLexicographic_Internal(), PetscDualSpaceLatticePointLexicographic_Internal()
2126 */
2127 static PetscErrorCode DMPlexTensorPointLexicographic_Private(PetscInt len, const PetscInt max[], PetscInt tup[])
2128 {
2129   PetscInt i;
2130 
2131   PetscFunctionBegin;
2132   for (i = 0; i < len; ++i) {
2133     if (tup[i] < max[i] - 1) {
2134       break;
2135     } else {
2136       tup[i] = 0;
2137     }
2138   }
2139   if (i == len) tup[i - 1] = max[i - 1];
2140   else ++tup[i];
2141   PetscFunctionReturn(PETSC_SUCCESS);
2142 }
2143 
2144 static PetscInt TupleToIndex_Private(PetscInt len, const PetscInt max[], const PetscInt tup[])
2145 {
2146   PetscInt idx = tup[len - 1];
2147 
2148   for (PetscInt i = len - 2; i >= 0; --i) {
2149     idx *= max[i];
2150     idx += tup[i];
2151   }
2152   return idx;
2153 }
2154 
2155 static void IndexToTuple_Private(PetscInt len, const PetscInt max[], PetscInt idx, PetscInt tup[])
2156 {
2157   for (PetscInt i = 0; i < len; ++i) {
2158     tup[i] = idx % max[i];
2159     idx    = (idx - tup[i]) / max[i];
2160   }
2161 }
2162 
2163 static void TupleToRanks_Private(PetscInt len, const PetscInt max[], const PetscInt procs[], const PetscInt tup[], PetscInt ranks[])
2164 {
2165   for (PetscInt i = 0; i < len; ++i) {
2166     const PetscInt div = max[i] / procs[i];
2167     const PetscInt rem = max[i] % procs[i];
2168     const PetscInt idx = (tup[i] < 0 ? max[i] + tup[i] : tup[i]) % max[i];
2169 
2170     if (idx < rem * (div + 1)) ranks[i] = idx / (div + 1);
2171     else ranks[i] = rem + (idx - rem * (div + 1)) / div;
2172   }
2173 }
2174 
2175 static void RanksToSizes_Private(PetscInt len, const PetscInt max[], const PetscInt procs[], const PetscInt ranks[], PetscInt sizes[])
2176 {
2177   for (PetscInt i = 0; i < len; ++i) {
2178     const PetscInt div = max[i] / procs[i];
2179     const PetscInt rem = max[i] % procs[i];
2180 
2181     sizes[i] = ranks[i] < rem ? div + 1 : div;
2182   }
2183 }
2184 
2185 /*
2186   In serial, the mesh is completely periodic. In parallel, we will include a layer of ghost vertices around the patch, so our edges will not wrap around in parallel. This will instead be handled by the SF.
2187 
2188   The cone for all edges will always be complete. Even in the presence of boundaries, we will keep all support sizes constant. When an edge would not exist in the support, we will create one to wrap back periodically. This allows for uniform stencils, and that edge will not be used for computation.
2189 
2190   All point which do not attain the vertex lower or upper bound in any dimension are owned, the rest are leaves owned by another process and present in the SF.
2191 
2192   Parallel Layout:
2193 
2194   We create a cubic process grid of dimension P^{1/d} on each side. The IndexToTuple_Private() function maps the global rank to the local rank in each dimension. We divide edges using the PETSc distribution rule in each dimension, and then add overlap. TupleToRanks_Private() returns the local rank in ech dimension for a tuple. RanksToSizes_Private() gives the size in each dimension for the domain with those local ranks.
2195 */
2196 static PetscErrorCode DMPlexCreateHypercubicMesh_Internal(DM dm, PetscInt dim, const PetscReal lower[], const PetscReal upper[], const PetscInt edges[], PetscInt overlap, const DMBoundaryType bd[])
2197 {
2198   const PetscInt debug = ((DM_Plex *)dm->data)->printAdj;
2199   PetscSF        sf;
2200   Vec            coordinates;
2201   PetscSection   coordSection;
2202   DMLabel        cutLabel    = NULL;
2203   PetscBool      cutMarker   = PETSC_FALSE;
2204   PetscBool      periodic    = PETSC_FALSE;
2205   PetscInt       numCells    = 1;
2206   PetscInt       numVertices = 1;
2207   PetscSFNode   *remotes;
2208   PetscScalar   *coords;
2209   PetscInt      *procs;     // The number of processes along each dimension
2210   PetscInt      *lrank;     // Rank in each dimension, lrank[d] \in [0, procs[d])
2211   PetscInt      *ledges;    // The number of edges along each dimension for this process
2212   PetscInt      *vstart;    // The first vertex along each dimension on this processes
2213   PetscInt      *vertices;  // The number of vertices along each dimension on this process
2214   PetscInt      *rvert;     // The global (not local) vertex number along each dimension
2215   PetscInt      *rrank;     // The rank along each dimension for the process owning rvert[]
2216   PetscInt      *rvertices; // The number of vertices along each dimension for the process rrank[]
2217   PetscInt      *vert, *vtmp, *supp, cone[2], *leaves;
2218   PetscInt       cell = 0, coordSize, Nl = 0, Nl2 = 0;
2219   PetscMPIInt    rank, size;
2220   MPI_Comm       comm;
2221 
2222   PetscFunctionBegin;
2223   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
2224   PetscCallMPI(MPI_Comm_rank(comm, &rank));
2225   PetscCallMPI(MPI_Comm_size(comm, &size));
2226   PetscCall(DMSetDimension(dm, dim));
2227   PetscCall(DMPlexDistributeSetDefault(dm, PETSC_FALSE));
2228   PetscCall(PetscCalloc4(dim, &procs, dim, &lrank, dim, &rrank, 2 * dim, &supp));
2229   PetscCall(PetscCalloc7(dim, &ledges, dim, &vertices, dim, &rvertices, dim, &vert, dim, &rvert, dim, &vstart, dim, &vtmp));
2230   PetscCall(DMCreateLabel(dm, "marker"));
2231   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_periodic_cut", &cutMarker, NULL));
2232   for (PetscInt d = 0; d < dim; ++d) periodic = (periodic || bd[d] == DM_BOUNDARY_PERIODIC) ? PETSC_TRUE : PETSC_FALSE;
2233   if (periodic && cutMarker) {
2234     PetscCall(DMCreateLabel(dm, "periodic_cut"));
2235     PetscCall(DMGetLabel(dm, "periodic_cut", &cutLabel));
2236   }
2237   for (PetscInt d = 0; d < dim; ++d) PetscCheck(bd[d] == DM_BOUNDARY_PERIODIC, comm, PETSC_ERR_SUP, "Hypercubic mesh must be periodic now");
2238   overlap = overlap == PETSC_DETERMINE ? 1 : overlap;
2239   PetscCheck(overlap >= 1, comm, PETSC_ERR_SUP, "Overlap %" PetscInt_FMT " must be greater than 0", overlap);
2240   if (size > 1) {
2241     PetscInt Npr = 1;
2242 
2243     // Make process grid
2244     if (debug) PetscCall(PetscPrintf(comm, "Process grid:"));
2245     for (PetscInt d = 0; d < dim; ++d) {
2246       procs[d] = PetscRintReal(PetscPowReal(size, 1. / dim));
2247       Npr *= procs[d];
2248       if (debug) PetscCall(PetscPrintf(comm, " %" PetscInt_FMT, procs[d]));
2249     }
2250     if (debug) PetscCall(PetscPrintf(comm, "\n"));
2251     PetscCheck(Npr == size, comm, PETSC_ERR_PLIB, "Process grid size %" PetscInt_FMT " != %d comm size", Npr, size);
2252     IndexToTuple_Private(dim, procs, rank, lrank);
2253     for (PetscInt d = 0; d < dim; ++d) {
2254       ledges[d] = edges[d] / procs[d] + (edges[d] % procs[d] > lrank[d] ? 1 : 0);
2255       vstart[d] = 0;
2256       for (PetscInt r = 0; r < lrank[d]; ++r) vstart[d] += edges[d] / procs[d] + (edges[d] % procs[d] > r ? 1 : 0);
2257       vstart[d] -= overlap; // For halo
2258     }
2259   } else {
2260     for (PetscInt d = 0; d < dim; ++d) {
2261       procs[d]  = 1;
2262       ledges[d] = edges[d];
2263     }
2264   }
2265   // Calculate local patch size
2266   for (PetscInt d = 0; d < dim; ++d) {
2267     vertices[d] = ledges[d] + (procs[d] > 1 ? 2 * overlap : 0);
2268     numVertices *= vertices[d];
2269   }
2270   numCells = numVertices * dim;
2271   PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
2272   for (PetscInt c = 0; c < numCells; ++c) PetscCall(DMPlexSetConeSize(dm, c, 2));
2273   for (PetscInt v = numCells; v < numCells + numVertices; ++v) PetscCall(DMPlexSetSupportSize(dm, v, 2 * dim));
2274   PetscCall(DMSetUp(dm)); /* Allocate space for cones and supports */
2275   /* Build cell cones and vertex supports */
2276   PetscCall(DMCreateLabel(dm, "celltype"));
2277   if (debug) PetscCall(PetscSynchronizedPrintf(comm, "Topology for rank %d:\n", rank));
2278   while (vert[dim - 1] < vertices[dim - 1]) {
2279     const PetscInt vertex = TupleToIndex_Private(dim, vertices, vert) + numCells;
2280     PetscInt       s      = 0;
2281     PetscBool      leaf   = PETSC_FALSE;
2282 
2283     if (debug) {
2284       PetscCall(PetscSynchronizedPrintf(comm, "Vertex %" PetscInt_FMT ":", vertex));
2285       for (PetscInt d = 0; d < dim; ++d) PetscCall(PetscSynchronizedPrintf(comm, " %" PetscInt_FMT, vert[d]));
2286       PetscCall(PetscSynchronizedPrintf(comm, "\n"));
2287     }
2288     PetscCall(DMPlexSetCellType(dm, vertex, DM_POLYTOPE_POINT));
2289     // Define edge cones
2290     for (PetscInt d = 0; d < dim; ++d) {
2291       for (PetscInt e = 0; e < dim; ++e) vtmp[e] = vert[e];
2292       vtmp[d] = (vert[d] + 1) % vertices[d];
2293       cone[0] = vertex;
2294       cone[1] = TupleToIndex_Private(dim, vertices, vtmp) + numCells;
2295       if (debug) {
2296         PetscCall(PetscSynchronizedPrintf(comm, "  Vertex %" PetscInt_FMT ":", cone[1]));
2297         for (PetscInt e = 0; e < dim; ++e) PetscCall(PetscSynchronizedPrintf(comm, " %" PetscInt_FMT, vtmp[e]));
2298         PetscCall(PetscSynchronizedPrintf(comm, "\n"));
2299       }
2300       PetscCall(DMPlexSetCone(dm, cell, cone));
2301       PetscCall(DMPlexSetCellType(dm, cell, DM_POLYTOPE_SEGMENT));
2302       if (debug) PetscCall(PetscSynchronizedPrintf(comm, "  Edge %" PetscInt_FMT " (%" PetscInt_FMT " %" PetscInt_FMT ")\n", cell, cone[0], cone[1]));
2303       ++cell;
2304       // Shared vertices are any in the first or last overlap layers
2305       if (vert[d] < overlap || vert[d] >= vertices[d] - overlap) leaf = PETSC_TRUE;
2306     }
2307     if (size > 1 && leaf) ++Nl;
2308     // Define vertex supports
2309     for (PetscInt d = 0; d < dim; ++d) {
2310       for (PetscInt e = 0; e < dim; ++e) vtmp[e] = vert[e];
2311       vtmp[d]   = (vert[d] + vertices[d] - 1) % vertices[d];
2312       supp[s++] = TupleToIndex_Private(dim, vertices, vtmp) * dim + d;
2313       supp[s++] = (vertex - numCells) * dim + d;
2314       PetscCall(DMPlexSetSupport(dm, vertex, supp));
2315     }
2316     PetscCall(DMPlexTensorPointLexicographic_Private(dim, vertices, vert));
2317   }
2318   if (debug) PetscCall(PetscSynchronizedFlush(comm, NULL));
2319   PetscCall(DMPlexStratify(dm));
2320   // Allocate for SF
2321   PetscCall(PetscMalloc1(Nl, &leaves));
2322   PetscCall(PetscMalloc1(Nl, &remotes));
2323   // Build coordinates
2324   PetscCall(DMGetCoordinateSection(dm, &coordSection));
2325   PetscCall(PetscSectionSetNumFields(coordSection, 1));
2326   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
2327   PetscCall(PetscSectionSetChart(coordSection, numCells, numCells + numVertices));
2328   for (PetscInt v = numCells; v < numCells + numVertices; ++v) {
2329     PetscCall(PetscSectionSetDof(coordSection, v, dim));
2330     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
2331   }
2332   PetscCall(PetscSectionSetUp(coordSection));
2333   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
2334   PetscCall(VecCreate(comm, &coordinates));
2335   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
2336   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
2337   PetscCall(VecSetBlockSize(coordinates, dim));
2338   PetscCall(VecSetType(coordinates, VECSTANDARD));
2339   PetscCall(VecGetArray(coordinates, &coords));
2340   if (debug) PetscCall(PetscSynchronizedPrintf(comm, "Geometry for rank %d:\n", rank));
2341   for (PetscInt d = 0; d < dim; ++d) vert[d] = 0;
2342   while (vert[dim - 1] < vertices[dim - 1]) {
2343     const PetscInt vertex = TupleToIndex_Private(dim, vertices, vert);
2344     PetscBool      leaf   = PETSC_FALSE;
2345 
2346     for (PetscInt d = 0; d < dim; ++d) {
2347       coords[vertex * dim + d] = lower[d] + ((upper[d] - lower[d]) / edges[d]) * (vert[d] + vstart[d]);
2348       if (vert[d] < overlap || vert[d] >= vertices[d] - overlap) leaf = PETSC_TRUE;
2349     }
2350     if (size > 1 && leaf) {
2351       PetscInt rnumCells = 1;
2352 
2353       for (PetscInt d = 0; d < dim; ++d) rvert[d] = vert[d] + vstart[d];
2354       TupleToRanks_Private(dim, edges, procs, rvert, rrank);
2355       leaves[Nl2]       = vertex + numCells;
2356       remotes[Nl2].rank = TupleToIndex_Private(dim, procs, rrank);
2357       RanksToSizes_Private(dim, edges, procs, rrank, rvertices);
2358       for (PetscInt d = 0; d < dim; ++d) {
2359         rvertices[d] += 2 * overlap; // Add halo
2360         rnumCells *= rvertices[d];
2361       }
2362       rnumCells *= dim;
2363       for (PetscInt d = 0; d < dim; ++d) {
2364         const PetscInt diff = rrank[d] - lrank[d];
2365 
2366         if (!diff) rvert[d] = vert[d];                                     // Vertex is local
2367         else if (rvert[d] < 0) rvert[d] = rvertices[d] - 1 + rvert[d];     // Wrap around at the bottom
2368         else if (rvert[d] >= edges[d]) rvert[d] = rvert[d] - edges[d] + 1; // Wrap around at the top
2369         else if (diff == -1) rvert[d] = rvertices[d] - 1 + (vert[d] - overlap);
2370         else if (diff == 1) rvert[d] = (vertices[d] - vert[d] - 1) + overlap;
2371         else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Process distance %" PetscInt_FMT " in direction %" PetscInt_FMT " should not be possible", diff, d);
2372       }
2373       remotes[Nl2].index = TupleToIndex_Private(dim, rvertices, rvert) + rnumCells;
2374       if (debug) PetscCall(PetscSynchronizedPrintf(comm, "Shared Vertex %" PetscInt_FMT " (%" PetscInt_FMT ", %" PetscInt_FMT ")\n", leaves[Nl2], remotes[Nl2].rank, remotes[Nl2].index));
2375       ++Nl2;
2376     }
2377     if (debug) {
2378       PetscCall(PetscSynchronizedPrintf(comm, "Vertex %" PetscInt_FMT ":", vertex));
2379       for (PetscInt d = 0; d < dim; ++d) PetscCall(PetscSynchronizedPrintf(comm, " %" PetscInt_FMT, vert[d] + vstart[d]));
2380       for (PetscInt d = 0; d < dim; ++d) PetscCall(PetscSynchronizedPrintf(comm, " %g", (double)PetscRealPart(coords[vertex * dim + d])));
2381       PetscCall(PetscSynchronizedPrintf(comm, "\n"));
2382     }
2383     PetscCall(DMPlexTensorPointLexicographic_Private(dim, vertices, vert));
2384   }
2385   if (debug) PetscCall(PetscSynchronizedFlush(comm, NULL));
2386   PetscCall(VecRestoreArray(coordinates, &coords));
2387   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
2388   PetscCall(VecDestroy(&coordinates));
2389   // Build SF
2390   PetscCheck(Nl == Nl2, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Initial number of leaves %" PetscInt_FMT " != %" PetscInt_FMT " final number", Nl, Nl2);
2391   PetscCall(DMGetPointSF(dm, &sf));
2392   PetscCall(PetscSFSetGraph(sf, numCells + numVertices, Nl, leaves, PETSC_OWN_POINTER, remotes, PETSC_OWN_POINTER));
2393   if (debug) PetscCall(PetscSFView(sf, PETSC_VIEWER_STDOUT_WORLD));
2394   //PetscCall(DMSetPeriodicity(dm, NULL, lower, upper));
2395   // Attach the extent
2396   {
2397     PetscContainer c;
2398     PetscInt      *extent, *lextent;
2399 
2400     PetscCall(PetscMalloc1(dim, &extent));
2401     PetscCall(PetscMalloc1(dim, &lextent));
2402     for (PetscInt d = 0; d < dim; ++d) {
2403       extent[d]  = edges[d];
2404       lextent[d] = ledges[d];
2405     }
2406     PetscCall(PetscContainerCreate(PETSC_COMM_SELF, &c));
2407     PetscCall(PetscContainerSetCtxDestroy(c, PetscCtxDestroyDefault));
2408     PetscCall(PetscContainerSetPointer(c, extent));
2409     PetscCall(PetscObjectCompose((PetscObject)dm, "_extent", (PetscObject)c));
2410     PetscCall(PetscContainerDestroy(&c));
2411     PetscCall(PetscContainerCreate(PETSC_COMM_SELF, &c));
2412     PetscCall(PetscContainerSetCtxDestroy(c, PetscCtxDestroyDefault));
2413     PetscCall(PetscContainerSetPointer(c, lextent));
2414     PetscCall(PetscObjectCompose((PetscObject)dm, "_lextent", (PetscObject)c));
2415     PetscCall(PetscContainerDestroy(&c));
2416   }
2417   PetscCall(PetscFree4(procs, lrank, rrank, supp));
2418   PetscCall(PetscFree7(ledges, vertices, rvertices, vert, rvert, vstart, vtmp));
2419   PetscFunctionReturn(PETSC_SUCCESS);
2420 }
2421 
2422 /*@C
2423   DMPlexCreateHypercubicMesh - Creates a periodic mesh on the tensor product of unit intervals using only vertices and edges.
2424 
2425   Collective
2426 
2427   Input Parameters:
2428 + comm    - The communicator for the `DM` object
2429 . dim     - The spatial dimension
2430 . edges   - Number of edges per dimension, or `NULL` for (1,) in 1D and (2, 2) in 2D and (1, 1, 1) in 3D
2431 . lower   - The lower left corner, or `NULL` for (0, 0, 0)
2432 . upper   - The upper right corner, or `NULL` for (1, 1, 1)
2433 - overlap - The number of vertices in each direction to include in the overlap (default is 1)
2434 
2435   Output Parameter:
2436 . dm - The DM object
2437 
2438   Level: beginner
2439 
2440   Note:
2441   If you want to customize this mesh using options, you just need to
2442 .vb
2443   DMCreate(comm, &dm);
2444   DMSetType(dm, DMPLEX);
2445   DMSetFromOptions(dm);
2446 .ve
2447   and use the options on the `DMSetFromOptions()` page.
2448 
2449   The vertices are numbered is lexicographic order, and the dim edges exiting a vertex in the positive orthant are number consecutively,
2450 .vb
2451  18--0-19--2-20--4-18
2452   |     |     |     |
2453  13    15    17    13
2454   |     |     |     |
2455  24-12-25-14-26-16-24
2456   |     |     |     |
2457   7     9    11     7
2458   |     |     |     |
2459  21--6-22--8-23-10-21
2460   |     |     |     |
2461   1     3     5     1
2462   |     |     |     |
2463  18--0-19--2-20--4-18
2464 .ve
2465 
2466 .seealso: `DMSetFromOptions()`, `DMPlexCreateFromFile()`, `DMPlexCreateHexCylinderMesh()`, `DMSetType()`, `DMCreate()`
2467 @*/
2468 PetscErrorCode DMPlexCreateHypercubicMesh(MPI_Comm comm, PetscInt dim, const PetscInt edges[], const PetscReal lower[], const PetscReal upper[], PetscInt overlap, DM *dm)
2469 {
2470   PetscInt       *edg;
2471   PetscReal      *low, *upp;
2472   DMBoundaryType *bdt;
2473   PetscInt        d;
2474 
2475   PetscFunctionBegin;
2476   PetscCall(DMCreate(comm, dm));
2477   PetscCall(DMSetType(*dm, DMPLEX));
2478   PetscCall(PetscMalloc4(dim, &edg, dim, &low, dim, &upp, dim, &bdt));
2479   for (d = 0; d < dim; ++d) {
2480     edg[d] = edges ? edges[d] : 1;
2481     low[d] = lower ? lower[d] : 0.;
2482     upp[d] = upper ? upper[d] : 1.;
2483     bdt[d] = DM_BOUNDARY_PERIODIC;
2484   }
2485   PetscCall(DMPlexCreateHypercubicMesh_Internal(*dm, dim, low, upp, edg, overlap, bdt));
2486   PetscCall(PetscFree4(edg, low, upp, bdt));
2487   PetscFunctionReturn(PETSC_SUCCESS);
2488 }
2489 
2490 /*@
2491   DMPlexSetOptionsPrefix - Sets the prefix used for searching for all `DM` options in the database.
2492 
2493   Logically Collective
2494 
2495   Input Parameters:
2496 + dm     - the `DM` context
2497 - prefix - the prefix to prepend to all option names
2498 
2499   Level: advanced
2500 
2501   Note:
2502   A hyphen (-) must NOT be given at the beginning of the prefix name.
2503   The first character of all runtime options is AUTOMATICALLY the hyphen.
2504 
2505 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `SNESSetFromOptions()`
2506 @*/
2507 PetscErrorCode DMPlexSetOptionsPrefix(DM dm, const char prefix[])
2508 {
2509   DM_Plex *mesh = (DM_Plex *)dm->data;
2510 
2511   PetscFunctionBegin;
2512   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2513   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)dm, prefix));
2514   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)mesh->partitioner, prefix));
2515   PetscFunctionReturn(PETSC_SUCCESS);
2516 }
2517 
2518 /* Remap geometry to cylinder
2519    TODO: This only works for a single refinement, then it is broken
2520 
2521      Interior square: Linear interpolation is correct
2522      The other cells all have vertices on rays from the origin. We want to uniformly expand the spacing
2523      such that the last vertex is on the unit circle. So the closest and farthest vertices are at distance
2524 
2525        phi     = arctan(y/x)
2526        d_close = sqrt(1/8 + 1/4 sin^2(phi))
2527        d_far   = sqrt(1/2 + sin^2(phi))
2528 
2529      so we remap them using
2530 
2531        x_new = x_close + (x - x_close) (1 - d_close) / (d_far - d_close)
2532        y_new = y_close + (y - y_close) (1 - d_close) / (d_far - d_close)
2533 
2534      If pi/4 < phi < 3pi/4 or -3pi/4 < phi < -pi/4, then we switch x and y.
2535 */
2536 static void snapToCylinder(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
2537 {
2538   const PetscReal dis = 1.0 / PetscSqrtReal(2.0);
2539   const PetscReal ds2 = 0.5 * dis;
2540 
2541   if ((PetscAbsScalar(u[0]) <= ds2) && (PetscAbsScalar(u[1]) <= ds2)) {
2542     f0[0] = u[0];
2543     f0[1] = u[1];
2544   } else {
2545     PetscReal phi, sinp, cosp, dc, df, x, y, xc, yc;
2546 
2547     x    = PetscRealPart(u[0]);
2548     y    = PetscRealPart(u[1]);
2549     phi  = PetscAtan2Real(y, x);
2550     sinp = PetscSinReal(phi);
2551     cosp = PetscCosReal(phi);
2552     if ((PetscAbsReal(phi) > PETSC_PI / 4.0) && (PetscAbsReal(phi) < 3.0 * PETSC_PI / 4.0)) {
2553       dc = PetscAbsReal(ds2 / sinp);
2554       df = PetscAbsReal(dis / sinp);
2555       xc = ds2 * x / PetscAbsReal(y);
2556       yc = ds2 * PetscSignReal(y);
2557     } else {
2558       dc = PetscAbsReal(ds2 / cosp);
2559       df = PetscAbsReal(dis / cosp);
2560       xc = ds2 * PetscSignReal(x);
2561       yc = ds2 * y / PetscAbsReal(x);
2562     }
2563     f0[0] = xc + (u[0] - xc) * (1.0 - dc) / (df - dc);
2564     f0[1] = yc + (u[1] - yc) * (1.0 - dc) / (df - dc);
2565   }
2566   f0[2] = u[2];
2567 }
2568 
2569 static PetscErrorCode DMPlexCreateHexCylinderMesh_Internal(DM dm, DMBoundaryType periodicZ, PetscInt Nr)
2570 {
2571   const PetscInt dim = 3;
2572   PetscInt       numCells, numVertices;
2573   PetscMPIInt    rank;
2574 
2575   PetscFunctionBegin;
2576   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
2577   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
2578   PetscCall(DMSetDimension(dm, dim));
2579   /* Create topology */
2580   {
2581     PetscInt cone[8], c;
2582 
2583     numCells    = rank == 0 ? 5 : 0;
2584     numVertices = rank == 0 ? 16 : 0;
2585     if (periodicZ == DM_BOUNDARY_PERIODIC) {
2586       numCells *= 3;
2587       numVertices = rank == 0 ? 24 : 0;
2588     }
2589     PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
2590     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 8));
2591     PetscCall(DMSetUp(dm));
2592     if (rank == 0) {
2593       if (periodicZ == DM_BOUNDARY_PERIODIC) {
2594         cone[0] = 15;
2595         cone[1] = 18;
2596         cone[2] = 17;
2597         cone[3] = 16;
2598         cone[4] = 31;
2599         cone[5] = 32;
2600         cone[6] = 33;
2601         cone[7] = 34;
2602         PetscCall(DMPlexSetCone(dm, 0, cone));
2603         cone[0] = 16;
2604         cone[1] = 17;
2605         cone[2] = 24;
2606         cone[3] = 23;
2607         cone[4] = 32;
2608         cone[5] = 36;
2609         cone[6] = 37;
2610         cone[7] = 33; /* 22 25 26 21 */
2611         PetscCall(DMPlexSetCone(dm, 1, cone));
2612         cone[0] = 18;
2613         cone[1] = 27;
2614         cone[2] = 24;
2615         cone[3] = 17;
2616         cone[4] = 34;
2617         cone[5] = 33;
2618         cone[6] = 37;
2619         cone[7] = 38;
2620         PetscCall(DMPlexSetCone(dm, 2, cone));
2621         cone[0] = 29;
2622         cone[1] = 27;
2623         cone[2] = 18;
2624         cone[3] = 15;
2625         cone[4] = 35;
2626         cone[5] = 31;
2627         cone[6] = 34;
2628         cone[7] = 38;
2629         PetscCall(DMPlexSetCone(dm, 3, cone));
2630         cone[0] = 29;
2631         cone[1] = 15;
2632         cone[2] = 16;
2633         cone[3] = 23;
2634         cone[4] = 35;
2635         cone[5] = 36;
2636         cone[6] = 32;
2637         cone[7] = 31;
2638         PetscCall(DMPlexSetCone(dm, 4, cone));
2639 
2640         cone[0] = 31;
2641         cone[1] = 34;
2642         cone[2] = 33;
2643         cone[3] = 32;
2644         cone[4] = 19;
2645         cone[5] = 22;
2646         cone[6] = 21;
2647         cone[7] = 20;
2648         PetscCall(DMPlexSetCone(dm, 5, cone));
2649         cone[0] = 32;
2650         cone[1] = 33;
2651         cone[2] = 37;
2652         cone[3] = 36;
2653         cone[4] = 22;
2654         cone[5] = 25;
2655         cone[6] = 26;
2656         cone[7] = 21;
2657         PetscCall(DMPlexSetCone(dm, 6, cone));
2658         cone[0] = 34;
2659         cone[1] = 38;
2660         cone[2] = 37;
2661         cone[3] = 33;
2662         cone[4] = 20;
2663         cone[5] = 21;
2664         cone[6] = 26;
2665         cone[7] = 28;
2666         PetscCall(DMPlexSetCone(dm, 7, cone));
2667         cone[0] = 35;
2668         cone[1] = 38;
2669         cone[2] = 34;
2670         cone[3] = 31;
2671         cone[4] = 30;
2672         cone[5] = 19;
2673         cone[6] = 20;
2674         cone[7] = 28;
2675         PetscCall(DMPlexSetCone(dm, 8, cone));
2676         cone[0] = 35;
2677         cone[1] = 31;
2678         cone[2] = 32;
2679         cone[3] = 36;
2680         cone[4] = 30;
2681         cone[5] = 25;
2682         cone[6] = 22;
2683         cone[7] = 19;
2684         PetscCall(DMPlexSetCone(dm, 9, cone));
2685 
2686         cone[0] = 19;
2687         cone[1] = 20;
2688         cone[2] = 21;
2689         cone[3] = 22;
2690         cone[4] = 15;
2691         cone[5] = 16;
2692         cone[6] = 17;
2693         cone[7] = 18;
2694         PetscCall(DMPlexSetCone(dm, 10, cone));
2695         cone[0] = 22;
2696         cone[1] = 21;
2697         cone[2] = 26;
2698         cone[3] = 25;
2699         cone[4] = 16;
2700         cone[5] = 23;
2701         cone[6] = 24;
2702         cone[7] = 17;
2703         PetscCall(DMPlexSetCone(dm, 11, cone));
2704         cone[0] = 20;
2705         cone[1] = 28;
2706         cone[2] = 26;
2707         cone[3] = 21;
2708         cone[4] = 18;
2709         cone[5] = 17;
2710         cone[6] = 24;
2711         cone[7] = 27;
2712         PetscCall(DMPlexSetCone(dm, 12, cone));
2713         cone[0] = 30;
2714         cone[1] = 28;
2715         cone[2] = 20;
2716         cone[3] = 19;
2717         cone[4] = 29;
2718         cone[5] = 15;
2719         cone[6] = 18;
2720         cone[7] = 27;
2721         PetscCall(DMPlexSetCone(dm, 13, cone));
2722         cone[0] = 30;
2723         cone[1] = 19;
2724         cone[2] = 22;
2725         cone[3] = 25;
2726         cone[4] = 29;
2727         cone[5] = 23;
2728         cone[6] = 16;
2729         cone[7] = 15;
2730         PetscCall(DMPlexSetCone(dm, 14, cone));
2731       } else {
2732         cone[0] = 5;
2733         cone[1] = 8;
2734         cone[2] = 7;
2735         cone[3] = 6;
2736         cone[4] = 9;
2737         cone[5] = 12;
2738         cone[6] = 11;
2739         cone[7] = 10;
2740         PetscCall(DMPlexSetCone(dm, 0, cone));
2741         cone[0] = 6;
2742         cone[1] = 7;
2743         cone[2] = 14;
2744         cone[3] = 13;
2745         cone[4] = 12;
2746         cone[5] = 15;
2747         cone[6] = 16;
2748         cone[7] = 11;
2749         PetscCall(DMPlexSetCone(dm, 1, cone));
2750         cone[0] = 8;
2751         cone[1] = 17;
2752         cone[2] = 14;
2753         cone[3] = 7;
2754         cone[4] = 10;
2755         cone[5] = 11;
2756         cone[6] = 16;
2757         cone[7] = 18;
2758         PetscCall(DMPlexSetCone(dm, 2, cone));
2759         cone[0] = 19;
2760         cone[1] = 17;
2761         cone[2] = 8;
2762         cone[3] = 5;
2763         cone[4] = 20;
2764         cone[5] = 9;
2765         cone[6] = 10;
2766         cone[7] = 18;
2767         PetscCall(DMPlexSetCone(dm, 3, cone));
2768         cone[0] = 19;
2769         cone[1] = 5;
2770         cone[2] = 6;
2771         cone[3] = 13;
2772         cone[4] = 20;
2773         cone[5] = 15;
2774         cone[6] = 12;
2775         cone[7] = 9;
2776         PetscCall(DMPlexSetCone(dm, 4, cone));
2777       }
2778     }
2779     PetscCall(DMPlexSymmetrize(dm));
2780     PetscCall(DMPlexStratify(dm));
2781   }
2782   /* Create cube geometry */
2783   {
2784     Vec             coordinates;
2785     PetscSection    coordSection;
2786     PetscScalar    *coords;
2787     PetscInt        coordSize, v;
2788     const PetscReal dis = 1.0 / PetscSqrtReal(2.0);
2789     const PetscReal ds2 = dis / 2.0;
2790 
2791     /* Build coordinates */
2792     PetscCall(DMGetCoordinateSection(dm, &coordSection));
2793     PetscCall(PetscSectionSetNumFields(coordSection, 1));
2794     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
2795     PetscCall(PetscSectionSetChart(coordSection, numCells, numCells + numVertices));
2796     for (v = numCells; v < numCells + numVertices; ++v) {
2797       PetscCall(PetscSectionSetDof(coordSection, v, dim));
2798       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
2799     }
2800     PetscCall(PetscSectionSetUp(coordSection));
2801     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
2802     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
2803     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
2804     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
2805     PetscCall(VecSetBlockSize(coordinates, dim));
2806     PetscCall(VecSetType(coordinates, VECSTANDARD));
2807     PetscCall(VecGetArray(coordinates, &coords));
2808     if (rank == 0) {
2809       coords[0 * dim + 0]  = -ds2;
2810       coords[0 * dim + 1]  = -ds2;
2811       coords[0 * dim + 2]  = 0.0;
2812       coords[1 * dim + 0]  = ds2;
2813       coords[1 * dim + 1]  = -ds2;
2814       coords[1 * dim + 2]  = 0.0;
2815       coords[2 * dim + 0]  = ds2;
2816       coords[2 * dim + 1]  = ds2;
2817       coords[2 * dim + 2]  = 0.0;
2818       coords[3 * dim + 0]  = -ds2;
2819       coords[3 * dim + 1]  = ds2;
2820       coords[3 * dim + 2]  = 0.0;
2821       coords[4 * dim + 0]  = -ds2;
2822       coords[4 * dim + 1]  = -ds2;
2823       coords[4 * dim + 2]  = 1.0;
2824       coords[5 * dim + 0]  = -ds2;
2825       coords[5 * dim + 1]  = ds2;
2826       coords[5 * dim + 2]  = 1.0;
2827       coords[6 * dim + 0]  = ds2;
2828       coords[6 * dim + 1]  = ds2;
2829       coords[6 * dim + 2]  = 1.0;
2830       coords[7 * dim + 0]  = ds2;
2831       coords[7 * dim + 1]  = -ds2;
2832       coords[7 * dim + 2]  = 1.0;
2833       coords[8 * dim + 0]  = dis;
2834       coords[8 * dim + 1]  = -dis;
2835       coords[8 * dim + 2]  = 0.0;
2836       coords[9 * dim + 0]  = dis;
2837       coords[9 * dim + 1]  = dis;
2838       coords[9 * dim + 2]  = 0.0;
2839       coords[10 * dim + 0] = dis;
2840       coords[10 * dim + 1] = -dis;
2841       coords[10 * dim + 2] = 1.0;
2842       coords[11 * dim + 0] = dis;
2843       coords[11 * dim + 1] = dis;
2844       coords[11 * dim + 2] = 1.0;
2845       coords[12 * dim + 0] = -dis;
2846       coords[12 * dim + 1] = dis;
2847       coords[12 * dim + 2] = 0.0;
2848       coords[13 * dim + 0] = -dis;
2849       coords[13 * dim + 1] = dis;
2850       coords[13 * dim + 2] = 1.0;
2851       coords[14 * dim + 0] = -dis;
2852       coords[14 * dim + 1] = -dis;
2853       coords[14 * dim + 2] = 0.0;
2854       coords[15 * dim + 0] = -dis;
2855       coords[15 * dim + 1] = -dis;
2856       coords[15 * dim + 2] = 1.0;
2857       if (periodicZ == DM_BOUNDARY_PERIODIC) {
2858         /* 15 31 19 */ coords[16 * dim + 0] = -ds2;
2859         coords[16 * dim + 1]                = -ds2;
2860         coords[16 * dim + 2]                = 0.5;
2861         /* 16 32 22 */ coords[17 * dim + 0] = ds2;
2862         coords[17 * dim + 1]                = -ds2;
2863         coords[17 * dim + 2]                = 0.5;
2864         /* 17 33 21 */ coords[18 * dim + 0] = ds2;
2865         coords[18 * dim + 1]                = ds2;
2866         coords[18 * dim + 2]                = 0.5;
2867         /* 18 34 20 */ coords[19 * dim + 0] = -ds2;
2868         coords[19 * dim + 1]                = ds2;
2869         coords[19 * dim + 2]                = 0.5;
2870         /* 29 35 30 */ coords[20 * dim + 0] = -dis;
2871         coords[20 * dim + 1]                = -dis;
2872         coords[20 * dim + 2]                = 0.5;
2873         /* 23 36 25 */ coords[21 * dim + 0] = dis;
2874         coords[21 * dim + 1]                = -dis;
2875         coords[21 * dim + 2]                = 0.5;
2876         /* 24 37 26 */ coords[22 * dim + 0] = dis;
2877         coords[22 * dim + 1]                = dis;
2878         coords[22 * dim + 2]                = 0.5;
2879         /* 27 38 28 */ coords[23 * dim + 0] = -dis;
2880         coords[23 * dim + 1]                = dis;
2881         coords[23 * dim + 2]                = 0.5;
2882       }
2883     }
2884     PetscCall(VecRestoreArray(coordinates, &coords));
2885     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
2886     PetscCall(VecDestroy(&coordinates));
2887   }
2888   /* Create periodicity */
2889   if (periodicZ == DM_BOUNDARY_PERIODIC || periodicZ == DM_BOUNDARY_TWIST) {
2890     PetscReal L[3]       = {-1., -1., 0.};
2891     PetscReal maxCell[3] = {-1., -1., 0.};
2892     PetscReal lower[3]   = {0.0, 0.0, 0.0};
2893     PetscReal upper[3]   = {1.0, 1.0, 1.5};
2894     PetscInt  numZCells  = 3;
2895 
2896     L[2]       = upper[2] - lower[2];
2897     maxCell[2] = 1.1 * (L[2] / numZCells);
2898     PetscCall(DMSetPeriodicity(dm, maxCell, lower, L));
2899   }
2900   {
2901     DM          cdm;
2902     PetscDS     cds;
2903     PetscScalar c[2] = {1.0, 1.0};
2904 
2905     PetscCall(DMPlexCreateCoordinateSpace(dm, 1, PETSC_TRUE, NULL));
2906     PetscCall(DMGetCoordinateDM(dm, &cdm));
2907     PetscCall(DMGetDS(cdm, &cds));
2908     PetscCall(PetscDSSetConstants(cds, 2, c));
2909   }
2910   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
2911 
2912   /* Wait for coordinate creation before doing in-place modification */
2913   PetscCall(DMPlexInterpolateInPlace_Internal(dm));
2914 
2915   char        oldprefix[PETSC_MAX_PATH_LEN];
2916   const char *prefix;
2917 
2918   PetscCall(PetscObjectGetOptionsPrefix((PetscObject)dm, &prefix));
2919   PetscCall(PetscStrncpy(oldprefix, prefix, PETSC_MAX_PATH_LEN));
2920   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)dm, "petsc_cyl_ref_"));
2921   for (PetscInt r = 0; r < PetscMax(0, Nr); ++r) {
2922     DM rdm;
2923 
2924     PetscCall(DMRefine(dm, PetscObjectComm((PetscObject)dm), &rdm));
2925     PetscCall(DMPlexReplace_Internal(dm, &rdm));
2926   }
2927   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)dm, oldprefix));
2928   PetscCall(DMPlexRemapGeometry(dm, 0.0, snapToCylinder));
2929 
2930   DMLabel         bdlabel, edgelabel;
2931   IS              faceIS;
2932   const PetscInt *faces;
2933   PetscInt        Nf;
2934 
2935   PetscCall(DMCreateLabel(dm, "marker"));
2936   PetscCall(DMGetLabel(dm, "marker", &bdlabel));
2937   PetscCall(DMCreateLabel(dm, "generatrix"));
2938   PetscCall(DMGetLabel(dm, "generatrix", &edgelabel));
2939   PetscCall(DMPlexMarkBoundaryFaces(dm, PETSC_DETERMINE, bdlabel));
2940   // Remove faces on top and bottom
2941   PetscCall(DMLabelGetStratumIS(bdlabel, 1, &faceIS));
2942   if (faceIS) {
2943     PetscCall(ISGetLocalSize(faceIS, &Nf));
2944     PetscCall(ISGetIndices(faceIS, &faces));
2945     for (PetscInt f = 0; f < Nf; ++f) {
2946       PetscReal vol, normal[3];
2947 
2948       PetscCall(DMPlexComputeCellGeometryFVM(dm, faces[f], &vol, NULL, normal));
2949       if (PetscAbsReal(normal[2]) < PETSC_SMALL) PetscCall(DMLabelSetValue(edgelabel, faces[f], 1));
2950     }
2951     PetscCall(ISRestoreIndices(faceIS, &faces));
2952     PetscCall(ISDestroy(&faceIS));
2953   }
2954   PetscCall(DMPlexLabelComplete(dm, bdlabel));
2955   PetscCall(DMPlexLabelComplete(dm, edgelabel));
2956   PetscFunctionReturn(PETSC_SUCCESS);
2957 }
2958 
2959 /*@
2960   DMPlexCreateHexCylinderMesh - Creates a mesh on the tensor product of the unit interval with the circle (cylinder) using hexahedra.
2961 
2962   Collective
2963 
2964   Input Parameters:
2965 + comm      - The communicator for the `DM` object
2966 . periodicZ - The boundary type for the Z direction
2967 - Nr        - The number of refinements to carry out
2968 
2969   Output Parameter:
2970 . dm - The `DM` object
2971 
2972   Level: beginner
2973 
2974   Note:
2975   Here is the output numbering looking from the bottom of the cylinder\:
2976 .vb
2977        17-----14
2978         |     |
2979         |  2  |
2980         |     |
2981  17-----8-----7-----14
2982   |     |     |     |
2983   |  3  |  0  |  1  |
2984   |     |     |     |
2985  19-----5-----6-----13
2986         |     |
2987         |  4  |
2988         |     |
2989        19-----13
2990 
2991  and up through the top
2992 
2993        18-----16
2994         |     |
2995         |  2  |
2996         |     |
2997  18----10----11-----16
2998   |     |     |     |
2999   |  3  |  0  |  1  |
3000   |     |     |     |
3001  20-----9----12-----15
3002         |     |
3003         |  4  |
3004         |     |
3005        20-----15
3006 .ve
3007 
3008 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
3009 @*/
3010 PetscErrorCode DMPlexCreateHexCylinderMesh(MPI_Comm comm, DMBoundaryType periodicZ, PetscInt Nr, DM *dm)
3011 {
3012   PetscFunctionBegin;
3013   PetscAssertPointer(dm, 4);
3014   PetscCall(DMCreate(comm, dm));
3015   PetscCall(DMSetType(*dm, DMPLEX));
3016   PetscCall(DMPlexCreateHexCylinderMesh_Internal(*dm, periodicZ, Nr));
3017   PetscFunctionReturn(PETSC_SUCCESS);
3018 }
3019 
3020 static PetscErrorCode DMPlexCreateWedgeCylinderMesh_Internal(DM dm, PetscInt n, PetscBool interpolate)
3021 {
3022   const PetscInt dim = 3;
3023   PetscInt       numCells, numVertices, v;
3024   PetscMPIInt    rank;
3025 
3026   PetscFunctionBegin;
3027   PetscCheck(n >= 0, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of wedges %" PetscInt_FMT " cannot be negative", n);
3028   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
3029   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
3030   PetscCall(DMSetDimension(dm, dim));
3031   /* Must create the celltype label here so that we do not automatically try to compute the types */
3032   PetscCall(DMCreateLabel(dm, "celltype"));
3033   /* Create topology */
3034   {
3035     PetscInt cone[6], c;
3036 
3037     numCells    = rank == 0 ? n : 0;
3038     numVertices = rank == 0 ? 2 * (n + 1) : 0;
3039     PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
3040     for (c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 6));
3041     PetscCall(DMSetUp(dm));
3042     for (c = 0; c < numCells; c++) {
3043       cone[0] = c + n * 1;
3044       cone[1] = (c + 1) % n + n * 1;
3045       cone[2] = 0 + 3 * n;
3046       cone[3] = c + n * 2;
3047       cone[4] = (c + 1) % n + n * 2;
3048       cone[5] = 1 + 3 * n;
3049       PetscCall(DMPlexSetCone(dm, c, cone));
3050       PetscCall(DMPlexSetCellType(dm, c, DM_POLYTOPE_TRI_PRISM_TENSOR));
3051     }
3052     PetscCall(DMPlexSymmetrize(dm));
3053     PetscCall(DMPlexStratify(dm));
3054   }
3055   for (v = numCells; v < numCells + numVertices; ++v) PetscCall(DMPlexSetCellType(dm, v, DM_POLYTOPE_POINT));
3056   /* Create cylinder geometry */
3057   {
3058     Vec          coordinates;
3059     PetscSection coordSection;
3060     PetscScalar *coords;
3061     PetscInt     coordSize, c;
3062 
3063     /* Build coordinates */
3064     PetscCall(DMGetCoordinateSection(dm, &coordSection));
3065     PetscCall(PetscSectionSetNumFields(coordSection, 1));
3066     PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dim));
3067     PetscCall(PetscSectionSetChart(coordSection, numCells, numCells + numVertices));
3068     for (v = numCells; v < numCells + numVertices; ++v) {
3069       PetscCall(PetscSectionSetDof(coordSection, v, dim));
3070       PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dim));
3071     }
3072     PetscCall(PetscSectionSetUp(coordSection));
3073     PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
3074     PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
3075     PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
3076     PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
3077     PetscCall(VecSetBlockSize(coordinates, dim));
3078     PetscCall(VecSetType(coordinates, VECSTANDARD));
3079     PetscCall(VecGetArray(coordinates, &coords));
3080     for (c = 0; c < numCells; c++) {
3081       coords[(c + 0 * n) * dim + 0] = PetscCosReal(2.0 * c * PETSC_PI / n);
3082       coords[(c + 0 * n) * dim + 1] = PetscSinReal(2.0 * c * PETSC_PI / n);
3083       coords[(c + 0 * n) * dim + 2] = 1.0;
3084       coords[(c + 1 * n) * dim + 0] = PetscCosReal(2.0 * c * PETSC_PI / n);
3085       coords[(c + 1 * n) * dim + 1] = PetscSinReal(2.0 * c * PETSC_PI / n);
3086       coords[(c + 1 * n) * dim + 2] = 0.0;
3087     }
3088     if (rank == 0) {
3089       coords[(2 * n + 0) * dim + 0] = 0.0;
3090       coords[(2 * n + 0) * dim + 1] = 0.0;
3091       coords[(2 * n + 0) * dim + 2] = 1.0;
3092       coords[(2 * n + 1) * dim + 0] = 0.0;
3093       coords[(2 * n + 1) * dim + 1] = 0.0;
3094       coords[(2 * n + 1) * dim + 2] = 0.0;
3095     }
3096     PetscCall(VecRestoreArray(coordinates, &coords));
3097     PetscCall(DMSetCoordinatesLocal(dm, coordinates));
3098     PetscCall(VecDestroy(&coordinates));
3099   }
3100   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
3101   /* Interpolate */
3102   if (interpolate) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
3103   PetscFunctionReturn(PETSC_SUCCESS);
3104 }
3105 
3106 /*@
3107   DMPlexCreateWedgeCylinderMesh - Creates a mesh on the tensor product of the unit interval with the circle (cylinder) using wedges.
3108 
3109   Collective
3110 
3111   Input Parameters:
3112 + comm        - The communicator for the `DM` object
3113 . n           - The number of wedges around the origin
3114 - interpolate - Create edges and faces
3115 
3116   Output Parameter:
3117 . dm - The `DM` object
3118 
3119   Level: beginner
3120 
3121 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateHexCylinderMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
3122 @*/
3123 PetscErrorCode DMPlexCreateWedgeCylinderMesh(MPI_Comm comm, PetscInt n, PetscBool interpolate, DM *dm)
3124 {
3125   PetscFunctionBegin;
3126   PetscAssertPointer(dm, 4);
3127   PetscCall(DMCreate(comm, dm));
3128   PetscCall(DMSetType(*dm, DMPLEX));
3129   PetscCall(DMPlexCreateWedgeCylinderMesh_Internal(*dm, n, interpolate));
3130   PetscFunctionReturn(PETSC_SUCCESS);
3131 }
3132 
3133 static inline PetscReal DiffNormReal(PetscInt dim, const PetscReal x[], const PetscReal y[])
3134 {
3135   PetscReal prod = 0.0;
3136   PetscInt  i;
3137   for (i = 0; i < dim; ++i) prod += PetscSqr(x[i] - y[i]);
3138   return PetscSqrtReal(prod);
3139 }
3140 
3141 static inline PetscReal DotReal(PetscInt dim, const PetscReal x[], const PetscReal y[])
3142 {
3143   PetscReal prod = 0.0;
3144   PetscInt  i;
3145   for (i = 0; i < dim; ++i) prod += x[i] * y[i];
3146   return prod;
3147 }
3148 
3149 /* The first constant is the sphere radius */
3150 static void snapToSphere(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
3151 {
3152   PetscReal r     = PetscRealPart(constants[0]);
3153   PetscReal norm2 = 0.0, fac;
3154   PetscInt  n     = uOff[1] - uOff[0], d;
3155 
3156   for (d = 0; d < n; ++d) norm2 += PetscSqr(PetscRealPart(u[d]));
3157   fac = r / PetscSqrtReal(norm2);
3158   for (d = 0; d < n; ++d) f0[d] = u[d] * fac;
3159 }
3160 
3161 static PetscErrorCode DMPlexCreateSphereMesh_Internal(DM dm, PetscInt dim, PetscBool simplex, PetscReal R)
3162 {
3163   const PetscInt embedDim = dim + 1;
3164   PetscSection   coordSection;
3165   Vec            coordinates;
3166   PetscScalar   *coords;
3167   PetscReal     *coordsIn;
3168   PetscInt       numCells, numEdges, numVerts = 0, firstVertex = 0, v, firstEdge, coordSize, d, e;
3169   PetscMPIInt    rank;
3170 
3171   PetscFunctionBegin;
3172   PetscValidLogicalCollectiveBool(dm, simplex, 3);
3173   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
3174   PetscCall(DMSetDimension(dm, dim));
3175   PetscCall(DMSetCoordinateDim(dm, dim + 1));
3176   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
3177   switch (dim) {
3178   case 1:
3179     numCells = 16;
3180     numVerts = numCells;
3181 
3182     // Build Topology
3183     PetscCall(DMPlexSetChart(dm, 0, numCells + numVerts));
3184     for (PetscInt c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, embedDim));
3185     PetscCall(DMSetUp(dm));
3186     for (PetscInt c = 0; c < numCells; ++c) {
3187       PetscInt cone[2];
3188 
3189       cone[0] = c + numCells;
3190       cone[1] = (c + 1) % numVerts + numCells;
3191       PetscCall(DMPlexSetCone(dm, c, cone));
3192     }
3193     PetscCall(DMPlexSymmetrize(dm));
3194     PetscCall(DMPlexStratify(dm));
3195     PetscCall(PetscMalloc1(numVerts * embedDim, &coordsIn));
3196     for (PetscInt v = 0; v < numVerts; ++v) {
3197       const PetscReal rad = 2. * PETSC_PI * v / numVerts;
3198 
3199       coordsIn[v * embedDim + 0] = PetscCosReal(rad);
3200       coordsIn[v * embedDim + 1] = PetscSinReal(rad);
3201     }
3202     break;
3203   case 2:
3204     if (simplex) {
3205       const PetscReal radius    = PetscSqrtReal(1 + PETSC_PHI * PETSC_PHI) / (1.0 + PETSC_PHI);
3206       const PetscReal edgeLen   = 2.0 / (1.0 + PETSC_PHI) * (R / radius);
3207       const PetscInt  degree    = 5;
3208       PetscReal       vertex[3] = {0.0, 1.0 / (1.0 + PETSC_PHI), PETSC_PHI / (1.0 + PETSC_PHI)};
3209       PetscInt        s[3]      = {1, 1, 1};
3210       PetscInt        cone[3];
3211       PetscInt       *graph;
3212 
3213       vertex[0] *= R / radius;
3214       vertex[1] *= R / radius;
3215       vertex[2] *= R / radius;
3216       numCells    = rank == 0 ? 20 : 0;
3217       numVerts    = rank == 0 ? 12 : 0;
3218       firstVertex = numCells;
3219       /* Use icosahedron, which for a R-sphere has coordinates which are all cyclic permutations of
3220 
3221            (0, \pm 1/\phi+1, \pm \phi/\phi+1)
3222 
3223          where \phi^2 - \phi - 1 = 0, meaning \phi is the golden ratio \frac{1 + \sqrt{5}}{2}. The edge
3224          length is then given by 2/(1+\phi) = 2 * 0.38197 = 0.76393.
3225       */
3226       /* Construct vertices */
3227       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
3228       if (rank == 0) {
3229         for (PetscInt p = 0, i = 0; p < embedDim; ++p) {
3230           for (s[1] = -1; s[1] < 2; s[1] += 2) {
3231             for (s[2] = -1; s[2] < 2; s[2] += 2) {
3232               for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[(d + p) % embedDim] * vertex[(d + p) % embedDim];
3233               ++i;
3234             }
3235           }
3236         }
3237       }
3238       /* Construct graph */
3239       PetscCall(PetscCalloc1(numVerts * numVerts, &graph));
3240       for (PetscInt i = 0; i < numVerts; ++i) {
3241         PetscInt k = 0;
3242         for (PetscInt j = 0; j < numVerts; ++j) {
3243           if (PetscAbsReal(DiffNormReal(embedDim, &coordsIn[i * embedDim], &coordsIn[j * embedDim]) - edgeLen) < PETSC_SMALL) {
3244             graph[i * numVerts + j] = 1;
3245             ++k;
3246           }
3247         }
3248         PetscCheck(k == degree, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid icosahedron, vertex %" PetscInt_FMT " degree %" PetscInt_FMT " != %" PetscInt_FMT, i, k, degree);
3249       }
3250       /* Build Topology */
3251       PetscCall(DMPlexSetChart(dm, 0, numCells + numVerts));
3252       for (PetscInt c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, embedDim));
3253       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
3254       /* Cells */
3255       for (PetscInt i = 0, c = 0; i < numVerts; ++i) {
3256         for (PetscInt j = 0; j < i; ++j) {
3257           for (PetscInt k = 0; k < j; ++k) {
3258             if (graph[i * numVerts + j] && graph[j * numVerts + k] && graph[k * numVerts + i]) {
3259               cone[0] = firstVertex + i;
3260               cone[1] = firstVertex + j;
3261               cone[2] = firstVertex + k;
3262               /* Check orientation */
3263               {
3264                 const PetscInt epsilon[3][3][3] = {
3265                   {{0, 0, 0},  {0, 0, 1},  {0, -1, 0}},
3266                   {{0, 0, -1}, {0, 0, 0},  {1, 0, 0} },
3267                   {{0, 1, 0},  {-1, 0, 0}, {0, 0, 0} }
3268                 };
3269                 PetscReal normal[3];
3270                 PetscInt  e, f;
3271 
3272                 for (d = 0; d < embedDim; ++d) {
3273                   normal[d] = 0.0;
3274                   for (e = 0; e < embedDim; ++e) {
3275                     for (f = 0; f < embedDim; ++f) normal[d] += epsilon[d][e][f] * (coordsIn[j * embedDim + e] - coordsIn[i * embedDim + e]) * (coordsIn[k * embedDim + f] - coordsIn[i * embedDim + f]);
3276                   }
3277                 }
3278                 if (DotReal(embedDim, normal, &coordsIn[i * embedDim]) < 0) {
3279                   PetscInt tmp = cone[1];
3280                   cone[1]      = cone[2];
3281                   cone[2]      = tmp;
3282                 }
3283               }
3284               PetscCall(DMPlexSetCone(dm, c++, cone));
3285             }
3286           }
3287         }
3288       }
3289       PetscCall(DMPlexSymmetrize(dm));
3290       PetscCall(DMPlexStratify(dm));
3291       PetscCall(PetscFree(graph));
3292     } else {
3293       /*
3294         12-21--13
3295          |     |
3296         25  4  24
3297          |     |
3298   12-25--9-16--8-24--13
3299    |     |     |     |
3300   23  5 17  0 15  3  22
3301    |     |     |     |
3302   10-20--6-14--7-19--11
3303          |     |
3304         20  1  19
3305          |     |
3306         10-18--11
3307          |     |
3308         23  2  22
3309          |     |
3310         12-21--13
3311        */
3312       PetscInt cone[4], ornt[4];
3313 
3314       numCells    = rank == 0 ? 6 : 0;
3315       numEdges    = rank == 0 ? 12 : 0;
3316       numVerts    = rank == 0 ? 8 : 0;
3317       firstVertex = numCells;
3318       firstEdge   = numCells + numVerts;
3319       /* Build Topology */
3320       PetscCall(DMPlexSetChart(dm, 0, numCells + numEdges + numVerts));
3321       for (PetscInt c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, 4));
3322       for (e = firstEdge; e < firstEdge + numEdges; ++e) PetscCall(DMPlexSetConeSize(dm, e, 2));
3323       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
3324       if (rank == 0) {
3325         /* Cell 0 */
3326         cone[0] = 14;
3327         cone[1] = 15;
3328         cone[2] = 16;
3329         cone[3] = 17;
3330         PetscCall(DMPlexSetCone(dm, 0, cone));
3331         ornt[0] = 0;
3332         ornt[1] = 0;
3333         ornt[2] = 0;
3334         ornt[3] = 0;
3335         PetscCall(DMPlexSetConeOrientation(dm, 0, ornt));
3336         /* Cell 1 */
3337         cone[0] = 18;
3338         cone[1] = 19;
3339         cone[2] = 14;
3340         cone[3] = 20;
3341         PetscCall(DMPlexSetCone(dm, 1, cone));
3342         ornt[0] = 0;
3343         ornt[1] = 0;
3344         ornt[2] = -1;
3345         ornt[3] = 0;
3346         PetscCall(DMPlexSetConeOrientation(dm, 1, ornt));
3347         /* Cell 2 */
3348         cone[0] = 21;
3349         cone[1] = 22;
3350         cone[2] = 18;
3351         cone[3] = 23;
3352         PetscCall(DMPlexSetCone(dm, 2, cone));
3353         ornt[0] = 0;
3354         ornt[1] = 0;
3355         ornt[2] = -1;
3356         ornt[3] = 0;
3357         PetscCall(DMPlexSetConeOrientation(dm, 2, ornt));
3358         /* Cell 3 */
3359         cone[0] = 19;
3360         cone[1] = 22;
3361         cone[2] = 24;
3362         cone[3] = 15;
3363         PetscCall(DMPlexSetCone(dm, 3, cone));
3364         ornt[0] = -1;
3365         ornt[1] = -1;
3366         ornt[2] = 0;
3367         ornt[3] = -1;
3368         PetscCall(DMPlexSetConeOrientation(dm, 3, ornt));
3369         /* Cell 4 */
3370         cone[0] = 16;
3371         cone[1] = 24;
3372         cone[2] = 21;
3373         cone[3] = 25;
3374         PetscCall(DMPlexSetCone(dm, 4, cone));
3375         ornt[0] = -1;
3376         ornt[1] = -1;
3377         ornt[2] = -1;
3378         ornt[3] = 0;
3379         PetscCall(DMPlexSetConeOrientation(dm, 4, ornt));
3380         /* Cell 5 */
3381         cone[0] = 20;
3382         cone[1] = 17;
3383         cone[2] = 25;
3384         cone[3] = 23;
3385         PetscCall(DMPlexSetCone(dm, 5, cone));
3386         ornt[0] = -1;
3387         ornt[1] = -1;
3388         ornt[2] = -1;
3389         ornt[3] = -1;
3390         PetscCall(DMPlexSetConeOrientation(dm, 5, ornt));
3391         /* Edges */
3392         cone[0] = 6;
3393         cone[1] = 7;
3394         PetscCall(DMPlexSetCone(dm, 14, cone));
3395         cone[0] = 7;
3396         cone[1] = 8;
3397         PetscCall(DMPlexSetCone(dm, 15, cone));
3398         cone[0] = 8;
3399         cone[1] = 9;
3400         PetscCall(DMPlexSetCone(dm, 16, cone));
3401         cone[0] = 9;
3402         cone[1] = 6;
3403         PetscCall(DMPlexSetCone(dm, 17, cone));
3404         cone[0] = 10;
3405         cone[1] = 11;
3406         PetscCall(DMPlexSetCone(dm, 18, cone));
3407         cone[0] = 11;
3408         cone[1] = 7;
3409         PetscCall(DMPlexSetCone(dm, 19, cone));
3410         cone[0] = 6;
3411         cone[1] = 10;
3412         PetscCall(DMPlexSetCone(dm, 20, cone));
3413         cone[0] = 12;
3414         cone[1] = 13;
3415         PetscCall(DMPlexSetCone(dm, 21, cone));
3416         cone[0] = 13;
3417         cone[1] = 11;
3418         PetscCall(DMPlexSetCone(dm, 22, cone));
3419         cone[0] = 10;
3420         cone[1] = 12;
3421         PetscCall(DMPlexSetCone(dm, 23, cone));
3422         cone[0] = 13;
3423         cone[1] = 8;
3424         PetscCall(DMPlexSetCone(dm, 24, cone));
3425         cone[0] = 12;
3426         cone[1] = 9;
3427         PetscCall(DMPlexSetCone(dm, 25, cone));
3428       }
3429       PetscCall(DMPlexSymmetrize(dm));
3430       PetscCall(DMPlexStratify(dm));
3431       /* Build coordinates */
3432       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
3433       if (rank == 0) {
3434         coordsIn[0 * embedDim + 0] = -R;
3435         coordsIn[0 * embedDim + 1] = R;
3436         coordsIn[0 * embedDim + 2] = -R;
3437         coordsIn[1 * embedDim + 0] = R;
3438         coordsIn[1 * embedDim + 1] = R;
3439         coordsIn[1 * embedDim + 2] = -R;
3440         coordsIn[2 * embedDim + 0] = R;
3441         coordsIn[2 * embedDim + 1] = -R;
3442         coordsIn[2 * embedDim + 2] = -R;
3443         coordsIn[3 * embedDim + 0] = -R;
3444         coordsIn[3 * embedDim + 1] = -R;
3445         coordsIn[3 * embedDim + 2] = -R;
3446         coordsIn[4 * embedDim + 0] = -R;
3447         coordsIn[4 * embedDim + 1] = R;
3448         coordsIn[4 * embedDim + 2] = R;
3449         coordsIn[5 * embedDim + 0] = R;
3450         coordsIn[5 * embedDim + 1] = R;
3451         coordsIn[5 * embedDim + 2] = R;
3452         coordsIn[6 * embedDim + 0] = -R;
3453         coordsIn[6 * embedDim + 1] = -R;
3454         coordsIn[6 * embedDim + 2] = R;
3455         coordsIn[7 * embedDim + 0] = R;
3456         coordsIn[7 * embedDim + 1] = -R;
3457         coordsIn[7 * embedDim + 2] = R;
3458       }
3459     }
3460     break;
3461   case 3:
3462     if (simplex) {
3463       const PetscReal edgeLen         = 1.0 / PETSC_PHI;
3464       PetscReal       vertexA[4]      = {0.5, 0.5, 0.5, 0.5};
3465       PetscReal       vertexB[4]      = {1.0, 0.0, 0.0, 0.0};
3466       PetscReal       vertexC[4]      = {0.5, 0.5 * PETSC_PHI, 0.5 / PETSC_PHI, 0.0};
3467       const PetscInt  degree          = 12;
3468       PetscInt        s[4]            = {1, 1, 1};
3469       PetscInt        evenPerm[12][4] = {
3470         {0, 1, 2, 3},
3471         {0, 2, 3, 1},
3472         {0, 3, 1, 2},
3473         {1, 0, 3, 2},
3474         {1, 2, 0, 3},
3475         {1, 3, 2, 0},
3476         {2, 0, 1, 3},
3477         {2, 1, 3, 0},
3478         {2, 3, 0, 1},
3479         {3, 0, 2, 1},
3480         {3, 1, 0, 2},
3481         {3, 2, 1, 0}
3482       };
3483       PetscInt  cone[4];
3484       PetscInt *graph, p, i, j, k, l;
3485 
3486       vertexA[0] *= R;
3487       vertexA[1] *= R;
3488       vertexA[2] *= R;
3489       vertexA[3] *= R;
3490       vertexB[0] *= R;
3491       vertexB[1] *= R;
3492       vertexB[2] *= R;
3493       vertexB[3] *= R;
3494       vertexC[0] *= R;
3495       vertexC[1] *= R;
3496       vertexC[2] *= R;
3497       vertexC[3] *= R;
3498       numCells    = rank == 0 ? 600 : 0;
3499       numVerts    = rank == 0 ? 120 : 0;
3500       firstVertex = numCells;
3501       /* Use the 600-cell, which for a unit sphere has coordinates which are
3502 
3503            1/2 (\pm 1, \pm 1,    \pm 1, \pm 1)                          16
3504                (\pm 1,    0,       0,      0)  all cyclic permutations   8
3505            1/2 (\pm 1, \pm phi, \pm 1/phi, 0)  all even permutations    96
3506 
3507          where \phi^2 - \phi - 1 = 0, meaning \phi is the golden ratio \frac{1 + \sqrt{5}}{2}. The edge
3508          length is then given by 1/\phi = 0.61803.
3509 
3510          http://buzzard.pugetsound.edu/sage-practice/ch03s03.html
3511          http://mathworld.wolfram.com/600-Cell.html
3512       */
3513       /* Construct vertices */
3514       PetscCall(PetscCalloc1(numVerts * embedDim, &coordsIn));
3515       i = 0;
3516       if (rank == 0) {
3517         for (s[0] = -1; s[0] < 2; s[0] += 2) {
3518           for (s[1] = -1; s[1] < 2; s[1] += 2) {
3519             for (s[2] = -1; s[2] < 2; s[2] += 2) {
3520               for (s[3] = -1; s[3] < 2; s[3] += 2) {
3521                 for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[d] * vertexA[d];
3522                 ++i;
3523               }
3524             }
3525           }
3526         }
3527         for (p = 0; p < embedDim; ++p) {
3528           s[1] = s[2] = s[3] = 1;
3529           for (s[0] = -1; s[0] < 2; s[0] += 2) {
3530             for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[(d + p) % embedDim] * vertexB[(d + p) % embedDim];
3531             ++i;
3532           }
3533         }
3534         for (p = 0; p < 12; ++p) {
3535           s[3] = 1;
3536           for (s[0] = -1; s[0] < 2; s[0] += 2) {
3537             for (s[1] = -1; s[1] < 2; s[1] += 2) {
3538               for (s[2] = -1; s[2] < 2; s[2] += 2) {
3539                 for (d = 0; d < embedDim; ++d) coordsIn[i * embedDim + d] = s[evenPerm[p][d]] * vertexC[evenPerm[p][d]];
3540                 ++i;
3541               }
3542             }
3543           }
3544         }
3545       }
3546       PetscCheck(i == numVerts, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid 600-cell, vertices %" PetscInt_FMT " != %" PetscInt_FMT, i, numVerts);
3547       /* Construct graph */
3548       PetscCall(PetscCalloc1(numVerts * numVerts, &graph));
3549       for (i = 0; i < numVerts; ++i) {
3550         for (j = 0, k = 0; j < numVerts; ++j) {
3551           if (PetscAbsReal(DiffNormReal(embedDim, &coordsIn[i * embedDim], &coordsIn[j * embedDim]) - edgeLen) < PETSC_SMALL) {
3552             graph[i * numVerts + j] = 1;
3553             ++k;
3554           }
3555         }
3556         PetscCheck(k == degree, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid 600-cell, vertex %" PetscInt_FMT " degree %" PetscInt_FMT " != %" PetscInt_FMT, i, k, degree);
3557       }
3558       /* Build Topology */
3559       PetscCall(DMPlexSetChart(dm, 0, numCells + numVerts));
3560       for (PetscInt c = 0; c < numCells; c++) PetscCall(DMPlexSetConeSize(dm, c, embedDim));
3561       PetscCall(DMSetUp(dm)); /* Allocate space for cones */
3562       /* Cells */
3563       if (rank == 0) {
3564         for (PetscInt i = 0, c = 0; i < numVerts; ++i) {
3565           for (j = 0; j < i; ++j) {
3566             for (k = 0; k < j; ++k) {
3567               for (l = 0; l < k; ++l) {
3568                 if (graph[i * numVerts + j] && graph[j * numVerts + k] && graph[k * numVerts + i] && graph[l * numVerts + i] && graph[l * numVerts + j] && graph[l * numVerts + k]) {
3569                   cone[0] = firstVertex + i;
3570                   cone[1] = firstVertex + j;
3571                   cone[2] = firstVertex + k;
3572                   cone[3] = firstVertex + l;
3573                   /* Check orientation: https://ef.gy/linear-algebra:normal-vectors-in-higher-dimensional-spaces */
3574                   {
3575                     const PetscInt epsilon[4][4][4][4] = {
3576                       {{{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}},  {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 1}, {0, 0, -1, 0}}, {{0, 0, 0, 0}, {0, 0, 0, -1}, {0, 0, 0, 0}, {0, 1, 0, 0}}, {{0, 0, 0, 0}, {0, 0, 1, 0}, {0, -1, 0, 0}, {0, 0, 0, 0}}},
3577 
3578                       {{{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, -1}, {0, 0, 1, 0}}, {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}},  {{0, 0, 0, 1}, {0, 0, 0, 0}, {0, 0, 0, 0}, {-1, 0, 0, 0}}, {{0, 0, -1, 0}, {0, 0, 0, 0}, {1, 0, 0, 0}, {0, 0, 0, 0}}},
3579 
3580                       {{{0, 0, 0, 0}, {0, 0, 0, 1}, {0, 0, 0, 0}, {0, -1, 0, 0}}, {{0, 0, 0, -1}, {0, 0, 0, 0}, {0, 0, 0, 0}, {1, 0, 0, 0}}, {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}},  {{0, 1, 0, 0}, {-1, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}}},
3581 
3582                       {{{0, 0, 0, 0}, {0, 0, -1, 0}, {0, 1, 0, 0}, {0, 0, 0, 0}}, {{0, 0, 1, 0}, {0, 0, 0, 0}, {-1, 0, 0, 0}, {0, 0, 0, 0}}, {{0, -1, 0, 0}, {1, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}}, {{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}} }
3583                     };
3584                     PetscReal normal[4];
3585                     PetscInt  e, f, g;
3586 
3587                     for (d = 0; d < embedDim; ++d) {
3588                       normal[d] = 0.0;
3589                       for (e = 0; e < embedDim; ++e) {
3590                         for (f = 0; f < embedDim; ++f) {
3591                           for (g = 0; g < embedDim; ++g) {
3592                             normal[d] += epsilon[d][e][f][g] * (coordsIn[j * embedDim + e] - coordsIn[i * embedDim + e]) * (coordsIn[k * embedDim + f] - coordsIn[i * embedDim + f]) * (coordsIn[l * embedDim + f] - coordsIn[i * embedDim + f]);
3593                           }
3594                         }
3595                       }
3596                     }
3597                     if (DotReal(embedDim, normal, &coordsIn[i * embedDim]) < 0) {
3598                       PetscInt tmp = cone[1];
3599                       cone[1]      = cone[2];
3600                       cone[2]      = tmp;
3601                     }
3602                   }
3603                   PetscCall(DMPlexSetCone(dm, c++, cone));
3604                 }
3605               }
3606             }
3607           }
3608         }
3609       }
3610       PetscCall(DMPlexSymmetrize(dm));
3611       PetscCall(DMPlexStratify(dm));
3612       PetscCall(PetscFree(graph));
3613     }
3614     break;
3615   default:
3616     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Unsupported dimension for sphere: %" PetscInt_FMT, dim);
3617   }
3618   /* Create coordinates */
3619   PetscCall(DMGetCoordinateSection(dm, &coordSection));
3620   PetscCall(PetscSectionSetNumFields(coordSection, 1));
3621   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, embedDim));
3622   PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numVerts));
3623   for (v = firstVertex; v < firstVertex + numVerts; ++v) {
3624     PetscCall(PetscSectionSetDof(coordSection, v, embedDim));
3625     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, embedDim));
3626   }
3627   PetscCall(PetscSectionSetUp(coordSection));
3628   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
3629   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
3630   PetscCall(VecSetBlockSize(coordinates, embedDim));
3631   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
3632   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
3633   PetscCall(VecSetType(coordinates, VECSTANDARD));
3634   PetscCall(VecGetArray(coordinates, &coords));
3635   for (v = 0; v < numVerts; ++v)
3636     for (d = 0; d < embedDim; ++d) coords[v * embedDim + d] = coordsIn[v * embedDim + d];
3637   PetscCall(VecRestoreArray(coordinates, &coords));
3638   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
3639   PetscCall(VecDestroy(&coordinates));
3640   PetscCall(PetscFree(coordsIn));
3641   {
3642     DM          cdm;
3643     PetscDS     cds;
3644     PetscScalar c = R;
3645 
3646     PetscCall(DMPlexCreateCoordinateSpace(dm, 1, PETSC_TRUE, snapToSphere));
3647     PetscCall(DMGetCoordinateDM(dm, &cdm));
3648     PetscCall(DMGetDS(cdm, &cds));
3649     PetscCall(PetscDSSetConstants(cds, 1, &c));
3650   }
3651   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
3652   /* Wait for coordinate creation before doing in-place modification */
3653   if (simplex) PetscCall(DMPlexInterpolateInPlace_Internal(dm));
3654   PetscFunctionReturn(PETSC_SUCCESS);
3655 }
3656 
3657 typedef void (*TPSEvaluateFunc)(const PetscReal[], PetscReal *, PetscReal[], PetscReal (*)[3]);
3658 
3659 /*
3660  The Schwarz P implicit surface is
3661 
3662      f(x) = cos(x0) + cos(x1) + cos(x2) = 0
3663 */
3664 static void TPSEvaluate_SchwarzP(const PetscReal y[3], PetscReal *f, PetscReal grad[], PetscReal (*hess)[3])
3665 {
3666   PetscReal c[3] = {PetscCosReal(y[0] * PETSC_PI), PetscCosReal(y[1] * PETSC_PI), PetscCosReal(y[2] * PETSC_PI)};
3667   PetscReal g[3] = {-PetscSinReal(y[0] * PETSC_PI), -PetscSinReal(y[1] * PETSC_PI), -PetscSinReal(y[2] * PETSC_PI)};
3668   f[0]           = c[0] + c[1] + c[2];
3669   for (PetscInt i = 0; i < 3; i++) {
3670     grad[i] = PETSC_PI * g[i];
3671     for (PetscInt j = 0; j < 3; j++) hess[i][j] = (i == j) ? -PetscSqr(PETSC_PI) * c[i] : 0.;
3672   }
3673 }
3674 
3675 // u[] is a tentative normal on input. Replace with the implicit function gradient in the same direction
3676 static PetscErrorCode TPSExtrudeNormalFunc_SchwarzP(PetscInt dim, PetscReal time, const PetscReal x[], PetscInt r, PetscScalar u[], void *ctx)
3677 {
3678   for (PetscInt i = 0; i < 3; i++) u[i] = -PETSC_PI * PetscSinReal(x[i] * PETSC_PI);
3679   return PETSC_SUCCESS;
3680 }
3681 
3682 /*
3683  The Gyroid implicit surface is
3684 
3685  f(x,y,z) = sin(pi * x) * cos (pi * (y + 1/2))  + sin(pi * (y + 1/2)) * cos(pi * (z + 1/4)) + sin(pi * (z + 1/4)) * cos(pi * x)
3686 
3687 */
3688 static void TPSEvaluate_Gyroid(const PetscReal y[3], PetscReal *f, PetscReal grad[], PetscReal (*hess)[3])
3689 {
3690   PetscReal s[3] = {PetscSinReal(PETSC_PI * y[0]), PetscSinReal(PETSC_PI * (y[1] + .5)), PetscSinReal(PETSC_PI * (y[2] + .25))};
3691   PetscReal c[3] = {PetscCosReal(PETSC_PI * y[0]), PetscCosReal(PETSC_PI * (y[1] + .5)), PetscCosReal(PETSC_PI * (y[2] + .25))};
3692   f[0]           = s[0] * c[1] + s[1] * c[2] + s[2] * c[0];
3693   grad[0]        = PETSC_PI * (c[0] * c[1] - s[2] * s[0]);
3694   grad[1]        = PETSC_PI * (c[1] * c[2] - s[0] * s[1]);
3695   grad[2]        = PETSC_PI * (c[2] * c[0] - s[1] * s[2]);
3696   hess[0][0]     = -PetscSqr(PETSC_PI) * (s[0] * c[1] + s[2] * c[0]);
3697   hess[0][1]     = -PetscSqr(PETSC_PI) * (c[0] * s[1]);
3698   hess[0][2]     = -PetscSqr(PETSC_PI) * (c[2] * s[0]);
3699   hess[1][0]     = -PetscSqr(PETSC_PI) * (s[1] * c[2] + s[0] * c[1]);
3700   hess[1][1]     = -PetscSqr(PETSC_PI) * (c[1] * s[2]);
3701   hess[2][2]     = -PetscSqr(PETSC_PI) * (c[0] * s[1]);
3702   hess[2][0]     = -PetscSqr(PETSC_PI) * (s[2] * c[0] + s[1] * c[2]);
3703   hess[2][1]     = -PetscSqr(PETSC_PI) * (c[2] * s[0]);
3704   hess[2][2]     = -PetscSqr(PETSC_PI) * (c[1] * s[2]);
3705 }
3706 
3707 // u[] is a tentative normal on input. Replace with the implicit function gradient in the same direction
3708 static PetscErrorCode TPSExtrudeNormalFunc_Gyroid(PetscInt dim, PetscReal time, const PetscReal x[], PetscInt r, PetscScalar u[], void *ctx)
3709 {
3710   PetscReal s[3] = {PetscSinReal(PETSC_PI * x[0]), PetscSinReal(PETSC_PI * (x[1] + .5)), PetscSinReal(PETSC_PI * (x[2] + .25))};
3711   PetscReal c[3] = {PetscCosReal(PETSC_PI * x[0]), PetscCosReal(PETSC_PI * (x[1] + .5)), PetscCosReal(PETSC_PI * (x[2] + .25))};
3712   u[0]           = PETSC_PI * (c[0] * c[1] - s[2] * s[0]);
3713   u[1]           = PETSC_PI * (c[1] * c[2] - s[0] * s[1]);
3714   u[2]           = PETSC_PI * (c[2] * c[0] - s[1] * s[2]);
3715   return PETSC_SUCCESS;
3716 }
3717 
3718 /*
3719    We wish to solve
3720 
3721          min_y || y - x ||^2  subject to f(y) = 0
3722 
3723    Let g(y) = grad(f).  The minimization problem is equivalent to asking to satisfy
3724    f(y) = 0 and (y-x) is parallel to g(y).  We do this by using Householder QR to obtain a basis for the
3725    tangent space and ask for both components in the tangent space to be zero.
3726 
3727    Take g to be a column vector and compute the "full QR" factorization Q R = g,
3728    where Q = I - 2 n n^T is a symmetric orthogonal matrix.
3729    The first column of Q is parallel to g so the remaining two columns span the null space.
3730    Let Qn = Q[:,1:] be those remaining columns.  Then Qn Qn^T is an orthogonal projector into the tangent space.
3731    Since Q is symmetric, this is equivalent to multiplying by Q and taking the last two entries.
3732    In total, we have a system of 3 equations in 3 unknowns:
3733 
3734      f(y) = 0                       1 equation
3735      Qn^T (y - x) = 0               2 equations
3736 
3737    Here, we compute the residual and Jacobian of this system.
3738 */
3739 static void TPSNearestPointResJac(TPSEvaluateFunc feval, const PetscScalar x[], const PetscScalar y[], PetscScalar res[], PetscScalar J[])
3740 {
3741   PetscReal yreal[3] = {PetscRealPart(y[0]), PetscRealPart(y[1]), PetscRealPart(y[2])};
3742   PetscReal d[3]     = {PetscRealPart(y[0] - x[0]), PetscRealPart(y[1] - x[1]), PetscRealPart(y[2] - x[2])};
3743   PetscReal f, grad[3], n[3], norm, norm_y[3], nd, nd_y[3], sign;
3744   PetscReal n_y[3][3] = {
3745     {0, 0, 0},
3746     {0, 0, 0},
3747     {0, 0, 0}
3748   };
3749 
3750   feval(yreal, &f, grad, n_y);
3751 
3752   for (PetscInt i = 0; i < 3; i++) n[i] = grad[i];
3753   norm = PetscSqrtReal(PetscSqr(n[0]) + PetscSqr(n[1]) + PetscSqr(n[2]));
3754   for (PetscInt i = 0; i < 3; i++) norm_y[i] = 1. / norm * n[i] * n_y[i][i];
3755 
3756   // Define the Householder reflector
3757   sign = n[0] >= 0 ? 1. : -1.;
3758   n[0] += norm * sign;
3759   for (PetscInt i = 0; i < 3; i++) n_y[0][i] += norm_y[i] * sign;
3760 
3761   norm      = PetscSqrtReal(PetscSqr(n[0]) + PetscSqr(n[1]) + PetscSqr(n[2]));
3762   norm_y[0] = 1. / norm * (n[0] * n_y[0][0]);
3763   norm_y[1] = 1. / norm * (n[0] * n_y[0][1] + n[1] * n_y[1][1]);
3764   norm_y[2] = 1. / norm * (n[0] * n_y[0][2] + n[2] * n_y[2][2]);
3765 
3766   for (PetscInt i = 0; i < 3; i++) {
3767     n[i] /= norm;
3768     for (PetscInt j = 0; j < 3; j++) {
3769       // note that n[i] is n_old[i]/norm when executing the code below
3770       n_y[i][j] = n_y[i][j] / norm - n[i] / norm * norm_y[j];
3771     }
3772   }
3773 
3774   nd = n[0] * d[0] + n[1] * d[1] + n[2] * d[2];
3775   for (PetscInt i = 0; i < 3; i++) nd_y[i] = n[i] + n_y[0][i] * d[0] + n_y[1][i] * d[1] + n_y[2][i] * d[2];
3776 
3777   res[0] = f;
3778   res[1] = d[1] - 2 * n[1] * nd;
3779   res[2] = d[2] - 2 * n[2] * nd;
3780   // J[j][i] is J_{ij} (column major)
3781   for (PetscInt j = 0; j < 3; j++) {
3782     J[0 + j * 3] = grad[j];
3783     J[1 + j * 3] = (j == 1) * 1. - 2 * (n_y[1][j] * nd + n[1] * nd_y[j]);
3784     J[2 + j * 3] = (j == 2) * 1. - 2 * (n_y[2][j] * nd + n[2] * nd_y[j]);
3785   }
3786 }
3787 
3788 /*
3789    Project x to the nearest point on the implicit surface using Newton's method.
3790 */
3791 static PetscErrorCode TPSNearestPoint(TPSEvaluateFunc feval, PetscScalar x[])
3792 {
3793   PetscScalar y[3] = {x[0], x[1], x[2]}; // Initial guess
3794 
3795   PetscFunctionBegin;
3796   for (PetscInt iter = 0; iter < 10; iter++) {
3797     PetscScalar res[3], J[9];
3798     PetscReal   resnorm;
3799     TPSNearestPointResJac(feval, x, y, res, J);
3800     resnorm = PetscSqrtReal(PetscSqr(PetscRealPart(res[0])) + PetscSqr(PetscRealPart(res[1])) + PetscSqr(PetscRealPart(res[2])));
3801     if (0) { // Turn on this monitor if you need to confirm quadratic convergence
3802       PetscCall(PetscPrintf(PETSC_COMM_SELF, "[%" PetscInt_FMT "] res [%g %g %g]\n", iter, (double)PetscRealPart(res[0]), (double)PetscRealPart(res[1]), (double)PetscRealPart(res[2])));
3803     }
3804     if (resnorm < PETSC_SMALL) break;
3805 
3806     // Take the Newton step
3807     PetscCall(PetscKernel_A_gets_inverse_A_3(J, 0., PETSC_FALSE, NULL));
3808     PetscKernel_v_gets_v_minus_A_times_w_3(y, J, res);
3809   }
3810   for (PetscInt i = 0; i < 3; i++) x[i] = y[i];
3811   PetscFunctionReturn(PETSC_SUCCESS);
3812 }
3813 
3814 const char *const DMPlexTPSTypes[] = {"SCHWARZ_P", "GYROID", "DMPlexTPSType", "DMPLEX_TPS_", NULL};
3815 
3816 static PetscErrorCode DMPlexCreateTPSMesh_Internal(DM dm, DMPlexTPSType tpstype, const PetscInt extent[], const DMBoundaryType periodic[], PetscBool tps_distribute, PetscInt refinements, PetscInt layers, PetscReal thickness)
3817 {
3818   PetscMPIInt rank;
3819   PetscInt    topoDim = 2, spaceDim = 3, numFaces = 0, numVertices = 0, numEdges = 0;
3820   PetscInt (*edges)[2] = NULL, *edgeSets = NULL;
3821   PetscInt           *cells_flat = NULL;
3822   PetscReal          *vtxCoords  = NULL;
3823   TPSEvaluateFunc     evalFunc   = NULL;
3824   PetscSimplePointFn *normalFunc = NULL;
3825   DMLabel             label;
3826 
3827   PetscFunctionBegin;
3828   PetscCall(PetscLogEventBegin(DMPLEX_Generate, dm, 0, 0, 0));
3829   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
3830   PetscCheck((layers != 0) ^ (thickness == 0.), PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_INCOMP, "Layers %" PetscInt_FMT " must be nonzero iff thickness %g is nonzero", layers, (double)thickness);
3831   switch (tpstype) {
3832   case DMPLEX_TPS_SCHWARZ_P:
3833     PetscCheck(!periodic || (periodic[0] == DM_BOUNDARY_NONE && periodic[1] == DM_BOUNDARY_NONE && periodic[2] == DM_BOUNDARY_NONE), PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Schwarz P does not support periodic meshes");
3834     if (rank == 0) {
3835       PetscInt (*cells)[6][4][4] = NULL; // [junction, junction-face, cell, conn]
3836       PetscInt  Njunctions = 0, Ncuts = 0, Npipes[3], vcount;
3837       PetscReal L = 1;
3838 
3839       Npipes[0]   = (extent[0] + 1) * extent[1] * extent[2];
3840       Npipes[1]   = extent[0] * (extent[1] + 1) * extent[2];
3841       Npipes[2]   = extent[0] * extent[1] * (extent[2] + 1);
3842       Njunctions  = extent[0] * extent[1] * extent[2];
3843       Ncuts       = 2 * (extent[0] * extent[1] + extent[1] * extent[2] + extent[2] * extent[0]);
3844       numVertices = 4 * (Npipes[0] + Npipes[1] + Npipes[2]) + 8 * Njunctions;
3845       PetscCall(PetscMalloc1(3 * numVertices, &vtxCoords));
3846       PetscCall(PetscMalloc1(Njunctions, &cells));
3847       PetscCall(PetscMalloc1(Ncuts * 4, &edges));
3848       PetscCall(PetscMalloc1(Ncuts * 4, &edgeSets));
3849       // x-normal pipes
3850       vcount = 0;
3851       for (PetscInt i = 0; i < extent[0] + 1; i++) {
3852         for (PetscInt j = 0; j < extent[1]; j++) {
3853           for (PetscInt k = 0; k < extent[2]; k++) {
3854             for (PetscInt l = 0; l < 4; l++) {
3855               vtxCoords[vcount++] = (2 * i - 1) * L;
3856               vtxCoords[vcount++] = 2 * j * L + PetscCosReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
3857               vtxCoords[vcount++] = 2 * k * L + PetscSinReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
3858             }
3859           }
3860         }
3861       }
3862       // y-normal pipes
3863       for (PetscInt i = 0; i < extent[0]; i++) {
3864         for (PetscInt j = 0; j < extent[1] + 1; j++) {
3865           for (PetscInt k = 0; k < extent[2]; k++) {
3866             for (PetscInt l = 0; l < 4; l++) {
3867               vtxCoords[vcount++] = 2 * i * L + PetscSinReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
3868               vtxCoords[vcount++] = (2 * j - 1) * L;
3869               vtxCoords[vcount++] = 2 * k * L + PetscCosReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
3870             }
3871           }
3872         }
3873       }
3874       // z-normal pipes
3875       for (PetscInt i = 0; i < extent[0]; i++) {
3876         for (PetscInt j = 0; j < extent[1]; j++) {
3877           for (PetscInt k = 0; k < extent[2] + 1; k++) {
3878             for (PetscInt l = 0; l < 4; l++) {
3879               vtxCoords[vcount++] = 2 * i * L + PetscCosReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
3880               vtxCoords[vcount++] = 2 * j * L + PetscSinReal((2 * l + 1) * PETSC_PI / 4) * L / 2;
3881               vtxCoords[vcount++] = (2 * k - 1) * L;
3882             }
3883           }
3884         }
3885       }
3886       // junctions
3887       for (PetscInt i = 0; i < extent[0]; i++) {
3888         for (PetscInt j = 0; j < extent[1]; j++) {
3889           for (PetscInt k = 0; k < extent[2]; k++) {
3890             const PetscInt J = (i * extent[1] + j) * extent[2] + k, Jvoff = (Npipes[0] + Npipes[1] + Npipes[2]) * 4 + J * 8;
3891             PetscCheck(vcount / 3 == Jvoff, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Unexpected vertex count");
3892             for (PetscInt ii = 0; ii < 2; ii++) {
3893               for (PetscInt jj = 0; jj < 2; jj++) {
3894                 for (PetscInt kk = 0; kk < 2; kk++) {
3895                   double Ls           = (1 - sqrt(2) / 4) * L;
3896                   vtxCoords[vcount++] = 2 * i * L + (2 * ii - 1) * Ls;
3897                   vtxCoords[vcount++] = 2 * j * L + (2 * jj - 1) * Ls;
3898                   vtxCoords[vcount++] = 2 * k * L + (2 * kk - 1) * Ls;
3899                 }
3900               }
3901             }
3902             const PetscInt jfaces[3][2][4] = {
3903               {{3, 1, 0, 2}, {7, 5, 4, 6}}, // x-aligned
3904               {{5, 4, 0, 1}, {7, 6, 2, 3}}, // y-aligned
3905               {{6, 2, 0, 4}, {7, 3, 1, 5}}  // z-aligned
3906             };
3907             const PetscInt pipe_lo[3] = {// vertex numbers of pipes
3908                                          ((i * extent[1] + j) * extent[2] + k) * 4, ((i * (extent[1] + 1) + j) * extent[2] + k + Npipes[0]) * 4, ((i * extent[1] + j) * (extent[2] + 1) + k + Npipes[0] + Npipes[1]) * 4};
3909             const PetscInt pipe_hi[3] = {// vertex numbers of pipes
3910                                          (((i + 1) * extent[1] + j) * extent[2] + k) * 4, ((i * (extent[1] + 1) + j + 1) * extent[2] + k + Npipes[0]) * 4, ((i * extent[1] + j) * (extent[2] + 1) + k + 1 + Npipes[0] + Npipes[1]) * 4};
3911             for (PetscInt dir = 0; dir < 3; dir++) { // x,y,z
3912               const PetscInt ijk[3] = {i, j, k};
3913               for (PetscInt l = 0; l < 4; l++) { // rotations
3914                 cells[J][dir * 2 + 0][l][0] = pipe_lo[dir] + l;
3915                 cells[J][dir * 2 + 0][l][1] = Jvoff + jfaces[dir][0][l];
3916                 cells[J][dir * 2 + 0][l][2] = Jvoff + jfaces[dir][0][(l - 1 + 4) % 4];
3917                 cells[J][dir * 2 + 0][l][3] = pipe_lo[dir] + (l - 1 + 4) % 4;
3918                 cells[J][dir * 2 + 1][l][0] = Jvoff + jfaces[dir][1][l];
3919                 cells[J][dir * 2 + 1][l][1] = pipe_hi[dir] + l;
3920                 cells[J][dir * 2 + 1][l][2] = pipe_hi[dir] + (l - 1 + 4) % 4;
3921                 cells[J][dir * 2 + 1][l][3] = Jvoff + jfaces[dir][1][(l - 1 + 4) % 4];
3922                 if (ijk[dir] == 0) {
3923                   edges[numEdges][0] = pipe_lo[dir] + l;
3924                   edges[numEdges][1] = pipe_lo[dir] + (l + 1) % 4;
3925                   edgeSets[numEdges] = dir * 2 + 1;
3926                   numEdges++;
3927                 }
3928                 if (ijk[dir] + 1 == extent[dir]) {
3929                   edges[numEdges][0] = pipe_hi[dir] + l;
3930                   edges[numEdges][1] = pipe_hi[dir] + (l + 1) % 4;
3931                   edgeSets[numEdges] = dir * 2 + 2;
3932                   numEdges++;
3933                 }
3934               }
3935             }
3936           }
3937         }
3938       }
3939       PetscCheck(numEdges == Ncuts * 4, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Edge count %" PetscInt_FMT " incompatible with number of cuts %" PetscInt_FMT, numEdges, Ncuts);
3940       numFaces   = 24 * Njunctions;
3941       cells_flat = cells[0][0][0];
3942     }
3943     evalFunc   = TPSEvaluate_SchwarzP;
3944     normalFunc = TPSExtrudeNormalFunc_SchwarzP;
3945     break;
3946   case DMPLEX_TPS_GYROID:
3947     if (rank == 0) {
3948       // This is a coarse mesh approximation of the gyroid shifted to being the zero of the level set
3949       //
3950       //     sin(pi*x)*cos(pi*(y+1/2)) + sin(pi*(y+1/2))*cos(pi*(z+1/4)) + sin(pi*(z+1/4))*cos(x)
3951       //
3952       // on the cell [0,2]^3.
3953       //
3954       // Think about dividing that cell into four columns, and focus on the column [0,1]x[0,1]x[0,2].
3955       // If you looked at the gyroid in that column at different slices of z you would see that it kind of spins
3956       // like a boomerang:
3957       //
3958       //     z = 0          z = 1/4        z = 1/2        z = 3/4     //
3959       //     -----          -------        -------        -------     //
3960       //                                                              //
3961       //     +       +      +       +      +       +      +   \   +   //
3962       //      \                                   /            \      //
3963       //       \            `-_   _-'            /              }     //
3964       //        *-_            `-'            _-'              /      //
3965       //     +     `-+      +       +      +-'     +      +   /   +   //
3966       //                                                              //
3967       //                                                              //
3968       //     z = 1          z = 5/4        z = 3/2        z = 7/4     //
3969       //     -----          -------        -------        -------     //
3970       //                                                              //
3971       //     +-_     +      +       +      +     _-+      +   /   +   //
3972       //        `-_            _-_            _-`            /        //
3973       //           \        _-'   `-_        /              {         //
3974       //            \                       /                \        //
3975       //     +       +      +       +      +       +      +   \   +   //
3976       //
3977       //
3978       // This course mesh approximates each of these slices by two line segments,
3979       // and then connects the segments in consecutive layers with quadrilateral faces.
3980       // All of the end points of the segments are multiples of 1/4 except for the
3981       // point * in the picture for z = 0 above and the similar points in other layers.
3982       // That point is at (gamma, gamma, 0), where gamma is calculated below.
3983       //
3984       // The column  [1,2]x[1,2]x[0,2] looks the same as this column;
3985       // The columns [1,2]x[0,1]x[0,2] and [0,1]x[1,2]x[0,2] are mirror images.
3986       //
3987       // As for how this method turned into the names given to the vertices:
3988       // that was not systematic, it was just the way it worked out in my handwritten notes.
3989 
3990       PetscInt facesPerBlock = 64;
3991       PetscInt vertsPerBlock = 56;
3992       PetscInt extentPlus[3];
3993       PetscInt numBlocks, numBlocksPlus;
3994       const PetscInt A = 0, B = 1, C = 2, D = 3, E = 4, F = 5, G = 6, H = 7, II = 8, J = 9, K = 10, L = 11, M = 12, N = 13, O = 14, P = 15, Q = 16, R = 17, S = 18, T = 19, U = 20, V = 21, W = 22, X = 23, Y = 24, Z = 25, Ap = 26, Bp = 27, Cp = 28, Dp = 29, Ep = 30, Fp = 31, Gp = 32, Hp = 33, Ip = 34, Jp = 35, Kp = 36, Lp = 37, Mp = 38, Np = 39, Op = 40, Pp = 41, Qp = 42, Rp = 43, Sp = 44, Tp = 45, Up = 46, Vp = 47, Wp = 48, Xp = 49, Yp = 50, Zp = 51, Aq = 52, Bq = 53, Cq = 54, Dq = 55;
3995       const PetscInt pattern[64][4] = {
3996         /* face to vertex within the coarse discretization of a single gyroid block */
3997         /* layer 0 */
3998         {A,           C,           K,           G          },
3999         {C,           B,           II,          K          },
4000         {D,           A,           H,           L          },
4001         {B + 56 * 1,  D,           L,           J          },
4002         {E,           B + 56 * 1,  J,           N          },
4003         {A + 56 * 2,  E,           N,           H + 56 * 2 },
4004         {F,           A + 56 * 2,  G + 56 * 2,  M          },
4005         {B,           F,           M,           II         },
4006         /* layer 1 */
4007         {G,           K,           Q,           O          },
4008         {K,           II,          P,           Q          },
4009         {L,           H,           O + 56 * 1,  R          },
4010         {J,           L,           R,           P          },
4011         {N,           J,           P,           S          },
4012         {H + 56 * 2,  N,           S,           O + 56 * 3 },
4013         {M,           G + 56 * 2,  O + 56 * 2,  T          },
4014         {II,          M,           T,           P          },
4015         /* layer 2 */
4016         {O,           Q,           Y,           U          },
4017         {Q,           P,           W,           Y          },
4018         {R,           O + 56 * 1,  U + 56 * 1,  Ap         },
4019         {P,           R,           Ap,          W          },
4020         {S,           P,           X,           Bp         },
4021         {O + 56 * 3,  S,           Bp,          V + 56 * 1 },
4022         {T,           O + 56 * 2,  V,           Z          },
4023         {P,           T,           Z,           X          },
4024         /* layer 3 */
4025         {U,           Y,           Ep,          Dp         },
4026         {Y,           W,           Cp,          Ep         },
4027         {Ap,          U + 56 * 1,  Dp + 56 * 1, Gp         },
4028         {W,           Ap,          Gp,          Cp         },
4029         {Bp,          X,           Cp + 56 * 2, Fp         },
4030         {V + 56 * 1,  Bp,          Fp,          Dp + 56 * 1},
4031         {Z,           V,           Dp,          Hp         },
4032         {X,           Z,           Hp,          Cp + 56 * 2},
4033         /* layer 4 */
4034         {Dp,          Ep,          Mp,          Kp         },
4035         {Ep,          Cp,          Ip,          Mp         },
4036         {Gp,          Dp + 56 * 1, Lp,          Np         },
4037         {Cp,          Gp,          Np,          Jp         },
4038         {Fp,          Cp + 56 * 2, Jp + 56 * 2, Pp         },
4039         {Dp + 56 * 1, Fp,          Pp,          Lp         },
4040         {Hp,          Dp,          Kp,          Op         },
4041         {Cp + 56 * 2, Hp,          Op,          Ip + 56 * 2},
4042         /* layer 5 */
4043         {Kp,          Mp,          Sp,          Rp         },
4044         {Mp,          Ip,          Qp,          Sp         },
4045         {Np,          Lp,          Rp,          Tp         },
4046         {Jp,          Np,          Tp,          Qp + 56 * 1},
4047         {Pp,          Jp + 56 * 2, Qp + 56 * 3, Up         },
4048         {Lp,          Pp,          Up,          Rp         },
4049         {Op,          Kp,          Rp,          Vp         },
4050         {Ip + 56 * 2, Op,          Vp,          Qp + 56 * 2},
4051         /* layer 6 */
4052         {Rp,          Sp,          Aq,          Yp         },
4053         {Sp,          Qp,          Wp,          Aq         },
4054         {Tp,          Rp,          Yp,          Cq         },
4055         {Qp + 56 * 1, Tp,          Cq,          Wp + 56 * 1},
4056         {Up,          Qp + 56 * 3, Xp + 56 * 1, Dq         },
4057         {Rp,          Up,          Dq,          Zp         },
4058         {Vp,          Rp,          Zp,          Bq         },
4059         {Qp + 56 * 2, Vp,          Bq,          Xp         },
4060         /* layer 7 (the top is the periodic image of the bottom of layer 0) */
4061         {Yp,          Aq,          C + 56 * 4,  A + 56 * 4 },
4062         {Aq,          Wp,          B + 56 * 4,  C + 56 * 4 },
4063         {Cq,          Yp,          A + 56 * 4,  D + 56 * 4 },
4064         {Wp + 56 * 1, Cq,          D + 56 * 4,  B + 56 * 5 },
4065         {Dq,          Xp + 56 * 1, B + 56 * 5,  E + 56 * 4 },
4066         {Zp,          Dq,          E + 56 * 4,  A + 56 * 6 },
4067         {Bq,          Zp,          A + 56 * 6,  F + 56 * 4 },
4068         {Xp,          Bq,          F + 56 * 4,  B + 56 * 4 }
4069       };
4070       const PetscReal gamma                = PetscAcosReal((PetscSqrtReal(3.) - 1.) / PetscSqrtReal(2.)) / PETSC_PI;
4071       const PetscReal patternCoords[56][3] = {
4072         {1.,        0.,        0.  }, /* A  */
4073         {0.,        1.,        0.  }, /* B  */
4074         {gamma,     gamma,     0.  }, /* C  */
4075         {1 + gamma, 1 - gamma, 0.  }, /* D  */
4076         {2 - gamma, 2 - gamma, 0.  }, /* E  */
4077         {1 - gamma, 1 + gamma, 0.  }, /* F  */
4078 
4079         {.5,        0,         .25 }, /* G  */
4080         {1.5,       0.,        .25 }, /* H  */
4081         {.5,        1.,        .25 }, /* II */
4082         {1.5,       1.,        .25 }, /* J  */
4083         {.25,       .5,        .25 }, /* K  */
4084         {1.25,      .5,        .25 }, /* L  */
4085         {.75,       1.5,       .25 }, /* M  */
4086         {1.75,      1.5,       .25 }, /* N  */
4087 
4088         {0.,        0.,        .5  }, /* O  */
4089         {1.,        1.,        .5  }, /* P  */
4090         {gamma,     1 - gamma, .5  }, /* Q  */
4091         {1 + gamma, gamma,     .5  }, /* R  */
4092         {2 - gamma, 1 + gamma, .5  }, /* S  */
4093         {1 - gamma, 2 - gamma, .5  }, /* T  */
4094 
4095         {0.,        .5,        .75 }, /* U  */
4096         {0.,        1.5,       .75 }, /* V  */
4097         {1.,        .5,        .75 }, /* W  */
4098         {1.,        1.5,       .75 }, /* X  */
4099         {.5,        .75,       .75 }, /* Y  */
4100         {.5,        1.75,      .75 }, /* Z  */
4101         {1.5,       .25,       .75 }, /* Ap */
4102         {1.5,       1.25,      .75 }, /* Bp */
4103 
4104         {1.,        0.,        1.  }, /* Cp */
4105         {0.,        1.,        1.  }, /* Dp */
4106         {1 - gamma, 1 - gamma, 1.  }, /* Ep */
4107         {1 + gamma, 1 + gamma, 1.  }, /* Fp */
4108         {2 - gamma, gamma,     1.  }, /* Gp */
4109         {gamma,     2 - gamma, 1.  }, /* Hp */
4110 
4111         {.5,        0.,        1.25}, /* Ip */
4112         {1.5,       0.,        1.25}, /* Jp */
4113         {.5,        1.,        1.25}, /* Kp */
4114         {1.5,       1.,        1.25}, /* Lp */
4115         {.75,       .5,        1.25}, /* Mp */
4116         {1.75,      .5,        1.25}, /* Np */
4117         {.25,       1.5,       1.25}, /* Op */
4118         {1.25,      1.5,       1.25}, /* Pp */
4119 
4120         {0.,        0.,        1.5 }, /* Qp */
4121         {1.,        1.,        1.5 }, /* Rp */
4122         {1 - gamma, gamma,     1.5 }, /* Sp */
4123         {2 - gamma, 1 - gamma, 1.5 }, /* Tp */
4124         {1 + gamma, 2 - gamma, 1.5 }, /* Up */
4125         {gamma,     1 + gamma, 1.5 }, /* Vp */
4126 
4127         {0.,        .5,        1.75}, /* Wp */
4128         {0.,        1.5,       1.75}, /* Xp */
4129         {1.,        .5,        1.75}, /* Yp */
4130         {1.,        1.5,       1.75}, /* Zp */
4131         {.5,        .25,       1.75}, /* Aq */
4132         {.5,        1.25,      1.75}, /* Bq */
4133         {1.5,       .75,       1.75}, /* Cq */
4134         {1.5,       1.75,      1.75}, /* Dq */
4135       };
4136       PetscInt (*cells)[64][4] = NULL;
4137       PetscBool *seen;
4138       PetscInt  *vertToTrueVert;
4139       PetscInt   count;
4140 
4141       for (PetscInt i = 0; i < 3; i++) extentPlus[i] = extent[i] + 1;
4142       numBlocks = 1;
4143       for (PetscInt i = 0; i < 3; i++) numBlocks *= extent[i];
4144       numBlocksPlus = 1;
4145       for (PetscInt i = 0; i < 3; i++) numBlocksPlus *= extentPlus[i];
4146       numFaces = numBlocks * facesPerBlock;
4147       PetscCall(PetscMalloc1(numBlocks, &cells));
4148       PetscCall(PetscCalloc1(numBlocksPlus * vertsPerBlock, &seen));
4149       for (PetscInt k = 0; k < extent[2]; k++) {
4150         for (PetscInt j = 0; j < extent[1]; j++) {
4151           for (PetscInt i = 0; i < extent[0]; i++) {
4152             for (PetscInt f = 0; f < facesPerBlock; f++) {
4153               for (PetscInt v = 0; v < 4; v++) {
4154                 PetscInt vertRaw     = pattern[f][v];
4155                 PetscInt blockidx    = vertRaw / 56;
4156                 PetscInt patternvert = vertRaw % 56;
4157                 PetscInt xplus       = (blockidx & 1);
4158                 PetscInt yplus       = (blockidx & 2) >> 1;
4159                 PetscInt zplus       = (blockidx & 4) >> 2;
4160                 PetscInt zcoord      = (periodic && periodic[2] == DM_BOUNDARY_PERIODIC) ? ((k + zplus) % extent[2]) : (k + zplus);
4161                 PetscInt ycoord      = (periodic && periodic[1] == DM_BOUNDARY_PERIODIC) ? ((j + yplus) % extent[1]) : (j + yplus);
4162                 PetscInt xcoord      = (periodic && periodic[0] == DM_BOUNDARY_PERIODIC) ? ((i + xplus) % extent[0]) : (i + xplus);
4163                 PetscInt vert        = ((zcoord * extentPlus[1] + ycoord) * extentPlus[0] + xcoord) * 56 + patternvert;
4164 
4165                 cells[(k * extent[1] + j) * extent[0] + i][f][v] = vert;
4166                 seen[vert]                                       = PETSC_TRUE;
4167               }
4168             }
4169           }
4170         }
4171       }
4172       for (PetscInt i = 0; i < numBlocksPlus * vertsPerBlock; i++)
4173         if (seen[i]) numVertices++;
4174       count = 0;
4175       PetscCall(PetscMalloc1(numBlocksPlus * vertsPerBlock, &vertToTrueVert));
4176       PetscCall(PetscMalloc1(numVertices * 3, &vtxCoords));
4177       for (PetscInt i = 0; i < numBlocksPlus * vertsPerBlock; i++) vertToTrueVert[i] = -1;
4178       for (PetscInt k = 0; k < extentPlus[2]; k++) {
4179         for (PetscInt j = 0; j < extentPlus[1]; j++) {
4180           for (PetscInt i = 0; i < extentPlus[0]; i++) {
4181             for (PetscInt v = 0; v < vertsPerBlock; v++) {
4182               PetscInt vIdx = ((k * extentPlus[1] + j) * extentPlus[0] + i) * vertsPerBlock + v;
4183 
4184               if (seen[vIdx]) {
4185                 PetscInt thisVert;
4186 
4187                 vertToTrueVert[vIdx] = thisVert = count++;
4188 
4189                 for (PetscInt d = 0; d < 3; d++) vtxCoords[3 * thisVert + d] = patternCoords[v][d];
4190                 vtxCoords[3 * thisVert + 0] += i * 2;
4191                 vtxCoords[3 * thisVert + 1] += j * 2;
4192                 vtxCoords[3 * thisVert + 2] += k * 2;
4193               }
4194             }
4195           }
4196         }
4197       }
4198       for (PetscInt i = 0; i < numBlocks; i++) {
4199         for (PetscInt f = 0; f < facesPerBlock; f++) {
4200           for (PetscInt v = 0; v < 4; v++) cells[i][f][v] = vertToTrueVert[cells[i][f][v]];
4201         }
4202       }
4203       PetscCall(PetscFree(vertToTrueVert));
4204       PetscCall(PetscFree(seen));
4205       cells_flat = cells[0][0];
4206       numEdges   = 0;
4207       for (PetscInt i = 0; i < numFaces; i++) {
4208         for (PetscInt e = 0; e < 4; e++) {
4209           PetscInt         ev[]       = {cells_flat[i * 4 + e], cells_flat[i * 4 + ((e + 1) % 4)]};
4210           const PetscReal *evCoords[] = {&vtxCoords[3 * ev[0]], &vtxCoords[3 * ev[1]]};
4211 
4212           for (PetscInt d = 0; d < 3; d++) {
4213             if (!periodic || periodic[0] != DM_BOUNDARY_PERIODIC) {
4214               if (evCoords[0][d] == 0. && evCoords[1][d] == 0.) numEdges++;
4215               if (evCoords[0][d] == 2. * extent[d] && evCoords[1][d] == 2. * extent[d]) numEdges++;
4216             }
4217           }
4218         }
4219       }
4220       PetscCall(PetscMalloc1(numEdges, &edges));
4221       PetscCall(PetscMalloc1(numEdges, &edgeSets));
4222       for (PetscInt edge = 0, i = 0; i < numFaces; i++) {
4223         for (PetscInt e = 0; e < 4; e++) {
4224           PetscInt         ev[]       = {cells_flat[i * 4 + e], cells_flat[i * 4 + ((e + 1) % 4)]};
4225           const PetscReal *evCoords[] = {&vtxCoords[3 * ev[0]], &vtxCoords[3 * ev[1]]};
4226 
4227           for (PetscInt d = 0; d < 3; d++) {
4228             if (!periodic || periodic[d] != DM_BOUNDARY_PERIODIC) {
4229               if (evCoords[0][d] == 0. && evCoords[1][d] == 0.) {
4230                 edges[edge][0]   = ev[0];
4231                 edges[edge][1]   = ev[1];
4232                 edgeSets[edge++] = 2 * d;
4233               }
4234               if (evCoords[0][d] == 2. * extent[d] && evCoords[1][d] == 2. * extent[d]) {
4235                 edges[edge][0]   = ev[0];
4236                 edges[edge][1]   = ev[1];
4237                 edgeSets[edge++] = 2 * d + 1;
4238               }
4239             }
4240           }
4241         }
4242       }
4243     }
4244     evalFunc   = TPSEvaluate_Gyroid;
4245     normalFunc = TPSExtrudeNormalFunc_Gyroid;
4246     break;
4247   }
4248 
4249   PetscCall(DMSetDimension(dm, topoDim));
4250   if (rank == 0) PetscCall(DMPlexBuildFromCellList(dm, numFaces, numVertices, 4, cells_flat));
4251   else PetscCall(DMPlexBuildFromCellList(dm, 0, 0, 0, NULL));
4252   PetscCall(PetscFree(cells_flat));
4253   {
4254     DM idm;
4255     PetscCall(DMPlexInterpolate(dm, &idm));
4256     PetscCall(DMPlexReplace_Internal(dm, &idm));
4257   }
4258   if (rank == 0) PetscCall(DMPlexBuildCoordinatesFromCellList(dm, spaceDim, vtxCoords));
4259   else PetscCall(DMPlexBuildCoordinatesFromCellList(dm, spaceDim, NULL));
4260   PetscCall(PetscFree(vtxCoords));
4261 
4262   PetscCall(DMCreateLabel(dm, "Face Sets"));
4263   PetscCall(DMGetLabel(dm, "Face Sets", &label));
4264   for (PetscInt e = 0; e < numEdges; e++) {
4265     PetscInt        njoin;
4266     const PetscInt *join, verts[] = {numFaces + edges[e][0], numFaces + edges[e][1]};
4267     PetscCall(DMPlexGetJoin(dm, 2, verts, &njoin, &join));
4268     PetscCheck(njoin == 1, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Expected unique join of vertices %" PetscInt_FMT " and %" PetscInt_FMT, edges[e][0], edges[e][1]);
4269     PetscCall(DMLabelSetValue(label, join[0], edgeSets[e]));
4270     PetscCall(DMPlexRestoreJoin(dm, 2, verts, &njoin, &join));
4271   }
4272   PetscCall(PetscFree(edges));
4273   PetscCall(PetscFree(edgeSets));
4274   if (tps_distribute) {
4275     DM               pdm = NULL;
4276     PetscPartitioner part;
4277 
4278     PetscCall(DMPlexGetPartitioner(dm, &part));
4279     PetscCall(PetscPartitionerSetFromOptions(part));
4280     PetscCall(DMPlexDistribute(dm, 0, NULL, &pdm));
4281     if (pdm) PetscCall(DMPlexReplace_Internal(dm, &pdm));
4282     // Do not auto-distribute again
4283     PetscCall(DMPlexDistributeSetDefault(dm, PETSC_FALSE));
4284   }
4285 
4286   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
4287   for (PetscInt refine = 0; refine < refinements; refine++) {
4288     PetscInt     m;
4289     DM           dmf, cdm, cdmf;
4290     Vec          X;
4291     PetscScalar *x;
4292 
4293     PetscCall(DMRefine(dm, MPI_COMM_NULL, &dmf));
4294     PetscCall(DMGetCoordinateDM(dm, &cdm));
4295     PetscCall(DMGetCoordinateDM(dmf, &cdmf));
4296     PetscCall(DMCopyDisc(cdm, cdmf));
4297     PetscCall(DMPlexReplace_Internal(dm, &dmf));
4298 
4299     PetscCall(DMGetCoordinatesLocal(dm, &X));
4300     PetscCall(VecGetLocalSize(X, &m));
4301     PetscCall(VecGetArray(X, &x));
4302     for (PetscInt i = 0; i < m; i += 3) PetscCall(TPSNearestPoint(evalFunc, &x[i]));
4303     PetscCall(VecRestoreArray(X, &x));
4304   }
4305 
4306   // Face Sets has already been propagated to new vertices during refinement; this propagates to the initial vertices.
4307   PetscCall(DMGetLabel(dm, "Face Sets", &label));
4308   PetscCall(DMPlexLabelComplete(dm, label));
4309 
4310   PetscCall(PetscLogEventEnd(DMPLEX_Generate, dm, 0, 0, 0));
4311 
4312   if (thickness > 0) {
4313     DM              edm, cdm, ecdm;
4314     DMPlexTransform tr;
4315     const char     *prefix;
4316     PetscOptions    options;
4317     // Code from DMPlexExtrude
4318     PetscCall(DMPlexTransformCreate(PetscObjectComm((PetscObject)dm), &tr));
4319     PetscCall(DMPlexTransformSetDM(tr, dm));
4320     PetscCall(DMPlexTransformSetType(tr, DMPLEXEXTRUDETYPE));
4321     PetscCall(PetscObjectGetOptionsPrefix((PetscObject)dm, &prefix));
4322     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)tr, prefix));
4323     PetscCall(PetscObjectGetOptions((PetscObject)dm, &options));
4324     PetscCall(PetscObjectSetOptions((PetscObject)tr, options));
4325     PetscCall(DMPlexTransformExtrudeSetLayers(tr, layers));
4326     PetscCall(DMPlexTransformExtrudeSetThickness(tr, thickness));
4327     PetscCall(DMPlexTransformExtrudeSetTensor(tr, PETSC_FALSE));
4328     PetscCall(DMPlexTransformExtrudeSetSymmetric(tr, PETSC_TRUE));
4329     PetscCall(DMPlexTransformExtrudeSetNormalFunction(tr, normalFunc));
4330     PetscCall(DMPlexTransformSetFromOptions(tr));
4331     PetscCall(PetscObjectSetOptions((PetscObject)tr, NULL));
4332     PetscCall(DMPlexTransformSetUp(tr));
4333     PetscCall(PetscObjectViewFromOptions((PetscObject)tr, NULL, "-dm_plex_tps_transform_view"));
4334     PetscCall(DMPlexTransformApply(tr, dm, &edm));
4335     PetscCall(DMCopyDisc(dm, edm));
4336     PetscCall(DMGetCoordinateDM(dm, &cdm));
4337     PetscCall(DMGetCoordinateDM(edm, &ecdm));
4338     PetscCall(DMCopyDisc(cdm, ecdm));
4339     PetscCall(DMPlexTransformCreateDiscLabels(tr, edm));
4340     PetscCall(DMPlexTransformDestroy(&tr));
4341     PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, edm));
4342     PetscCall(DMPlexReplace_Internal(dm, &edm));
4343   }
4344   PetscFunctionReturn(PETSC_SUCCESS);
4345 }
4346 
4347 /*@
4348   DMPlexCreateTPSMesh - Create a distributed, interpolated mesh of a triply-periodic surface
4349 
4350   Collective
4351 
4352   Input Parameters:
4353 + comm           - The communicator for the `DM` object
4354 . tpstype        - Type of triply-periodic surface
4355 . extent         - Array of length 3 containing number of periods in each direction
4356 . periodic       - array of length 3 with periodicity, or `NULL` for non-periodic
4357 . tps_distribute - Distribute 2D manifold mesh prior to refinement and extrusion (more scalable)
4358 . refinements    - Number of factor-of-2 refinements of 2D manifold mesh
4359 . layers         - Number of cell layers extruded in normal direction
4360 - thickness      - Thickness in normal direction
4361 
4362   Output Parameter:
4363 . dm - The `DM` object
4364 
4365   Level: beginner
4366 
4367   Notes:
4368   This meshes the surface of the Schwarz P or Gyroid surfaces.  Schwarz P is the simplest member of the triply-periodic minimal surfaces.
4369   <https://en.wikipedia.org/wiki/Schwarz_minimal_surface#Schwarz_P_(%22Primitive%22)> and can be cut with "clean" boundaries.
4370   The Gyroid <https://en.wikipedia.org/wiki/Gyroid> is another triply-periodic minimal surface with applications in additive manufacturing; it is much more difficult to "cut" since there are no planes of symmetry.
4371   Our implementation creates a very coarse mesh of the surface and refines (by 4-way splitting) as many times as requested.
4372   On each refinement, all vertices are projected to their nearest point on the surface.
4373   This projection could readily be extended to related surfaces.
4374 
4375   See {cite}`maskery2018insights`
4376 
4377   The face (edge) sets for the Schwarz P surface are numbered $1(-x), 2(+x), 3(-y), 4(+y), 5(-z), 6(+z)$.
4378   When the mesh is refined, "Face Sets" contain the new vertices (created during refinement).
4379   Use `DMPlexLabelComplete()` to propagate to coarse-level vertices.
4380 
4381   Developer Notes:
4382   The Gyroid mesh does not currently mark boundary sets.
4383 
4384 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateSphereMesh()`, `DMSetType()`, `DMCreate()`
4385 @*/
4386 PetscErrorCode DMPlexCreateTPSMesh(MPI_Comm comm, DMPlexTPSType tpstype, const PetscInt extent[], const DMBoundaryType periodic[], PetscBool tps_distribute, PetscInt refinements, PetscInt layers, PetscReal thickness, DM *dm)
4387 {
4388   PetscFunctionBegin;
4389   PetscCall(DMCreate(comm, dm));
4390   PetscCall(DMSetType(*dm, DMPLEX));
4391   PetscCall(DMPlexCreateTPSMesh_Internal(*dm, tpstype, extent, periodic, tps_distribute, refinements, layers, thickness));
4392   PetscFunctionReturn(PETSC_SUCCESS);
4393 }
4394 
4395 /*@
4396   DMPlexCreateSphereMesh - Creates a mesh on the d-dimensional sphere, S^d.
4397 
4398   Collective
4399 
4400   Input Parameters:
4401 + comm    - The communicator for the `DM` object
4402 . dim     - The dimension
4403 . simplex - Use simplices, or tensor product cells
4404 - R       - The radius
4405 
4406   Output Parameter:
4407 . dm - The `DM` object
4408 
4409   Level: beginner
4410 
4411 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateBallMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
4412 @*/
4413 PetscErrorCode DMPlexCreateSphereMesh(MPI_Comm comm, PetscInt dim, PetscBool simplex, PetscReal R, DM *dm)
4414 {
4415   PetscFunctionBegin;
4416   PetscAssertPointer(dm, 5);
4417   PetscCall(DMCreate(comm, dm));
4418   PetscCall(DMSetType(*dm, DMPLEX));
4419   PetscCall(DMPlexCreateSphereMesh_Internal(*dm, dim, simplex, R));
4420   PetscFunctionReturn(PETSC_SUCCESS);
4421 }
4422 
4423 static PetscErrorCode DMPlexCreateBallMesh_Internal(DM dm, PetscInt dim, PetscReal R)
4424 {
4425   DM          sdm, vol;
4426   DMLabel     bdlabel;
4427   const char *prefix;
4428 
4429   PetscFunctionBegin;
4430   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &sdm));
4431   PetscCall(DMSetType(sdm, DMPLEX));
4432   PetscCall(DMGetOptionsPrefix(dm, &prefix));
4433   PetscCall(DMSetOptionsPrefix(sdm, prefix));
4434   PetscCall(DMAppendOptionsPrefix(sdm, "bd_"));
4435   PetscCall(DMPlexDistributeSetDefault(sdm, PETSC_FALSE));
4436   PetscCall(DMPlexCreateSphereMesh_Internal(sdm, dim - 1, PETSC_TRUE, R));
4437   PetscCall(DMSetFromOptions(sdm));
4438   PetscCall(DMViewFromOptions(sdm, NULL, "-dm_view"));
4439   PetscCall(DMPlexGenerate(sdm, NULL, PETSC_TRUE, &vol));
4440   PetscCall(DMDestroy(&sdm));
4441   PetscCall(DMPlexReplace_Internal(dm, &vol));
4442   PetscCall(DMCreateLabel(dm, "marker"));
4443   PetscCall(DMGetLabel(dm, "marker", &bdlabel));
4444   PetscCall(DMPlexMarkBoundaryFaces(dm, PETSC_DETERMINE, bdlabel));
4445   PetscCall(DMPlexLabelComplete(dm, bdlabel));
4446   PetscFunctionReturn(PETSC_SUCCESS);
4447 }
4448 
4449 /*@
4450   DMPlexCreateBallMesh - Creates a simplex mesh on the d-dimensional ball, B^d.
4451 
4452   Collective
4453 
4454   Input Parameters:
4455 + comm - The communicator for the `DM` object
4456 . dim  - The dimension
4457 - R    - The radius
4458 
4459   Output Parameter:
4460 . dm - The `DM` object
4461 
4462   Options Database Key:
4463 . bd_dm_refine - This will refine the surface mesh preserving the sphere geometry
4464 
4465   Level: beginner
4466 
4467 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateSphereMesh()`, `DMPlexCreateBoxMesh()`, `DMSetType()`, `DMCreate()`
4468 @*/
4469 PetscErrorCode DMPlexCreateBallMesh(MPI_Comm comm, PetscInt dim, PetscReal R, DM *dm)
4470 {
4471   PetscFunctionBegin;
4472   PetscCall(DMCreate(comm, dm));
4473   PetscCall(DMSetType(*dm, DMPLEX));
4474   PetscCall(DMPlexCreateBallMesh_Internal(*dm, dim, R));
4475   PetscFunctionReturn(PETSC_SUCCESS);
4476 }
4477 
4478 static PetscErrorCode DMPlexCreateReferenceCell_Internal(DM rdm, DMPolytopeType ct)
4479 {
4480   PetscFunctionBegin;
4481   switch (ct) {
4482   case DM_POLYTOPE_POINT: {
4483     PetscInt    numPoints[1]        = {1};
4484     PetscInt    coneSize[1]         = {0};
4485     PetscInt    cones[1]            = {0};
4486     PetscInt    coneOrientations[1] = {0};
4487     PetscScalar vertexCoords[1]     = {0.0};
4488 
4489     PetscCall(DMSetDimension(rdm, 0));
4490     PetscCall(DMPlexCreateFromDAG(rdm, 0, numPoints, coneSize, cones, coneOrientations, vertexCoords));
4491   } break;
4492   case DM_POLYTOPE_SEGMENT: {
4493     PetscInt    numPoints[2]        = {2, 1};
4494     PetscInt    coneSize[3]         = {2, 0, 0};
4495     PetscInt    cones[2]            = {1, 2};
4496     PetscInt    coneOrientations[2] = {0, 0};
4497     PetscScalar vertexCoords[2]     = {-1.0, 1.0};
4498 
4499     PetscCall(DMSetDimension(rdm, 1));
4500     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
4501   } break;
4502   case DM_POLYTOPE_POINT_PRISM_TENSOR: {
4503     PetscInt    numPoints[2]        = {2, 1};
4504     PetscInt    coneSize[3]         = {2, 0, 0};
4505     PetscInt    cones[2]            = {1, 2};
4506     PetscInt    coneOrientations[2] = {0, 0};
4507     PetscScalar vertexCoords[2]     = {-1.0, 1.0};
4508 
4509     PetscCall(DMSetDimension(rdm, 1));
4510     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
4511   } break;
4512   case DM_POLYTOPE_TRIANGLE: {
4513     PetscInt    numPoints[2]        = {3, 1};
4514     PetscInt    coneSize[4]         = {3, 0, 0, 0};
4515     PetscInt    cones[3]            = {1, 2, 3};
4516     PetscInt    coneOrientations[3] = {0, 0, 0};
4517     PetscScalar vertexCoords[6]     = {-1.0, -1.0, 1.0, -1.0, -1.0, 1.0};
4518 
4519     PetscCall(DMSetDimension(rdm, 2));
4520     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
4521   } break;
4522   case DM_POLYTOPE_QUADRILATERAL: {
4523     PetscInt    numPoints[2]        = {4, 1};
4524     PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
4525     PetscInt    cones[4]            = {1, 2, 3, 4};
4526     PetscInt    coneOrientations[4] = {0, 0, 0, 0};
4527     PetscScalar vertexCoords[8]     = {-1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0};
4528 
4529     PetscCall(DMSetDimension(rdm, 2));
4530     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
4531   } break;
4532   case DM_POLYTOPE_SEG_PRISM_TENSOR: {
4533     PetscInt    numPoints[2]        = {4, 1};
4534     PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
4535     PetscInt    cones[4]            = {1, 2, 3, 4};
4536     PetscInt    coneOrientations[4] = {0, 0, 0, 0};
4537     PetscScalar vertexCoords[8]     = {-1.0, -1.0, 1.0, -1.0, -1.0, 1.0, 1.0, 1.0};
4538 
4539     PetscCall(DMSetDimension(rdm, 2));
4540     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
4541   } break;
4542   case DM_POLYTOPE_TETRAHEDRON: {
4543     PetscInt    numPoints[2]        = {4, 1};
4544     PetscInt    coneSize[5]         = {4, 0, 0, 0, 0};
4545     PetscInt    cones[4]            = {1, 2, 3, 4};
4546     PetscInt    coneOrientations[4] = {0, 0, 0, 0};
4547     PetscScalar vertexCoords[12]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0};
4548 
4549     PetscCall(DMSetDimension(rdm, 3));
4550     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
4551   } break;
4552   case DM_POLYTOPE_HEXAHEDRON: {
4553     PetscInt    numPoints[2]        = {8, 1};
4554     PetscInt    coneSize[9]         = {8, 0, 0, 0, 0, 0, 0, 0, 0};
4555     PetscInt    cones[8]            = {1, 2, 3, 4, 5, 6, 7, 8};
4556     PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
4557     PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, 1.0, 1.0, 1.0, -1.0, 1.0, 1.0};
4558 
4559     PetscCall(DMSetDimension(rdm, 3));
4560     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
4561   } break;
4562   case DM_POLYTOPE_TRI_PRISM: {
4563     PetscInt    numPoints[2]        = {6, 1};
4564     PetscInt    coneSize[7]         = {6, 0, 0, 0, 0, 0, 0};
4565     PetscInt    cones[6]            = {1, 2, 3, 4, 5, 6};
4566     PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
4567     PetscScalar vertexCoords[18]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, 1.0, 1.0};
4568 
4569     PetscCall(DMSetDimension(rdm, 3));
4570     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
4571   } break;
4572   case DM_POLYTOPE_TRI_PRISM_TENSOR: {
4573     PetscInt    numPoints[2]        = {6, 1};
4574     PetscInt    coneSize[7]         = {6, 0, 0, 0, 0, 0, 0};
4575     PetscInt    cones[6]            = {1, 2, 3, 4, 5, 6};
4576     PetscInt    coneOrientations[6] = {0, 0, 0, 0, 0, 0};
4577     PetscScalar vertexCoords[18]    = {-1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, 1.0, 1.0};
4578 
4579     PetscCall(DMSetDimension(rdm, 3));
4580     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
4581   } break;
4582   case DM_POLYTOPE_QUAD_PRISM_TENSOR: {
4583     PetscInt    numPoints[2]        = {8, 1};
4584     PetscInt    coneSize[9]         = {8, 0, 0, 0, 0, 0, 0, 0, 0};
4585     PetscInt    cones[8]            = {1, 2, 3, 4, 5, 6, 7, 8};
4586     PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
4587     PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0, 1.0, -1.0, -1.0, 1.0, 1.0, -1.0, -1.0, 1.0, -1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, 1.0, 1.0, 1.0, -1.0, 1.0, 1.0};
4588 
4589     PetscCall(DMSetDimension(rdm, 3));
4590     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
4591   } break;
4592   case DM_POLYTOPE_PYRAMID: {
4593     PetscInt    numPoints[2]        = {5, 1};
4594     PetscInt    coneSize[6]         = {5, 0, 0, 0, 0, 0};
4595     PetscInt    cones[5]            = {1, 2, 3, 4, 5};
4596     PetscInt    coneOrientations[8] = {0, 0, 0, 0, 0, 0, 0, 0};
4597     PetscScalar vertexCoords[24]    = {-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, -1.0, 0.0, 0.0, 1.0};
4598 
4599     PetscCall(DMSetDimension(rdm, 3));
4600     PetscCall(DMPlexCreateFromDAG(rdm, 1, numPoints, coneSize, cones, coneOrientations, vertexCoords));
4601   } break;
4602   default:
4603     SETERRQ(PetscObjectComm((PetscObject)rdm), PETSC_ERR_ARG_WRONG, "Cannot create reference cell for cell type %s", DMPolytopeTypes[ct]);
4604   }
4605   {
4606     PetscInt Nv, v;
4607 
4608     /* Must create the celltype label here so that we do not automatically try to compute the types */
4609     PetscCall(DMCreateLabel(rdm, "celltype"));
4610     PetscCall(DMPlexSetCellType(rdm, 0, ct));
4611     PetscCall(DMPlexGetChart(rdm, NULL, &Nv));
4612     for (v = 1; v < Nv; ++v) PetscCall(DMPlexSetCellType(rdm, v, DM_POLYTOPE_POINT));
4613   }
4614   PetscCall(DMPlexInterpolateInPlace_Internal(rdm));
4615   PetscCall(PetscObjectSetName((PetscObject)rdm, DMPolytopeTypes[ct]));
4616   PetscFunctionReturn(PETSC_SUCCESS);
4617 }
4618 
4619 /*@
4620   DMPlexCreateReferenceCell - Create a `DMPLEX` with the appropriate FEM reference cell
4621 
4622   Collective
4623 
4624   Input Parameters:
4625 + comm - The communicator
4626 - ct   - The cell type of the reference cell
4627 
4628   Output Parameter:
4629 . refdm - The reference cell
4630 
4631   Level: intermediate
4632 
4633 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateBoxMesh()`
4634 @*/
4635 PetscErrorCode DMPlexCreateReferenceCell(MPI_Comm comm, DMPolytopeType ct, DM *refdm)
4636 {
4637   PetscFunctionBegin;
4638   PetscCall(DMCreate(comm, refdm));
4639   PetscCall(DMSetType(*refdm, DMPLEX));
4640   PetscCall(DMPlexCreateReferenceCell_Internal(*refdm, ct));
4641   PetscFunctionReturn(PETSC_SUCCESS);
4642 }
4643 
4644 static PetscErrorCode DMPlexCreateBoundaryLabel_Private(DM dm, const char name[])
4645 {
4646   DM        plex;
4647   DMLabel   label;
4648   PetscBool hasLabel;
4649 
4650   PetscFunctionBegin;
4651   PetscCall(DMHasLabel(dm, name, &hasLabel));
4652   if (hasLabel) PetscFunctionReturn(PETSC_SUCCESS);
4653   PetscCall(DMCreateLabel(dm, name));
4654   PetscCall(DMGetLabel(dm, name, &label));
4655   PetscCall(DMConvert(dm, DMPLEX, &plex));
4656   PetscCall(DMPlexMarkBoundaryFaces(plex, 1, label));
4657   PetscCall(DMPlexLabelComplete(plex, label));
4658   PetscCall(DMDestroy(&plex));
4659   PetscFunctionReturn(PETSC_SUCCESS);
4660 }
4661 
4662 /*
4663   We use the last coordinate as the radius, the inner radius is lower[dim-1] and the outer radius is upper[dim-1]. Then we map the first coordinate around the circle.
4664 
4665     (x, y) -> (r, theta) = (x[1], (x[0] - lower[0]) * 2\pi/(upper[0] - lower[0]))
4666 */
4667 static void boxToAnnulus(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
4668 {
4669   const PetscReal low = PetscRealPart(constants[0]);
4670   const PetscReal upp = PetscRealPart(constants[1]);
4671   const PetscReal r   = PetscRealPart(u[1]);
4672   const PetscReal th  = 2. * PETSC_PI * (PetscRealPart(u[0]) - low) / (upp - low);
4673 
4674   f0[0] = r * PetscCosReal(th);
4675   f0[1] = r * PetscSinReal(th);
4676 }
4677 
4678 // Insert vertices and their joins, marked by depth
4679 static PetscErrorCode ProcessCohesiveLabel_Vertices(DM dm, DMLabel label, DMLabel vlabel, PetscInt val, PetscInt n, const PetscInt vertices[])
4680 {
4681   PetscFunctionBegin;
4682   PetscCall(DMPlexMarkSubmesh_Interpolated(dm, vlabel, val, PETSC_FALSE, PETSC_FALSE, label, NULL));
4683   PetscFunctionReturn(PETSC_SUCCESS);
4684 }
4685 
4686 // Insert faces and their closures, marked by depth
4687 static PetscErrorCode ProcessCohesiveLabel_Faces(DM dm, DMLabel label, PetscInt n, const PetscInt faces[])
4688 {
4689   PetscFunctionBegin;
4690   for (PetscInt p = 0; p < n; ++p) {
4691     const PetscInt point   = faces[p];
4692     PetscInt      *closure = NULL;
4693     PetscInt       clSize, pdepth;
4694 
4695     PetscCall(DMPlexGetPointDepth(dm, point, &pdepth));
4696     PetscCall(DMLabelSetValue(label, point, pdepth));
4697     PetscCall(DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &clSize, &closure));
4698     for (PetscInt cl = 0; cl < clSize * 2; cl += 2) {
4699       PetscCall(DMPlexGetPointDepth(dm, closure[cl], &pdepth));
4700       PetscCall(DMLabelSetValue(label, closure[cl], pdepth));
4701     }
4702     PetscCall(DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &clSize, &closure));
4703   }
4704   PetscFunctionReturn(PETSC_SUCCESS);
4705 }
4706 
4707 PETSC_EXTERN PetscErrorCode PetscOptionsFindPairPrefix_Private(PetscOptions, const char pre[], const char name[], const char *option[], const char *value[], PetscBool *flg);
4708 
4709 const char *const DMPlexShapes[] = {"box", "box_surface", "ball", "sphere", "cylinder", "schwarz_p", "gyroid", "doublet", "annulus", "hypercubic", "zbox", "unknown", "DMPlexShape", "DM_SHAPE_", NULL};
4710 
4711 static PetscErrorCode DMPlexCreateFromOptions_Internal(PetscOptionItems PetscOptionsObject, PetscBool *useCoordSpace, DM dm)
4712 {
4713   DMPlexShape    shape   = DM_SHAPE_BOX;
4714   DMPolytopeType cell    = DM_POLYTOPE_TRIANGLE;
4715   PetscInt       dim     = 2;
4716   PetscBool      simplex = PETSC_TRUE, interpolate = PETSC_TRUE, orient = PETSC_FALSE, adjCone = PETSC_FALSE, adjClosure = PETSC_TRUE, refDomain = PETSC_FALSE;
4717   PetscBool      flg, flg2, fflg, strflg, bdfflg, nameflg;
4718   MPI_Comm       comm;
4719   char           filename[PETSC_MAX_PATH_LEN]   = "<unspecified>";
4720   char           bdFilename[PETSC_MAX_PATH_LEN] = "<unspecified>";
4721   char           plexname[PETSC_MAX_PATH_LEN]   = "";
4722   const char    *option;
4723 
4724   PetscFunctionBegin;
4725   PetscCall(PetscLogEventBegin(DMPLEX_CreateFromOptions, dm, 0, 0, 0));
4726   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
4727   /* TODO Turn this into a registration interface */
4728   PetscCall(PetscOptionsString("-dm_plex_filename", "File containing a mesh", "DMPlexCreateFromFile", filename, filename, sizeof(filename), &fflg));
4729   PetscCall(PetscOptionsString("-dm_plex_file_contents", "Contents of a file format in a string", "DMPlexCreateFromFile", filename, filename, sizeof(filename), &strflg));
4730   PetscCall(PetscOptionsString("-dm_plex_boundary_filename", "File containing a mesh boundary", "DMPlexCreateFromFile", bdFilename, bdFilename, sizeof(bdFilename), &bdfflg));
4731   PetscCall(PetscOptionsString("-dm_plex_name", "Name of the mesh in the file", "DMPlexCreateFromFile", plexname, plexname, sizeof(plexname), &nameflg));
4732   PetscCall(PetscOptionsEnum("-dm_plex_cell", "Cell shape", "", DMPolytopeTypes, (PetscEnum)cell, (PetscEnum *)&cell, NULL));
4733   PetscCall(PetscOptionsBool("-dm_plex_reference_cell_domain", "Use a reference cell domain", "", refDomain, &refDomain, NULL));
4734   PetscCall(PetscOptionsEnum("-dm_plex_shape", "Shape for built-in mesh", "", DMPlexShapes, (PetscEnum)shape, (PetscEnum *)&shape, &flg));
4735   PetscCall(PetscOptionsBoundedInt("-dm_plex_dim", "Topological dimension of the mesh", "DMGetDimension", dim, &dim, &flg, 0));
4736   PetscCall(PetscOptionsBool("-dm_plex_simplex", "Mesh cell shape", "", simplex, &simplex, &flg));
4737   PetscCall(PetscOptionsBool("-dm_plex_interpolate", "Flag to create edges and faces automatically", "", interpolate, &interpolate, &flg));
4738   PetscCall(PetscOptionsBool("-dm_plex_orient", "Orient the constructed mesh", "DMPlexOrient", orient, &orient, &flg));
4739   PetscCall(PetscOptionsBool("-dm_plex_adj_cone", "Set adjacency direction", "DMSetBasicAdjacency", adjCone, &adjCone, &flg));
4740   PetscCall(PetscOptionsBool("-dm_plex_adj_closure", "Set adjacency size", "DMSetBasicAdjacency", adjClosure, &adjClosure, &flg2));
4741   if (flg || flg2) PetscCall(DMSetBasicAdjacency(dm, adjCone, adjClosure));
4742   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_adj", "Debug output level all adjacency computations", "", 0, &((DM_Plex *)dm->data)->printAdj, NULL, 0));
4743 
4744   switch (cell) {
4745   case DM_POLYTOPE_POINT:
4746   case DM_POLYTOPE_SEGMENT:
4747   case DM_POLYTOPE_POINT_PRISM_TENSOR:
4748   case DM_POLYTOPE_TRIANGLE:
4749   case DM_POLYTOPE_QUADRILATERAL:
4750   case DM_POLYTOPE_TETRAHEDRON:
4751   case DM_POLYTOPE_HEXAHEDRON:
4752     *useCoordSpace = PETSC_TRUE;
4753     break;
4754   default:
4755     *useCoordSpace = PETSC_FALSE;
4756     break;
4757   }
4758 
4759   if (fflg) {
4760     DM          dmnew;
4761     const char *name;
4762 
4763     PetscCall(PetscObjectGetName((PetscObject)dm, &name));
4764     PetscCall(DMPlexCreateFromFile(PetscObjectComm((PetscObject)dm), filename, nameflg ? plexname : name, interpolate, &dmnew));
4765     PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
4766     PetscCall(DMPlexReplace_Internal(dm, &dmnew));
4767   } else if (refDomain) {
4768     PetscCall(DMPlexCreateReferenceCell_Internal(dm, cell));
4769   } else if (bdfflg) {
4770     DM          bdm, dmnew;
4771     const char *name;
4772 
4773     PetscCall(PetscObjectGetName((PetscObject)dm, &name));
4774     PetscCall(DMPlexCreateFromFile(PetscObjectComm((PetscObject)dm), bdFilename, nameflg ? plexname : name, interpolate, &bdm));
4775     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)bdm, "bd_"));
4776     PetscCall(DMSetFromOptions(bdm));
4777     PetscCall(DMPlexGenerate(bdm, NULL, interpolate, &dmnew));
4778     PetscCall(DMDestroy(&bdm));
4779     PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
4780     PetscCall(DMPlexReplace_Internal(dm, &dmnew));
4781   } else if (strflg) {
4782     DM          dmnew;
4783     PetscViewer viewer;
4784     const char *contents;
4785     char       *strname;
4786     char        tmpdir[PETSC_MAX_PATH_LEN];
4787     char        tmpfilename[PETSC_MAX_PATH_LEN];
4788     char        name[PETSC_MAX_PATH_LEN];
4789     MPI_Comm    comm;
4790     PetscMPIInt rank;
4791 
4792     PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
4793     PetscCallMPI(MPI_Comm_rank(comm, &rank));
4794     PetscCall(PetscStrchr(filename, ':', &strname));
4795     PetscCheck(strname, comm, PETSC_ERR_ARG_WRONG, "File contents must have the form \"ext:string_name\", not %s", filename);
4796     strname[0] = '\0';
4797     ++strname;
4798     PetscCall(PetscDLSym(NULL, strname, (void **)&contents));
4799     PetscCheck(contents, comm, PETSC_ERR_ARG_WRONG, "Could not locate mesh string %s", strname);
4800     PetscCall(PetscGetTmp(comm, tmpdir, PETSC_MAX_PATH_LEN));
4801     PetscCall(PetscStrlcat(tmpdir, "/meshXXXXXX", PETSC_MAX_PATH_LEN));
4802     PetscCall(PetscMkdtemp(tmpdir));
4803     PetscCall(PetscSNPrintf(tmpfilename, PETSC_MAX_PATH_LEN, "%s/mesh.%s", tmpdir, filename));
4804     PetscCall(PetscViewerASCIIOpen(comm, tmpfilename, &viewer));
4805     PetscCall(PetscViewerASCIIPrintf(viewer, "%s\n", contents));
4806     PetscCall(PetscViewerDestroy(&viewer));
4807     PetscCall(DMPlexCreateFromFile(PetscObjectComm((PetscObject)dm), tmpfilename, plexname, interpolate, &dmnew));
4808     PetscCall(PetscRMTree(tmpdir));
4809     PetscCall(PetscSNPrintf(name, PETSC_MAX_PATH_LEN, "%s Mesh", strname));
4810     PetscCall(PetscObjectSetName((PetscObject)dm, name));
4811     PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
4812     PetscCall(DMPlexReplace_Internal(dm, &dmnew));
4813   } else {
4814     PetscCall(PetscObjectSetName((PetscObject)dm, DMPlexShapes[shape]));
4815     switch (shape) {
4816     case DM_SHAPE_BOX:
4817     case DM_SHAPE_ZBOX:
4818     case DM_SHAPE_ANNULUS: {
4819       PetscInt       faces[3]  = {0, 0, 0};
4820       PetscReal      lower[3]  = {0, 0, 0};
4821       PetscReal      upper[3]  = {1, 1, 1};
4822       DMBoundaryType bdt[3]    = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
4823       PetscBool      isAnnular = shape == DM_SHAPE_ANNULUS ? PETSC_TRUE : PETSC_FALSE;
4824       PetscInt       i, n;
4825 
4826       n = dim;
4827       for (i = 0; i < dim; ++i) faces[i] = (dim == 1 ? 1 : 4 - dim);
4828       PetscCall(PetscOptionsIntArray("-dm_plex_box_faces", "Number of faces along each dimension", "", faces, &n, &flg));
4829       n = 3;
4830       PetscCall(PetscOptionsRealArray("-dm_plex_box_lower", "Lower left corner of box", "", lower, &n, &flg));
4831       PetscCheck(!flg || !(n != dim), comm, PETSC_ERR_ARG_SIZ, "Lower box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
4832       n = 3;
4833       PetscCall(PetscOptionsRealArray("-dm_plex_box_upper", "Upper right corner of box", "", upper, &n, &flg));
4834       PetscCheck(!flg || !(n != dim), comm, PETSC_ERR_ARG_SIZ, "Upper box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
4835       n = 3;
4836       PetscCall(PetscOptionsEnumArray("-dm_plex_box_bd", "Boundary type for each dimension", "", DMBoundaryTypes, (PetscEnum *)bdt, &n, &flg));
4837       PetscCheck(!flg || !(n != dim), comm, PETSC_ERR_ARG_SIZ, "Box boundary types had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
4838 
4839       PetscCheck(!isAnnular || dim == 2, comm, PETSC_ERR_ARG_OUTOFRANGE, "Only two dimensional annuli have been implemented");
4840       if (isAnnular)
4841         for (i = 0; i < dim - 1; ++i) bdt[i] = DM_BOUNDARY_PERIODIC;
4842 
4843       switch (cell) {
4844       case DM_POLYTOPE_TRI_PRISM_TENSOR:
4845         PetscCall(DMPlexCreateWedgeBoxMesh_Internal(dm, faces, lower, upper, bdt));
4846         if (!interpolate) {
4847           DM udm;
4848 
4849           PetscCall(DMPlexUninterpolate(dm, &udm));
4850           PetscCall(DMPlexReplace_Internal(dm, &udm));
4851         }
4852         break;
4853       default:
4854         PetscCall(DMPlexCreateBoxMesh_Internal(dm, shape, dim, simplex, faces, lower, upper, bdt, interpolate));
4855         break;
4856       }
4857       if (isAnnular) {
4858         DM          cdm;
4859         PetscDS     cds;
4860         PetscScalar bounds[2] = {lower[0], upper[0]};
4861 
4862         // Fix coordinates for annular region
4863         PetscCall(DMSetPeriodicity(dm, NULL, NULL, NULL));
4864         PetscCall(DMSetCellCoordinatesLocal(dm, NULL));
4865         PetscCall(DMSetCellCoordinates(dm, NULL));
4866         PetscCall(DMPlexCreateCoordinateSpace(dm, 1, PETSC_TRUE, NULL));
4867         PetscCall(DMGetCoordinateDM(dm, &cdm));
4868         PetscCall(DMGetDS(cdm, &cds));
4869         PetscCall(PetscDSSetConstants(cds, 2, bounds));
4870         PetscCall(DMPlexRemapGeometry(dm, 0.0, boxToAnnulus));
4871       }
4872     } break;
4873     case DM_SHAPE_BOX_SURFACE: {
4874       PetscInt  faces[3] = {0, 0, 0};
4875       PetscReal lower[3] = {0, 0, 0};
4876       PetscReal upper[3] = {1, 1, 1};
4877       PetscInt  i, n;
4878 
4879       n = dim + 1;
4880       for (i = 0; i < dim + 1; ++i) faces[i] = (dim + 1 == 1 ? 1 : 4 - (dim + 1));
4881       PetscCall(PetscOptionsIntArray("-dm_plex_box_faces", "Number of faces along each dimension", "", faces, &n, &flg));
4882       n = 3;
4883       PetscCall(PetscOptionsRealArray("-dm_plex_box_lower", "Lower left corner of box", "", lower, &n, &flg));
4884       PetscCheck(!flg || !(n != dim + 1), comm, PETSC_ERR_ARG_SIZ, "Lower box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim + 1);
4885       n = 3;
4886       PetscCall(PetscOptionsRealArray("-dm_plex_box_upper", "Upper right corner of box", "", upper, &n, &flg));
4887       PetscCheck(!flg || !(n != dim + 1), comm, PETSC_ERR_ARG_SIZ, "Upper box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim + 1);
4888       PetscCall(DMPlexCreateBoxSurfaceMesh_Internal(dm, dim + 1, faces, lower, upper, interpolate));
4889     } break;
4890     case DM_SHAPE_SPHERE: {
4891       PetscReal R = 1.0;
4892 
4893       PetscCall(PetscOptionsReal("-dm_plex_sphere_radius", "Radius of the sphere", "", R, &R, &flg));
4894       PetscCall(DMPlexCreateSphereMesh_Internal(dm, dim, simplex, R));
4895     } break;
4896     case DM_SHAPE_BALL: {
4897       PetscReal R = 1.0;
4898 
4899       PetscCall(PetscOptionsReal("-dm_plex_ball_radius", "Radius of the ball", "", R, &R, &flg));
4900       PetscCall(DMPlexCreateBallMesh_Internal(dm, dim, R));
4901     } break;
4902     case DM_SHAPE_CYLINDER: {
4903       DMBoundaryType bdt = DM_BOUNDARY_NONE;
4904       PetscInt       Nw  = 6;
4905       PetscInt       Nr  = 0;
4906 
4907       PetscCall(PetscOptionsEnum("-dm_plex_cylinder_bd", "Boundary type in the z direction", "", DMBoundaryTypes, (PetscEnum)bdt, (PetscEnum *)&bdt, NULL));
4908       PetscCall(PetscOptionsInt("-dm_plex_cylinder_num_wedges", "Number of wedges around the cylinder", "", Nw, &Nw, NULL));
4909       PetscCall(PetscOptionsInt("-dm_plex_cylinder_num_refine", "Number of refinements before projection", "", Nr, &Nr, NULL));
4910       switch (cell) {
4911       case DM_POLYTOPE_TRI_PRISM_TENSOR:
4912         PetscCall(DMPlexCreateWedgeCylinderMesh_Internal(dm, Nw, interpolate));
4913         break;
4914       default:
4915         PetscCall(DMPlexCreateHexCylinderMesh_Internal(dm, bdt, Nr));
4916         break;
4917       }
4918     } break;
4919     case DM_SHAPE_SCHWARZ_P: // fallthrough
4920     case DM_SHAPE_GYROID: {
4921       PetscInt       extent[3] = {1, 1, 1}, refine = 0, layers = 0, three;
4922       PetscReal      thickness   = 0.;
4923       DMBoundaryType periodic[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
4924       DMPlexTPSType  tps_type    = shape == DM_SHAPE_SCHWARZ_P ? DMPLEX_TPS_SCHWARZ_P : DMPLEX_TPS_GYROID;
4925       PetscBool      tps_distribute;
4926       PetscCall(PetscOptionsIntArray("-dm_plex_tps_extent", "Number of replicas for each of three dimensions", NULL, extent, (three = 3, &three), NULL));
4927       PetscCall(PetscOptionsInt("-dm_plex_tps_refine", "Number of refinements", NULL, refine, &refine, NULL));
4928       PetscCall(PetscOptionsEnumArray("-dm_plex_tps_periodic", "Periodicity in each of three dimensions", NULL, DMBoundaryTypes, (PetscEnum *)periodic, (three = 3, &three), NULL));
4929       PetscCall(PetscOptionsInt("-dm_plex_tps_layers", "Number of layers in volumetric extrusion (or zero to not extrude)", NULL, layers, &layers, NULL));
4930       PetscCall(PetscOptionsReal("-dm_plex_tps_thickness", "Thickness of volumetric extrusion", NULL, thickness, &thickness, NULL));
4931       PetscCall(DMPlexDistributeGetDefault(dm, &tps_distribute));
4932       PetscCall(PetscOptionsBool("-dm_plex_tps_distribute", "Distribute the 2D mesh prior to refinement and extrusion", NULL, tps_distribute, &tps_distribute, NULL));
4933       PetscCall(DMPlexCreateTPSMesh_Internal(dm, tps_type, extent, periodic, tps_distribute, refine, layers, thickness));
4934     } break;
4935     case DM_SHAPE_DOUBLET: {
4936       DM        dmnew;
4937       PetscReal rl = 0.0;
4938 
4939       PetscCall(PetscOptionsReal("-dm_plex_doublet_refinementlimit", "Refinement limit", NULL, rl, &rl, NULL));
4940       PetscCall(DMPlexCreateDoublet(PetscObjectComm((PetscObject)dm), dim, simplex, interpolate, rl, &dmnew));
4941       PetscCall(DMPlexCopy_Internal(dm, PETSC_FALSE, PETSC_FALSE, dmnew));
4942       PetscCall(DMPlexReplace_Internal(dm, &dmnew));
4943     } break;
4944     case DM_SHAPE_HYPERCUBIC: {
4945       PetscInt       *edges, overlap = 1;
4946       PetscReal      *lower, *upper;
4947       DMBoundaryType *bdt;
4948       PetscInt        n, d;
4949 
4950       *useCoordSpace = PETSC_FALSE;
4951       PetscCall(PetscMalloc4(dim, &edges, dim, &lower, dim, &upper, dim, &bdt));
4952       for (d = 0; d < dim; ++d) {
4953         edges[d] = 1;
4954         lower[d] = 0.;
4955         upper[d] = 1.;
4956         bdt[d]   = DM_BOUNDARY_PERIODIC;
4957       }
4958       n = dim;
4959       PetscCall(PetscOptionsIntArray("-dm_plex_box_faces", "Number of faces along each dimension", "", edges, &n, &flg));
4960       n = dim;
4961       PetscCall(PetscOptionsRealArray("-dm_plex_box_lower", "Lower left corner of box", "", lower, &n, &flg));
4962       PetscCheck(!flg || n == dim, comm, PETSC_ERR_ARG_SIZ, "Lower box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
4963       n = dim;
4964       PetscCall(PetscOptionsRealArray("-dm_plex_box_upper", "Upper right corner of box", "", upper, &n, &flg));
4965       PetscCheck(!flg || n == dim, comm, PETSC_ERR_ARG_SIZ, "Upper box point had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
4966       n = dim;
4967       PetscCall(PetscOptionsEnumArray("-dm_plex_box_bd", "Boundary type for each dimension", "", DMBoundaryTypes, (PetscEnum *)bdt, &n, &flg));
4968       PetscCheck(!flg || n == dim, comm, PETSC_ERR_ARG_SIZ, "Box boundary types had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
4969       PetscCall(PetscOptionsBoundedInt("-dm_distribute_overlap", "The size of the overlap halo", "DMPlexDistribute", overlap, &overlap, NULL, 0));
4970       PetscCall(DMPlexCreateHypercubicMesh_Internal(dm, dim, lower, upper, edges, overlap, bdt));
4971       PetscCall(PetscFree4(edges, lower, upper, bdt));
4972     } break;
4973     default:
4974       SETERRQ(comm, PETSC_ERR_SUP, "Domain shape %s is unsupported", DMPlexShapes[shape]);
4975     }
4976   }
4977   PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
4978   if (!((PetscObject)dm)->name && nameflg) PetscCall(PetscObjectSetName((PetscObject)dm, plexname));
4979   if (orient) PetscCall(DMPlexOrient(dm));
4980   // Allow label creation
4981   PetscCall(PetscOptionsFindPairPrefix_Private(NULL, ((PetscObject)dm)->prefix, "-dm_plex_label_", &option, NULL, &flg));
4982   if (flg) {
4983     DMLabel     label;
4984     PetscInt    points[1024], n = 1024;
4985     char        fulloption[PETSC_MAX_PATH_LEN];
4986     const char *name = &option[14];
4987 
4988     PetscCall(DMCreateLabel(dm, name));
4989     PetscCall(DMGetLabel(dm, name, &label));
4990     fulloption[0] = '-';
4991     fulloption[1] = 0;
4992     PetscCall(PetscStrlcat(fulloption, option, PETSC_MAX_PATH_LEN));
4993     PetscCall(PetscOptionsGetIntArray(NULL, ((PetscObject)dm)->prefix, fulloption, points, &n, NULL));
4994     for (PetscInt p = 0; p < n; ++p) PetscCall(DMLabelSetValue(label, points[p], 1));
4995   }
4996   // Allow cohesive label creation
4997   //   Faces are input, completed, and all points are marked with their depth
4998   PetscCall(PetscOptionsFindPairPrefix_Private(NULL, ((PetscObject)dm)->prefix, "-dm_plex_cohesive_label_", &option, NULL, &flg));
4999   if (flg) {
5000     DMLabel   label;
5001     PetscInt  points[1024], n, pStart, pEnd, Nl = 1;
5002     PetscBool noCreate = PETSC_FALSE;
5003     char      fulloption[PETSC_MAX_PATH_LEN];
5004     char      name[PETSC_MAX_PATH_LEN];
5005     size_t    len;
5006 
5007     PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
5008     PetscCall(PetscStrncpy(name, &option[23], PETSC_MAX_PATH_LEN));
5009     PetscCall(PetscStrlen(name, &len));
5010     if (name[len - 1] == '0') Nl = 10;
5011     for (PetscInt l = 0; l < Nl; ++l) {
5012       if (l > 0) name[len - 1] = (char)('0' + l);
5013       fulloption[0] = 0;
5014       PetscCall(PetscStrlcat(fulloption, "-dm_plex_cohesive_label_", 32));
5015       PetscCall(PetscStrlcat(fulloption, name, PETSC_MAX_PATH_LEN - 32));
5016       n = 1024;
5017       PetscCall(PetscOptionsGetIntArray(NULL, ((PetscObject)dm)->prefix, fulloption, points, &n, &flg));
5018       if (!flg) break;
5019       PetscCall(DMHasLabel(dm, name, &noCreate));
5020       if (noCreate) {
5021         DMLabel         inlabel;
5022         IS              pointIS;
5023         const PetscInt *lpoints;
5024         PetscInt        pdep, ln, inval = points[0];
5025         char            newname[PETSC_MAX_PATH_LEN];
5026 
5027         PetscCheck(n == 1, comm, PETSC_ERR_ARG_WRONG, "Must specify a label value with this option");
5028         PetscCall(DMGetLabel(dm, name, &inlabel));
5029         PetscCall(DMLabelGetStratumIS(inlabel, inval, &pointIS));
5030         PetscCall(ISGetLocalSize(pointIS, &ln));
5031         PetscCall(ISGetIndices(pointIS, &lpoints));
5032         PetscCall(DMPlexGetPointDepth(dm, lpoints[0], &pdep));
5033         PetscCall(PetscSNPrintf(newname, PETSC_MAX_PATH_LEN, "%s%" PetscInt_FMT, name, points[0]));
5034         PetscCall(DMCreateLabel(dm, newname));
5035         PetscCall(DMGetLabel(dm, newname, &label));
5036         if (!pdep) PetscCall(ProcessCohesiveLabel_Vertices(dm, label, inlabel, inval, ln, lpoints));
5037         else PetscCall(ProcessCohesiveLabel_Faces(dm, label, ln, lpoints));
5038         PetscCall(ISRestoreIndices(pointIS, &lpoints));
5039         PetscCall(ISDestroy(&pointIS));
5040       } else {
5041         PetscCall(DMCreateLabel(dm, name));
5042         PetscCall(DMGetLabel(dm, name, &label));
5043         if (pStart >= pEnd) n = 0;
5044         PetscCall(ProcessCohesiveLabel_Faces(dm, label, n, points));
5045       }
5046       PetscCall(DMPlexOrientLabel(dm, label));
5047       PetscCall(DMPlexLabelCohesiveComplete(dm, label, NULL, 1, PETSC_FALSE, PETSC_FALSE, NULL));
5048     }
5049   }
5050   PetscCall(DMViewFromOptions(dm, NULL, "-created_dm_view"));
5051   PetscCall(PetscLogEventEnd(DMPLEX_CreateFromOptions, dm, 0, 0, 0));
5052   PetscFunctionReturn(PETSC_SUCCESS);
5053 }
5054 
5055 PetscErrorCode DMSetFromOptions_NonRefinement_Plex(DM dm, PetscOptionItems PetscOptionsObject)
5056 {
5057   DM_Plex  *mesh = (DM_Plex *)dm->data;
5058   PetscBool flg, flg2;
5059   char      bdLabel[PETSC_MAX_PATH_LEN];
5060   char      method[PETSC_MAX_PATH_LEN];
5061 
5062   PetscFunctionBegin;
5063   /* Handle viewing */
5064   PetscCall(PetscOptionsBool("-dm_plex_print_set_values", "Output all set values info", "DMPlexMatSetClosure", PETSC_FALSE, &mesh->printSetValues, NULL));
5065   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_fem", "Debug output level for all fem computations", "DMPlexSNESComputeResidualFEM", 0, &mesh->printFEM, NULL, 0));
5066   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_fvm", "Debug output level for all fvm computations", "DMPlexSNESComputeResidualFVM", 0, &mesh->printFVM, NULL, 0));
5067   PetscCall(PetscOptionsReal("-dm_plex_print_tol", "Tolerance for FEM output", "DMPlexSNESComputeResidualFEM", mesh->printTol, &mesh->printTol, NULL));
5068   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_l2", "Debug output level all L2 diff computations", "DMComputeL2Diff", 0, &mesh->printL2, NULL, 0));
5069   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_locate", "Debug output level all point location computations", "DMLocatePoints", 0, &mesh->printLocate, NULL, 0));
5070   PetscCall(PetscOptionsBoundedInt("-dm_plex_print_project", "Debug output level all projection computations", "DMPlexProject", 0, &mesh->printProject, NULL, 0));
5071   PetscCall(DMMonitorSetFromOptions(dm, "-dm_plex_monitor_throughput", "Monitor the simulation throughput", "DMPlexMonitorThroughput", DMPlexMonitorThroughput, NULL, &flg));
5072   if (flg) PetscCall(PetscLogDefaultBegin());
5073   // Interpolation
5074   PetscCall(PetscOptionsBool("-dm_plex_interpolate_prefer_tensor", "When different orderings exist, prefer the tensor order", "DMPlexSetInterpolationPreferTensor", mesh->interpolatePreferTensor, &mesh->interpolatePreferTensor, NULL));
5075   /* Labeling */
5076   PetscCall(PetscOptionsString("-dm_plex_boundary_label", "Label to mark the mesh boundary", "", bdLabel, bdLabel, sizeof(bdLabel), &flg));
5077   if (flg) PetscCall(DMPlexCreateBoundaryLabel_Private(dm, bdLabel));
5078   /* Point Location */
5079   PetscCall(PetscOptionsBool("-dm_plex_hash_location", "Use grid hashing for point location", "DMInterpolate", PETSC_FALSE, &mesh->useHashLocation, NULL));
5080   /* Partitioning and distribution */
5081   PetscCall(PetscOptionsBool("-dm_plex_partition_balance", "Attempt to evenly divide points on partition boundary between processes", "DMPlexSetPartitionBalance", PETSC_FALSE, &mesh->partitionBalance, NULL));
5082   /* Reordering */
5083   PetscCall(PetscOptionsBool("-dm_reorder_section", "Compute point permutation for local section", "DMReorderSectionSetDefault", PETSC_FALSE, &flg2, &flg));
5084   if (flg) PetscCall(DMReorderSectionSetDefault(dm, flg2 ? DM_REORDER_DEFAULT_TRUE : DM_REORDER_DEFAULT_FALSE));
5085   PetscCall(PetscOptionsString("-dm_reorder_section_type", "Reordering method for local section", "DMReorderSectionSetType", method, method, PETSC_MAX_PATH_LEN, &flg));
5086   if (flg) PetscCall(DMReorderSectionSetType(dm, method));
5087   /* Generation and remeshing */
5088   PetscCall(PetscOptionsBool("-dm_plex_remesh_bd", "Allow changes to the boundary on remeshing", "DMAdapt", PETSC_FALSE, &mesh->remeshBd, NULL));
5089   /* Projection behavior */
5090   PetscCall(PetscOptionsBoundedInt("-dm_plex_max_projection_height", "Maximum mesh point height used to project locally", "DMPlexSetMaxProjectionHeight", 0, &mesh->maxProjectionHeight, NULL, 0));
5091   PetscCall(PetscOptionsBool("-dm_plex_regular_refinement", "Use special nested projection algorithm for regular refinement", "DMPlexSetRegularRefinement", mesh->regularRefinement, &mesh->regularRefinement, NULL));
5092   /* Checking structure */
5093   {
5094     PetscBool all = PETSC_FALSE;
5095 
5096     PetscCall(PetscOptionsBool("-dm_plex_check_all", "Perform all basic checks", "DMPlexCheck", PETSC_FALSE, &all, NULL));
5097     if (all) {
5098       PetscCall(DMPlexCheck(dm));
5099     } else {
5100       PetscCall(PetscOptionsBool("-dm_plex_check_symmetry", "Check that the adjacency information in the mesh is symmetric", "DMPlexCheckSymmetry", PETSC_FALSE, &flg, &flg2));
5101       if (flg && flg2) PetscCall(DMPlexCheckSymmetry(dm));
5102       PetscCall(PetscOptionsBool("-dm_plex_check_skeleton", "Check that each cell has the correct number of vertices (only for homogeneous simplex or tensor meshes)", "DMPlexCheckSkeleton", PETSC_FALSE, &flg, &flg2));
5103       if (flg && flg2) PetscCall(DMPlexCheckSkeleton(dm, 0));
5104       PetscCall(PetscOptionsBool("-dm_plex_check_faces", "Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type", "DMPlexCheckFaces", PETSC_FALSE, &flg, &flg2));
5105       if (flg && flg2) PetscCall(DMPlexCheckFaces(dm, 0));
5106       PetscCall(PetscOptionsBool("-dm_plex_check_geometry", "Check that cells have positive volume", "DMPlexCheckGeometry", PETSC_FALSE, &flg, &flg2));
5107       if (flg && flg2) PetscCall(DMPlexCheckGeometry(dm));
5108       PetscCall(PetscOptionsBool("-dm_plex_check_pointsf", "Check some necessary conditions for PointSF", "DMPlexCheckPointSF", PETSC_FALSE, &flg, &flg2));
5109       if (flg && flg2) PetscCall(DMPlexCheckPointSF(dm, NULL, PETSC_FALSE));
5110       PetscCall(PetscOptionsBool("-dm_plex_check_interface_cones", "Check points on inter-partition interfaces have conforming order of cone points", "DMPlexCheckInterfaceCones", PETSC_FALSE, &flg, &flg2));
5111       if (flg && flg2) PetscCall(DMPlexCheckInterfaceCones(dm));
5112     }
5113     PetscCall(PetscOptionsBool("-dm_plex_check_cell_shape", "Check cell shape", "DMPlexCheckCellShape", PETSC_FALSE, &flg, &flg2));
5114     if (flg && flg2) PetscCall(DMPlexCheckCellShape(dm, PETSC_TRUE, PETSC_DETERMINE));
5115   }
5116   {
5117     PetscReal scale = 1.0;
5118 
5119     PetscCall(PetscOptionsReal("-dm_plex_scale", "Scale factor for mesh coordinates", "DMPlexScale", scale, &scale, &flg));
5120     if (flg) {
5121       Vec coordinates, coordinatesLocal;
5122 
5123       PetscCall(DMGetCoordinates(dm, &coordinates));
5124       PetscCall(DMGetCoordinatesLocal(dm, &coordinatesLocal));
5125       PetscCall(VecScale(coordinates, scale));
5126       PetscCall(VecScale(coordinatesLocal, scale));
5127     }
5128   }
5129   PetscCall(PetscPartitionerSetFromOptions(mesh->partitioner));
5130   PetscFunctionReturn(PETSC_SUCCESS);
5131 }
5132 
5133 PetscErrorCode DMSetFromOptions_Overlap_Plex(DM dm, PetscOptionItems PetscOptionsObject, PetscInt *overlap)
5134 {
5135   PetscInt  numOvLabels = 16, numOvExLabels = 16;
5136   char     *ovLabelNames[16], *ovExLabelNames[16];
5137   PetscInt  numOvValues = 16, numOvExValues = 16, l;
5138   PetscBool flg;
5139 
5140   PetscFunctionBegin;
5141   PetscCall(PetscOptionsBoundedInt("-dm_distribute_overlap", "The size of the overlap halo", "DMPlexDistribute", *overlap, overlap, NULL, 0));
5142   PetscCall(PetscOptionsStringArray("-dm_distribute_overlap_labels", "List of overlap label names", "DMPlexDistribute", ovLabelNames, &numOvLabels, &flg));
5143   if (!flg) numOvLabels = 0;
5144   if (numOvLabels) {
5145     ((DM_Plex *)dm->data)->numOvLabels = numOvLabels;
5146     for (l = 0; l < numOvLabels; ++l) {
5147       PetscCall(DMGetLabel(dm, ovLabelNames[l], &((DM_Plex *)dm->data)->ovLabels[l]));
5148       PetscCheck(((DM_Plex *)dm->data)->ovLabels[l], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid label name %s", ovLabelNames[l]);
5149       PetscCall(PetscFree(ovLabelNames[l]));
5150     }
5151     PetscCall(PetscOptionsIntArray("-dm_distribute_overlap_values", "List of overlap label values", "DMPlexDistribute", ((DM_Plex *)dm->data)->ovValues, &numOvValues, &flg));
5152     if (!flg) numOvValues = 0;
5153     PetscCheck(numOvLabels == numOvValues, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "The number of labels %" PetscInt_FMT " must match the number of values %" PetscInt_FMT, numOvLabels, numOvValues);
5154 
5155     PetscCall(PetscOptionsStringArray("-dm_distribute_overlap_exclude_labels", "List of overlap exclude label names", "DMPlexDistribute", ovExLabelNames, &numOvExLabels, &flg));
5156     if (!flg) numOvExLabels = 0;
5157     ((DM_Plex *)dm->data)->numOvExLabels = numOvExLabels;
5158     for (l = 0; l < numOvExLabels; ++l) {
5159       PetscCall(DMGetLabel(dm, ovExLabelNames[l], &((DM_Plex *)dm->data)->ovExLabels[l]));
5160       PetscCheck(((DM_Plex *)dm->data)->ovExLabels[l], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid label name %s", ovExLabelNames[l]);
5161       PetscCall(PetscFree(ovExLabelNames[l]));
5162     }
5163     PetscCall(PetscOptionsIntArray("-dm_distribute_overlap_exclude_values", "List of overlap exclude label values", "DMPlexDistribute", ((DM_Plex *)dm->data)->ovExValues, &numOvExValues, &flg));
5164     if (!flg) numOvExValues = 0;
5165     PetscCheck(numOvExLabels == numOvExValues, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "The number of exclude labels %" PetscInt_FMT " must match the number of values %" PetscInt_FMT, numOvExLabels, numOvExValues);
5166   }
5167   PetscFunctionReturn(PETSC_SUCCESS);
5168 }
5169 
5170 static PetscErrorCode DMSetFromOptions_Plex(DM dm, PetscOptionItems PetscOptionsObject)
5171 {
5172   PetscFunctionList    ordlist;
5173   char                 oname[256];
5174   char                 sublabelname[PETSC_MAX_PATH_LEN] = "";
5175   DMReorderDefaultFlag reorder;
5176   PetscReal            volume    = -1.0;
5177   PetscInt             prerefine = 0, refine = 0, r, coarsen = 0, overlap = 0, extLayers = 0, dim;
5178   PetscBool            uniformOrig = PETSC_FALSE, uniform = PETSC_TRUE, distribute, saveSF = PETSC_FALSE, interpolate = PETSC_TRUE, coordSpace = PETSC_FALSE, remap = PETSC_TRUE, ghostCells = PETSC_FALSE, isHierarchy, flg;
5179 
5180   PetscFunctionBegin;
5181   PetscOptionsHeadBegin(PetscOptionsObject, "DMPlex Options");
5182   if (dm->cloneOpts) goto non_refine;
5183   /* Handle automatic creation */
5184   PetscCall(DMGetDimension(dm, &dim));
5185   if (dim < 0) PetscCall(DMPlexCreateFromOptions_Internal(PetscOptionsObject, &coordSpace, dm));
5186   PetscCall(DMGetDimension(dm, &dim));
5187   /* Handle interpolation before distribution */
5188   PetscCall(PetscOptionsBool("-dm_plex_interpolate_pre", "Flag to interpolate mesh before distribution", "", interpolate, &interpolate, &flg));
5189   if (flg) {
5190     DMPlexInterpolatedFlag interpolated;
5191 
5192     PetscCall(DMPlexIsInterpolated(dm, &interpolated));
5193     if (interpolated == DMPLEX_INTERPOLATED_FULL && !interpolate) {
5194       DM udm;
5195 
5196       PetscCall(DMPlexUninterpolate(dm, &udm));
5197       PetscCall(DMPlexReplace_Internal(dm, &udm));
5198     } else if (interpolated != DMPLEX_INTERPOLATED_FULL && interpolate) {
5199       DM idm;
5200 
5201       PetscCall(DMPlexInterpolate(dm, &idm));
5202       PetscCall(DMPlexReplace_Internal(dm, &idm));
5203     }
5204   }
5205   // Handle submesh selection before distribution
5206   PetscCall(PetscOptionsString("-dm_plex_submesh", "Label to use for submesh selection", "", sublabelname, sublabelname, PETSC_MAX_PATH_LEN, &flg));
5207   if (flg) {
5208     DM              subdm;
5209     DMLabel         label;
5210     IS              valueIS, pointIS;
5211     const PetscInt *values, *points;
5212     PetscBool       markedFaces = PETSC_FALSE;
5213     PetscInt        Nv, value, Np;
5214 
5215     PetscCall(DMGetLabel(dm, sublabelname, &label));
5216     PetscCall(DMLabelGetNumValues(label, &Nv));
5217     PetscCheck(Nv == 1, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Only a single label value is currently supported for submesh selection, not %" PetscInt_FMT, Nv);
5218     PetscCall(DMLabelGetValueIS(label, &valueIS));
5219     PetscCall(ISGetIndices(valueIS, &values));
5220     value = values[0];
5221     PetscCall(ISRestoreIndices(valueIS, &values));
5222     PetscCall(ISDestroy(&valueIS));
5223     PetscCall(DMLabelGetStratumSize(label, value, &Np));
5224     PetscCall(DMLabelGetStratumIS(label, value, &pointIS));
5225     PetscCall(ISGetIndices(pointIS, &points));
5226     for (PetscInt p = 0; p < Np; ++p) {
5227       PetscInt pdepth;
5228 
5229       PetscCall(DMPlexGetPointDepth(dm, points[p], &pdepth));
5230       if (pdepth) {
5231         markedFaces = PETSC_TRUE;
5232         break;
5233       }
5234     }
5235     PetscCall(ISRestoreIndices(pointIS, &points));
5236     PetscCall(ISDestroy(&pointIS));
5237     PetscCall(DMPlexCreateSubmesh(dm, label, value, markedFaces, &subdm));
5238     PetscCall(DMPlexReplace_Internal(dm, &subdm));
5239     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
5240   }
5241   /* Handle DMPlex refinement before distribution */
5242   PetscCall(DMPlexGetRefinementUniform(dm, &uniformOrig));
5243   PetscCall(PetscOptionsBoundedInt("-dm_refine_pre", "The number of refinements before distribution", "DMCreate", prerefine, &prerefine, NULL, 0));
5244   PetscCall(PetscOptionsBool("-dm_refine_remap_pre", "Flag to control coordinate remapping", "DMCreate", remap, &remap, NULL));
5245   PetscCall(PetscOptionsBool("-dm_refine_uniform_pre", "Flag for uniform refinement before distribution", "DMCreate", uniform, &uniform, &flg));
5246   if (flg) PetscCall(DMPlexSetRefinementUniform(dm, uniform));
5247   PetscCall(PetscOptionsReal("-dm_refine_volume_limit_pre", "The maximum cell volume after refinement before distribution", "DMCreate", volume, &volume, &flg));
5248   if (flg) {
5249     PetscCall(DMPlexSetRefinementUniform(dm, PETSC_FALSE));
5250     PetscCall(DMPlexSetRefinementLimit(dm, volume));
5251     prerefine = PetscMax(prerefine, 1);
5252   }
5253   if (prerefine) PetscCall(DMLocalizeCoordinates(dm));
5254   for (r = 0; r < prerefine; ++r) {
5255     DM             rdm;
5256     PetscPointFunc coordFunc = ((DM_Plex *)dm->data)->coordFunc;
5257 
5258     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
5259     PetscCall(DMRefine(dm, PetscObjectComm((PetscObject)dm), &rdm));
5260     PetscCall(DMPlexReplace_Internal(dm, &rdm));
5261     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
5262     if (coordFunc && remap) {
5263       PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
5264       ((DM_Plex *)dm->data)->coordFunc = coordFunc;
5265     }
5266   }
5267   PetscCall(DMPlexSetRefinementUniform(dm, uniformOrig));
5268   /* Handle DMPlex extrusion before distribution */
5269   PetscCall(PetscOptionsBoundedInt("-dm_extrude", "The number of layers to extrude", "", extLayers, &extLayers, NULL, 0));
5270   if (extLayers) {
5271     DM edm;
5272 
5273     PetscCall(DMExtrude(dm, extLayers, &edm));
5274     PetscCall(DMPlexReplace_Internal(dm, &edm));
5275     ((DM_Plex *)dm->data)->coordFunc = NULL;
5276     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
5277     extLayers = 0;
5278     PetscCall(DMGetDimension(dm, &dim));
5279   }
5280   /* Handle DMPlex reordering before distribution */
5281   PetscCall(DMPlexReorderGetDefault(dm, &reorder));
5282   PetscCall(MatGetOrderingList(&ordlist));
5283   PetscCall(PetscStrncpy(oname, MATORDERINGNATURAL, sizeof(oname)));
5284   PetscCall(PetscOptionsFList("-dm_plex_reorder", "Set mesh reordering type", "DMPlexGetOrdering", ordlist, MATORDERINGNATURAL, oname, sizeof(oname), &flg));
5285   if (reorder == DM_REORDER_DEFAULT_TRUE || flg) {
5286     DM pdm;
5287     IS perm;
5288 
5289     PetscCall(DMPlexGetOrdering(dm, oname, NULL, &perm));
5290     PetscCall(DMPlexPermute(dm, perm, &pdm));
5291     PetscCall(ISDestroy(&perm));
5292     PetscCall(DMPlexReplace_Internal(dm, &pdm));
5293     PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
5294   }
5295   /* Handle DMPlex distribution */
5296   PetscCall(DMPlexDistributeGetDefault(dm, &distribute));
5297   PetscCall(PetscOptionsBool("-dm_distribute", "Flag to redistribute a mesh among processes", "DMPlexDistribute", distribute, &distribute, NULL));
5298   PetscCall(PetscOptionsBool("-dm_distribute_save_sf", "Flag to save the migration SF", "DMPlexSetMigrationSF", saveSF, &saveSF, NULL));
5299   PetscCall(DMSetFromOptions_Overlap_Plex(dm, PetscOptionsObject, &overlap));
5300   if (distribute) {
5301     DM               pdm = NULL;
5302     PetscPartitioner part;
5303     PetscSF          sfMigration;
5304 
5305     PetscCall(DMPlexGetPartitioner(dm, &part));
5306     PetscCall(PetscPartitionerSetFromOptions(part));
5307     PetscCall(DMPlexDistribute(dm, overlap, &sfMigration, &pdm));
5308     if (pdm) {
5309       // Delete the local section to force the existing one to be rebuilt with the distributed DM
5310       PetscCall(DMSetLocalSection(dm, pdm->localSection));
5311       PetscCall(DMPlexReplace_Internal(dm, &pdm));
5312     }
5313     if (saveSF) PetscCall(DMPlexSetMigrationSF(dm, sfMigration));
5314     PetscCall(PetscSFDestroy(&sfMigration));
5315   }
5316 
5317   {
5318     PetscBool useBoxLabel = PETSC_FALSE;
5319     PetscCall(PetscOptionsBool("-dm_plex_box_label", "Create 'Face Sets' assuming boundary faces align with cartesian directions", "DMCreate", useBoxLabel, &useBoxLabel, NULL));
5320     if (useBoxLabel) {
5321       PetscInt       n      = 3;
5322       DMBoundaryType bdt[3] = {DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE};
5323 
5324       PetscCall(PetscOptionsEnumArray("-dm_plex_box_label_bd", "Boundary type for each dimension when using -dm_plex_box_label", "", DMBoundaryTypes, (PetscEnum *)bdt, &n, &flg));
5325       PetscCheck(!flg || !(n != dim), PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_SIZ, "Box boundary types had %" PetscInt_FMT " values, should have been %" PetscInt_FMT, n, dim);
5326       PetscCall(DMPlexSetBoxLabel_Internal(dm, bdt));
5327     }
5328   }
5329   /* Must check CEED options before creating function space for coordinates */
5330   {
5331     PetscBool useCeed = PETSC_FALSE, flg;
5332 
5333     PetscCall(PetscOptionsBool("-dm_plex_use_ceed", "Use LibCEED as the FEM backend", "DMPlexSetUseCeed", useCeed, &useCeed, &flg));
5334     if (flg) PetscCall(DMPlexSetUseCeed(dm, useCeed));
5335   }
5336   /* Create coordinate space */
5337   PetscCall(PetscOptionsBool("-dm_coord_space", "Use an FEM space for coordinates", "", coordSpace, &coordSpace, NULL));
5338   if (coordSpace) {
5339     DM_Plex  *mesh   = (DM_Plex *)dm->data;
5340     PetscInt  degree = 1, deg;
5341     PetscInt  height = 0;
5342     DM        cdm;
5343     PetscBool localize = PETSC_TRUE, sparseLocalize = PETSC_TRUE;
5344 
5345     PetscCall(PetscOptionsInt("-dm_coord_petscspace_degree", "FEM degree for coordinate space", "", degree, &degree, NULL));
5346     PetscCall(DMGetCoordinateDegree_Internal(dm, &deg));
5347     if (coordSpace && deg <= 1) PetscCall(DMPlexCreateCoordinateSpace(dm, degree, PETSC_TRUE, mesh->coordFunc));
5348     PetscCall(DMGetCoordinateDM(dm, &cdm));
5349     if (!coordSpace) {
5350       PetscDS      cds;
5351       PetscObject  obj;
5352       PetscClassId id;
5353 
5354       PetscCall(DMGetDS(cdm, &cds));
5355       PetscCall(PetscDSGetDiscretization(cds, 0, &obj));
5356       PetscCall(PetscObjectGetClassId(obj, &id));
5357       if (id == PETSCFE_CLASSID) {
5358         PetscContainer dummy;
5359 
5360         PetscCall(PetscContainerCreate(PETSC_COMM_SELF, &dummy));
5361         PetscCall(PetscObjectSetName((PetscObject)dummy, "coordinates"));
5362         PetscCall(DMSetField(cdm, 0, NULL, (PetscObject)dummy));
5363         PetscCall(PetscContainerDestroy(&dummy));
5364         PetscCall(DMClearDS(cdm));
5365       }
5366       mesh->coordFunc = NULL;
5367     }
5368     PetscCall(PetscOptionsBool("-dm_localize", "Localize mesh coordinates", "", localize, &localize, NULL));
5369     PetscCall(PetscOptionsBool("-dm_sparse_localize", "Localize only necessary cells", "DMSetSparseLocalize", sparseLocalize, &sparseLocalize, &flg));
5370     if (flg) PetscCall(DMSetSparseLocalize(dm, sparseLocalize));
5371     PetscCall(PetscOptionsInt("-dm_localize_height", "Localize edges and faces in addition to cells", "", height, &height, &flg));
5372     if (flg) PetscCall(DMPlexSetMaxProjectionHeight(cdm, height));
5373     if (localize) PetscCall(DMLocalizeCoordinates(dm));
5374   }
5375   /* Handle DMPlex refinement */
5376   remap = PETSC_TRUE;
5377   PetscCall(PetscOptionsBoundedInt("-dm_refine", "The number of uniform refinements", "DMCreate", refine, &refine, NULL, 0));
5378   PetscCall(PetscOptionsBool("-dm_refine_remap", "Flag to control coordinate remapping", "DMCreate", remap, &remap, NULL));
5379   PetscCall(PetscOptionsBoundedInt("-dm_refine_hierarchy", "The number of uniform refinements", "DMCreate", refine, &refine, &isHierarchy, 0));
5380   if (refine) PetscCall(DMPlexSetRefinementUniform(dm, PETSC_TRUE));
5381   if (refine && isHierarchy) {
5382     DM *dms, coarseDM;
5383 
5384     PetscCall(DMGetCoarseDM(dm, &coarseDM));
5385     PetscCall(PetscObjectReference((PetscObject)coarseDM));
5386     PetscCall(PetscMalloc1(refine, &dms));
5387     PetscCall(DMRefineHierarchy(dm, refine, dms));
5388     /* Total hack since we do not pass in a pointer */
5389     PetscCall(DMPlexSwap_Static(dm, dms[refine - 1]));
5390     if (refine == 1) {
5391       PetscCall(DMSetCoarseDM(dm, dms[0]));
5392       PetscCall(DMPlexSetRegularRefinement(dm, PETSC_TRUE));
5393     } else {
5394       PetscCall(DMSetCoarseDM(dm, dms[refine - 2]));
5395       PetscCall(DMPlexSetRegularRefinement(dm, PETSC_TRUE));
5396       PetscCall(DMSetCoarseDM(dms[0], dms[refine - 1]));
5397       PetscCall(DMPlexSetRegularRefinement(dms[0], PETSC_TRUE));
5398     }
5399     PetscCall(DMSetCoarseDM(dms[refine - 1], coarseDM));
5400     PetscCall(PetscObjectDereference((PetscObject)coarseDM));
5401     /* Free DMs */
5402     for (r = 0; r < refine; ++r) {
5403       PetscCall(DMSetFromOptions_NonRefinement_Plex(dms[r], PetscOptionsObject));
5404       PetscCall(DMDestroy(&dms[r]));
5405     }
5406     PetscCall(PetscFree(dms));
5407   } else {
5408     for (r = 0; r < refine; ++r) {
5409       DM             rdm;
5410       PetscPointFunc coordFunc = ((DM_Plex *)dm->data)->coordFunc;
5411 
5412       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
5413       PetscCall(DMRefine(dm, PetscObjectComm((PetscObject)dm), &rdm));
5414       /* Total hack since we do not pass in a pointer */
5415       PetscCall(DMPlexReplace_Internal(dm, &rdm));
5416       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
5417       if (coordFunc && remap) {
5418         PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
5419         ((DM_Plex *)dm->data)->coordFunc = coordFunc;
5420       }
5421     }
5422   }
5423   /* Handle DMPlex coarsening */
5424   PetscCall(PetscOptionsBoundedInt("-dm_coarsen", "Coarsen the mesh", "DMCreate", coarsen, &coarsen, NULL, 0));
5425   PetscCall(PetscOptionsBoundedInt("-dm_coarsen_hierarchy", "The number of coarsenings", "DMCreate", coarsen, &coarsen, &isHierarchy, 0));
5426   if (coarsen && isHierarchy) {
5427     DM *dms;
5428 
5429     PetscCall(PetscMalloc1(coarsen, &dms));
5430     PetscCall(DMCoarsenHierarchy(dm, coarsen, dms));
5431     /* Free DMs */
5432     for (r = 0; r < coarsen; ++r) {
5433       PetscCall(DMSetFromOptions_NonRefinement_Plex(dms[r], PetscOptionsObject));
5434       PetscCall(DMDestroy(&dms[r]));
5435     }
5436     PetscCall(PetscFree(dms));
5437   } else {
5438     for (r = 0; r < coarsen; ++r) {
5439       DM             cdm;
5440       PetscPointFunc coordFunc = ((DM_Plex *)dm->data)->coordFunc;
5441 
5442       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
5443       PetscCall(DMCoarsen(dm, PetscObjectComm((PetscObject)dm), &cdm));
5444       /* Total hack since we do not pass in a pointer */
5445       PetscCall(DMPlexReplace_Internal(dm, &cdm));
5446       PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
5447       if (coordFunc) {
5448         PetscCall(DMPlexRemapGeometry(dm, 0.0, coordFunc));
5449         ((DM_Plex *)dm->data)->coordFunc = coordFunc;
5450       }
5451     }
5452   }
5453   // Handle coordinate remapping
5454   remap = PETSC_FALSE;
5455   PetscCall(PetscOptionsBool("-dm_coord_remap", "Flag to control coordinate remapping", "", remap, &remap, NULL));
5456   if (remap) {
5457     DMPlexCoordMap map     = DM_COORD_MAP_NONE;
5458     PetscPointFunc mapFunc = NULL;
5459     PetscScalar    params[16];
5460     PetscInt       Np = PETSC_STATIC_ARRAY_LENGTH(params), cdim;
5461     MPI_Comm       comm;
5462 
5463     PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
5464     PetscCall(DMGetCoordinateDim(dm, &cdim));
5465     PetscCall(PetscOptionsScalarArray("-dm_coord_map_params", "Parameters for the coordinate remapping", "", params, &Np, &flg));
5466     if (!flg) Np = 0;
5467     // TODO Allow user to pass a map function by name
5468     PetscCall(PetscOptionsEnum("-dm_coord_map", "Coordinate mapping for built-in mesh", "", DMPlexCoordMaps, (PetscEnum)map, (PetscEnum *)&map, &flg));
5469     if (flg) {
5470       switch (map) {
5471       case DM_COORD_MAP_NONE:
5472         mapFunc = coordMap_identity;
5473         break;
5474       case DM_COORD_MAP_SHEAR:
5475         mapFunc = coordMap_shear;
5476         if (!Np) {
5477           Np        = cdim + 1;
5478           params[0] = 0;
5479           for (PetscInt d = 1; d <= cdim; ++d) params[d] = 1.0;
5480         }
5481         PetscCheck(Np == cdim + 1, comm, PETSC_ERR_ARG_WRONG, "The shear coordinate map must have cdim + 1 = %" PetscInt_FMT " parameters, not %" PetscInt_FMT, cdim + 1, Np);
5482         break;
5483       case DM_COORD_MAP_FLARE:
5484         mapFunc = coordMap_flare;
5485         if (!Np) {
5486           Np        = cdim + 1;
5487           params[0] = 0;
5488           for (PetscInt d = 1; d <= cdim; ++d) params[d] = 1.0;
5489         }
5490         PetscCheck(Np == cdim + 1, comm, PETSC_ERR_ARG_WRONG, "The flare coordinate map must have cdim + 1 = %" PetscInt_FMT " parameters, not %" PetscInt_FMT, cdim + 1, Np);
5491         break;
5492       case DM_COORD_MAP_ANNULUS:
5493         mapFunc = coordMap_annulus;
5494         if (!Np) {
5495           Np        = 2;
5496           params[0] = 1.;
5497           params[1] = 2.;
5498         }
5499         PetscCheck(Np == 2, comm, PETSC_ERR_ARG_WRONG, "The annulus coordinate map must have 2 parameters, not %" PetscInt_FMT, Np);
5500         break;
5501       case DM_COORD_MAP_SHELL:
5502         mapFunc = coordMap_shell;
5503         if (!Np) {
5504           Np        = 2;
5505           params[0] = 1.;
5506           params[1] = 2.;
5507         }
5508         PetscCheck(Np == 2, comm, PETSC_ERR_ARG_WRONG, "The spherical shell coordinate map must have 2 parameters, not %" PetscInt_FMT, Np);
5509         break;
5510       case DM_COORD_MAP_SINUSOID:
5511         mapFunc = coordMap_sinusoid;
5512         if (!Np) {
5513           Np        = 3;
5514           params[0] = 1.;
5515           params[1] = 1.;
5516           params[2] = 1.;
5517         }
5518         PetscCheck(Np == 3, comm, PETSC_ERR_ARG_WRONG, "The sinusoidal coordinate map must have 3 parameters, not %" PetscInt_FMT, Np);
5519         break;
5520       default:
5521         mapFunc = coordMap_identity;
5522       }
5523     }
5524     if (Np) {
5525       DM      cdm;
5526       PetscDS cds;
5527 
5528       PetscCall(DMGetCoordinateDM(dm, &cdm));
5529       PetscCall(DMGetDS(cdm, &cds));
5530       PetscCall(PetscDSSetConstants(cds, Np, params));
5531     }
5532     PetscCall(DMPlexRemapGeometry(dm, 0.0, mapFunc));
5533   }
5534   /* Handle ghost cells */
5535   PetscCall(PetscOptionsBool("-dm_plex_create_fv_ghost_cells", "Flag to create finite volume ghost cells on the boundary", "DMCreate", ghostCells, &ghostCells, NULL));
5536   if (ghostCells) {
5537     DM   gdm;
5538     char lname[PETSC_MAX_PATH_LEN];
5539 
5540     lname[0] = '\0';
5541     PetscCall(PetscOptionsString("-dm_plex_fv_ghost_cells_label", "Label name for ghost cells boundary", "DMCreate", lname, lname, sizeof(lname), &flg));
5542     PetscCall(DMPlexConstructGhostCells(dm, flg ? lname : NULL, NULL, &gdm));
5543     PetscCall(DMPlexReplace_Internal(dm, &gdm));
5544   }
5545   /* Handle 1D order */
5546   if (reorder != DM_REORDER_DEFAULT_FALSE && dim == 1) {
5547     DM           cdm, rdm;
5548     PetscDS      cds;
5549     PetscObject  obj;
5550     PetscClassId id = PETSC_OBJECT_CLASSID;
5551     IS           perm;
5552     PetscInt     Nf;
5553     PetscBool    distributed;
5554 
5555     PetscCall(DMPlexIsDistributed(dm, &distributed));
5556     PetscCall(DMGetCoordinateDM(dm, &cdm));
5557     PetscCall(DMGetDS(cdm, &cds));
5558     PetscCall(PetscDSGetNumFields(cds, &Nf));
5559     if (Nf) {
5560       PetscCall(PetscDSGetDiscretization(cds, 0, &obj));
5561       PetscCall(PetscObjectGetClassId(obj, &id));
5562     }
5563     if (!distributed && id != PETSCFE_CLASSID) {
5564       PetscCall(DMPlexGetOrdering1D(dm, &perm));
5565       PetscCall(DMPlexPermute(dm, perm, &rdm));
5566       PetscCall(DMPlexReplace_Internal(dm, &rdm));
5567       PetscCall(ISDestroy(&perm));
5568     }
5569   }
5570 /* Handle */
5571 non_refine:
5572   PetscCall(DMSetFromOptions_NonRefinement_Plex(dm, PetscOptionsObject));
5573   char    *phases[16];
5574   PetscInt Nphases = 16;
5575   PetscCall(PetscOptionsStringArray("-dm_plex_option_phases", "Option phase prefixes", "DMSetFromOptions", phases, &Nphases, &flg));
5576   PetscOptionsHeadEnd();
5577 
5578   // Phases
5579   if (flg) {
5580     DM          cdm;
5581     char       *oldPrefix, *oldCoordPrefix;
5582     const char *tmp;
5583 
5584     PetscCall(DMGetCoordinateDM(dm, &cdm));
5585     PetscCall(PetscObjectGetOptionsPrefix((PetscObject)dm, &tmp));
5586     PetscCall(PetscStrallocpy(tmp, &oldPrefix));
5587     PetscCall(PetscObjectGetOptionsPrefix((PetscObject)cdm, &tmp));
5588     PetscCall(PetscStrallocpy(tmp, &oldCoordPrefix));
5589     for (PetscInt ph = 0; ph < Nphases; ++ph) {
5590       PetscCall(PetscObjectAppendOptionsPrefix((PetscObject)dm, phases[ph]));
5591       PetscCall(PetscObjectAppendOptionsPrefix((PetscObject)cdm, phases[ph]));
5592       PetscCall(PetscInfo(dm, "Options phase %s for DM %s\n", phases[ph], dm->hdr.name));
5593       PetscCall(DMSetFromOptions(dm));
5594       PetscCall(PetscObjectSetOptionsPrefix((PetscObject)dm, oldPrefix));
5595       PetscCall(DMGetCoordinateDM(dm, &cdm));
5596       PetscCall(PetscObjectSetOptionsPrefix((PetscObject)cdm, oldCoordPrefix));
5597       PetscCall(PetscFree(phases[ph]));
5598     }
5599     PetscCall(PetscFree(oldPrefix));
5600     PetscCall(PetscFree(oldCoordPrefix));
5601   }
5602   PetscFunctionReturn(PETSC_SUCCESS);
5603 }
5604 
5605 static PetscErrorCode DMCreateGlobalVector_Plex(DM dm, Vec *vec)
5606 {
5607   PetscFunctionBegin;
5608   PetscCall(DMCreateGlobalVector_Section_Private(dm, vec));
5609   /* PetscCall(VecSetOperation(*vec, VECOP_DUPLICATE, (void(*)(void)) VecDuplicate_MPI_DM)); */
5610   PetscCall(VecSetOperation(*vec, VECOP_VIEW, (void (*)(void))VecView_Plex));
5611   PetscCall(VecSetOperation(*vec, VECOP_VIEWNATIVE, (void (*)(void))VecView_Plex_Native));
5612   PetscCall(VecSetOperation(*vec, VECOP_LOAD, (void (*)(void))VecLoad_Plex));
5613   PetscCall(VecSetOperation(*vec, VECOP_LOADNATIVE, (void (*)(void))VecLoad_Plex_Native));
5614   PetscFunctionReturn(PETSC_SUCCESS);
5615 }
5616 
5617 static PetscErrorCode DMCreateLocalVector_Plex(DM dm, Vec *vec)
5618 {
5619   PetscFunctionBegin;
5620   PetscCall(DMCreateLocalVector_Section_Private(dm, vec));
5621   PetscCall(VecSetOperation(*vec, VECOP_VIEW, (void (*)(void))VecView_Plex_Local));
5622   PetscCall(VecSetOperation(*vec, VECOP_LOAD, (void (*)(void))VecLoad_Plex_Local));
5623   PetscFunctionReturn(PETSC_SUCCESS);
5624 }
5625 
5626 static PetscErrorCode DMGetDimPoints_Plex(DM dm, PetscInt dim, PetscInt *pStart, PetscInt *pEnd)
5627 {
5628   PetscInt depth, d;
5629 
5630   PetscFunctionBegin;
5631   PetscCall(DMPlexGetDepth(dm, &depth));
5632   if (depth == 1) {
5633     PetscCall(DMGetDimension(dm, &d));
5634     if (dim == 0) PetscCall(DMPlexGetDepthStratum(dm, dim, pStart, pEnd));
5635     else if (dim == d) PetscCall(DMPlexGetDepthStratum(dm, 1, pStart, pEnd));
5636     else {
5637       *pStart = 0;
5638       *pEnd   = 0;
5639     }
5640   } else {
5641     PetscCall(DMPlexGetDepthStratum(dm, dim, pStart, pEnd));
5642   }
5643   PetscFunctionReturn(PETSC_SUCCESS);
5644 }
5645 
5646 static PetscErrorCode DMGetNeighbors_Plex(DM dm, PetscInt *nranks, const PetscMPIInt *ranks[])
5647 {
5648   PetscSF            sf;
5649   PetscMPIInt        niranks, njranks;
5650   PetscInt           n;
5651   const PetscMPIInt *iranks, *jranks;
5652   DM_Plex           *data = (DM_Plex *)dm->data;
5653 
5654   PetscFunctionBegin;
5655   PetscCall(DMGetPointSF(dm, &sf));
5656   if (!data->neighbors) {
5657     PetscCall(PetscSFSetUp(sf));
5658     PetscCall(PetscSFGetRootRanks(sf, &njranks, &jranks, NULL, NULL, NULL));
5659     PetscCall(PetscSFGetLeafRanks(sf, &niranks, &iranks, NULL, NULL));
5660     PetscCall(PetscMalloc1(njranks + niranks + 1, &data->neighbors));
5661     PetscCall(PetscArraycpy(data->neighbors + 1, jranks, njranks));
5662     PetscCall(PetscArraycpy(data->neighbors + njranks + 1, iranks, niranks));
5663     n = njranks + niranks;
5664     PetscCall(PetscSortRemoveDupsMPIInt(&n, data->neighbors + 1));
5665     /* The following cast should never fail: can't have more neighbors than PETSC_MPI_INT_MAX */
5666     PetscCall(PetscMPIIntCast(n, data->neighbors));
5667   }
5668   if (nranks) *nranks = data->neighbors[0];
5669   if (ranks) {
5670     if (data->neighbors[0]) *ranks = data->neighbors + 1;
5671     else *ranks = NULL;
5672   }
5673   PetscFunctionReturn(PETSC_SUCCESS);
5674 }
5675 
5676 PETSC_INTERN PetscErrorCode DMInterpolateSolution_Plex(DM, DM, Mat, Vec, Vec);
5677 
5678 static PetscErrorCode DMInitialize_Plex(DM dm)
5679 {
5680   PetscFunctionBegin;
5681   dm->ops->view                      = DMView_Plex;
5682   dm->ops->load                      = DMLoad_Plex;
5683   dm->ops->setfromoptions            = DMSetFromOptions_Plex;
5684   dm->ops->clone                     = DMClone_Plex;
5685   dm->ops->setup                     = DMSetUp_Plex;
5686   dm->ops->createlocalsection        = DMCreateLocalSection_Plex;
5687   dm->ops->createsectionpermutation  = DMCreateSectionPermutation_Plex;
5688   dm->ops->createdefaultconstraints  = DMCreateDefaultConstraints_Plex;
5689   dm->ops->createglobalvector        = DMCreateGlobalVector_Plex;
5690   dm->ops->createlocalvector         = DMCreateLocalVector_Plex;
5691   dm->ops->getlocaltoglobalmapping   = NULL;
5692   dm->ops->createfieldis             = NULL;
5693   dm->ops->createcoordinatedm        = DMCreateCoordinateDM_Plex;
5694   dm->ops->createcoordinatefield     = DMCreateCoordinateField_Plex;
5695   dm->ops->getcoloring               = NULL;
5696   dm->ops->creatematrix              = DMCreateMatrix_Plex;
5697   dm->ops->createinterpolation       = DMCreateInterpolation_Plex;
5698   dm->ops->createmassmatrix          = DMCreateMassMatrix_Plex;
5699   dm->ops->createmassmatrixlumped    = DMCreateMassMatrixLumped_Plex;
5700   dm->ops->createinjection           = DMCreateInjection_Plex;
5701   dm->ops->refine                    = DMRefine_Plex;
5702   dm->ops->coarsen                   = DMCoarsen_Plex;
5703   dm->ops->refinehierarchy           = DMRefineHierarchy_Plex;
5704   dm->ops->coarsenhierarchy          = DMCoarsenHierarchy_Plex;
5705   dm->ops->extrude                   = DMExtrude_Plex;
5706   dm->ops->globaltolocalbegin        = NULL;
5707   dm->ops->globaltolocalend          = NULL;
5708   dm->ops->localtoglobalbegin        = NULL;
5709   dm->ops->localtoglobalend          = NULL;
5710   dm->ops->destroy                   = DMDestroy_Plex;
5711   dm->ops->createsubdm               = DMCreateSubDM_Plex;
5712   dm->ops->createsuperdm             = DMCreateSuperDM_Plex;
5713   dm->ops->getdimpoints              = DMGetDimPoints_Plex;
5714   dm->ops->locatepoints              = DMLocatePoints_Plex;
5715   dm->ops->projectfunctionlocal      = DMProjectFunctionLocal_Plex;
5716   dm->ops->projectfunctionlabellocal = DMProjectFunctionLabelLocal_Plex;
5717   dm->ops->projectfieldlocal         = DMProjectFieldLocal_Plex;
5718   dm->ops->projectfieldlabellocal    = DMProjectFieldLabelLocal_Plex;
5719   dm->ops->projectbdfieldlabellocal  = DMProjectBdFieldLabelLocal_Plex;
5720   dm->ops->computel2diff             = DMComputeL2Diff_Plex;
5721   dm->ops->computel2gradientdiff     = DMComputeL2GradientDiff_Plex;
5722   dm->ops->computel2fielddiff        = DMComputeL2FieldDiff_Plex;
5723   dm->ops->getneighbors              = DMGetNeighbors_Plex;
5724   dm->ops->getlocalboundingbox       = DMGetLocalBoundingBox_Coordinates;
5725   dm->ops->createdomaindecomposition = DMCreateDomainDecomposition_Plex;
5726   dm->ops->createddscatters          = DMCreateDomainDecompositionScatters_Plex;
5727   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexInsertBoundaryValues_C", DMPlexInsertBoundaryValues_Plex));
5728   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexInsertTimeDerivativeBoundaryValues_C", DMPlexInsertTimeDerivativeBoundaryValues_Plex));
5729   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMSetUpGLVisViewer_C", DMSetUpGLVisViewer_Plex));
5730   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMCreateNeumannOverlap_C", DMCreateNeumannOverlap_Plex));
5731   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexDistributeGetDefault_C", DMPlexDistributeGetDefault_Plex));
5732   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexDistributeSetDefault_C", DMPlexDistributeSetDefault_Plex));
5733   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexReorderGetDefault_C", DMPlexReorderGetDefault_Plex));
5734   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexReorderSetDefault_C", DMPlexReorderSetDefault_Plex));
5735   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMReorderSectionGetDefault_C", DMReorderSectionGetDefault_Plex));
5736   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMReorderSectionSetDefault_C", DMReorderSectionSetDefault_Plex));
5737   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMReorderSectionGetType_C", DMReorderSectionGetType_Plex));
5738   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMReorderSectionSetType_C", DMReorderSectionSetType_Plex));
5739   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMInterpolateSolution_C", DMInterpolateSolution_Plex));
5740   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexGetOverlap_C", DMPlexGetOverlap_Plex));
5741   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexSetOverlap_C", DMPlexSetOverlap_Plex));
5742   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexGetUseCeed_C", DMPlexGetUseCeed_Plex));
5743   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexSetUseCeed_C", DMPlexSetUseCeed_Plex));
5744   PetscFunctionReturn(PETSC_SUCCESS);
5745 }
5746 
5747 PETSC_INTERN PetscErrorCode DMClone_Plex(DM dm, DM *newdm)
5748 {
5749   DM_Plex       *mesh = (DM_Plex *)dm->data;
5750   const PetscSF *face_sfs;
5751   PetscInt       num_face_sfs;
5752 
5753   PetscFunctionBegin;
5754   mesh->refct++;
5755   (*newdm)->data = mesh;
5756   PetscCall(DMPlexGetIsoperiodicFaceSF(dm, &num_face_sfs, &face_sfs));
5757   PetscCall(DMPlexSetIsoperiodicFaceSF(*newdm, num_face_sfs, (PetscSF *)face_sfs));
5758   PetscCall(PetscObjectChangeTypeName((PetscObject)*newdm, DMPLEX));
5759   PetscCall(DMInitialize_Plex(*newdm));
5760   PetscFunctionReturn(PETSC_SUCCESS);
5761 }
5762 
5763 /*MC
5764   DMPLEX = "plex" - A `DM` object that encapsulates an unstructured mesh (or grid), or CW Complex {cite}`cwcomplex`,
5765            which can be expressed using a Hasse Diagram {cite}`hassediagram`.
5766            In the local representation, `Vec`s contain all unknowns in the interior and shared boundary. This is
5767            specified by a `PetscSection` object. Ownership in the global representation is determined by
5768            ownership of the underlying `DMPLEX` points. This is specified by another `PetscSection` object.
5769 
5770   Options Database Keys:
5771 + -dm_refine_pre                     - Refine mesh before distribution
5772 + -dm_refine_uniform_pre             - Choose uniform or generator-based refinement
5773 + -dm_refine_volume_limit_pre        - Cell volume limit after pre-refinement using generator
5774 . -dm_distribute                     - Distribute mesh across processes
5775 . -dm_distribute_overlap             - Number of cells to overlap for distribution
5776 . -dm_refine                         - Refine mesh after distribution
5777 . -dm_localize <bool>                - Whether to localize coordinates for periodic meshes
5778 . -dm_sparse_localize <bool>         - Whether to only localize cells on the periodic boundary
5779 . -dm_plex_hash_location             - Use grid hashing for point location
5780 . -dm_plex_hash_box_faces <n,m,p>    - The number of divisions in each direction of the grid hash
5781 . -dm_plex_partition_balance         - Attempt to evenly divide points on partition boundary between processes
5782 . -dm_plex_remesh_bd                 - Allow changes to the boundary on remeshing
5783 . -dm_plex_max_projection_height     - Maximum mesh point height used to project locally
5784 . -dm_plex_regular_refinement        - Use special nested projection algorithm for regular refinement
5785 . -dm_plex_reorder_section           - Use specialized blocking if available
5786 . -dm_plex_check_all                 - Perform all checks below
5787 . -dm_plex_check_symmetry            - Check that the adjacency information in the mesh is symmetric
5788 . -dm_plex_check_skeleton <celltype> - Check that each cell has the correct number of vertices
5789 . -dm_plex_check_faces <celltype>    - Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type
5790 . -dm_plex_check_geometry            - Check that cells have positive volume
5791 . -dm_view :mesh.tex:ascii_latex     - View the mesh in LaTeX/TikZ
5792 . -dm_plex_view_scale <num>          - Scale the TikZ
5793 . -dm_plex_print_fem <num>           - View FEM assembly information, such as element vectors and matrices
5794 - -dm_plex_print_fvm <num>           - View FVM assembly information, such as flux updates
5795 
5796   Level: intermediate
5797 
5798 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMType`, `DMPlexCreate()`, `DMCreate()`, `DMSetType()`, `PetscSection`
5799 M*/
5800 
5801 PETSC_EXTERN PetscErrorCode DMCreate_Plex(DM dm)
5802 {
5803   DM_Plex *mesh;
5804   PetscInt unit;
5805 
5806   PetscFunctionBegin;
5807   PetscCall(PetscCitationsRegister(PlexCitation, &Plexcite));
5808   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5809   PetscCall(PetscNew(&mesh));
5810   dm->reorderSection = DM_REORDER_DEFAULT_NOTSET;
5811   dm->data           = mesh;
5812 
5813   mesh->refct = 1;
5814   PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->coneSection));
5815   PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dm), &mesh->supportSection));
5816   mesh->refinementUniform       = PETSC_TRUE;
5817   mesh->refinementLimit         = -1.0;
5818   mesh->distDefault             = PETSC_TRUE;
5819   mesh->reorderDefault          = DM_REORDER_DEFAULT_NOTSET;
5820   mesh->distributionName        = NULL;
5821   mesh->interpolated            = DMPLEX_INTERPOLATED_INVALID;
5822   mesh->interpolatedCollective  = DMPLEX_INTERPOLATED_INVALID;
5823   mesh->interpolatePreferTensor = PETSC_TRUE;
5824 
5825   PetscCall(PetscPartitionerCreate(PetscObjectComm((PetscObject)dm), &mesh->partitioner));
5826   mesh->remeshBd = PETSC_FALSE;
5827 
5828   for (unit = 0; unit < NUM_PETSC_UNITS; ++unit) mesh->scale[unit] = 1.0;
5829 
5830   mesh->depthState    = -1;
5831   mesh->celltypeState = -1;
5832   mesh->printTol      = 1.0e-10;
5833   mesh->nonempty_comm = MPI_COMM_SELF;
5834 
5835   PetscCall(DMInitialize_Plex(dm));
5836   PetscFunctionReturn(PETSC_SUCCESS);
5837 }
5838 
5839 /*@
5840   DMPlexCreate - Creates a `DMPLEX` object, which encapsulates an unstructured mesh, or CW complex, which can be expressed using a Hasse Diagram.
5841 
5842   Collective
5843 
5844   Input Parameter:
5845 . comm - The communicator for the `DMPLEX` object
5846 
5847   Output Parameter:
5848 . mesh - The `DMPLEX` object
5849 
5850   Level: beginner
5851 
5852 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMType`, `DMCreate()`, `DMSetType()`
5853 @*/
5854 PetscErrorCode DMPlexCreate(MPI_Comm comm, DM *mesh)
5855 {
5856   PetscFunctionBegin;
5857   PetscAssertPointer(mesh, 2);
5858   PetscCall(DMCreate(comm, mesh));
5859   PetscCall(DMSetType(*mesh, DMPLEX));
5860   PetscFunctionReturn(PETSC_SUCCESS);
5861 }
5862 
5863 /*@C
5864   DMPlexBuildFromCellListParallel - Build distributed `DMPLEX` topology from a list of vertices for each cell (common mesh generator output) where all cells have the same celltype
5865 
5866   Collective; No Fortran Support
5867 
5868   Input Parameters:
5869 + dm          - The `DM`
5870 . numCells    - The number of cells owned by this process
5871 . numVertices - The number of vertices to be owned by this process, or `PETSC_DECIDE`
5872 . NVertices   - The global number of vertices, or `PETSC_DETERMINE`
5873 . numCorners  - The number of vertices for each cell
5874 - cells       - An array of numCells*numCorners numbers, the global vertex numbers for each cell
5875 
5876   Output Parameters:
5877 + vertexSF         - (Optional) `PetscSF` describing complete vertex ownership
5878 - verticesAdjSaved - (Optional) vertex adjacency array
5879 
5880   Level: advanced
5881 
5882   Notes:
5883   Two triangles sharing a face
5884 .vb
5885 
5886         2
5887       / | \
5888      /  |  \
5889     /   |   \
5890    0  0 | 1  3
5891     \   |   /
5892      \  |  /
5893       \ | /
5894         1
5895 .ve
5896   would have input
5897 .vb
5898   numCells = 2, numVertices = 4
5899   cells = [0 1 2  1 3 2]
5900 .ve
5901   which would result in the `DMPLEX`
5902 .vb
5903 
5904         4
5905       / | \
5906      /  |  \
5907     /   |   \
5908    2  0 | 1  5
5909     \   |   /
5910      \  |  /
5911       \ | /
5912         3
5913 .ve
5914 
5915   Vertices are implicitly numbered consecutively 0,...,NVertices.
5916   Each rank owns a chunk of numVertices consecutive vertices.
5917   If numVertices is `PETSC_DECIDE`, PETSc will distribute them as evenly as possible using PetscLayout.
5918   If NVertices is `PETSC_DETERMINE` and numVertices is PETSC_DECIDE, NVertices is computed by PETSc as the maximum vertex index in cells + 1.
5919   If only NVertices is `PETSC_DETERMINE`, it is computed as the sum of numVertices over all ranks.
5920 
5921   The cell distribution is arbitrary non-overlapping, independent of the vertex distribution.
5922 
5923 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexBuildFromCellList()`, `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildCoordinatesFromCellListParallel()`,
5924           `PetscSF`
5925 @*/
5926 PetscErrorCode DMPlexBuildFromCellListParallel(DM dm, PetscInt numCells, PetscInt numVertices, PetscInt NVertices, PetscInt numCorners, const PetscInt cells[], PeOp PetscSF *vertexSF, PeOp PetscInt *verticesAdjSaved[])
5927 {
5928   PetscSF     sfPoint;
5929   PetscLayout layout;
5930   PetscInt    numVerticesAdj, *verticesAdj, *cones, c, p;
5931 
5932   PetscFunctionBegin;
5933   PetscValidLogicalCollectiveInt(dm, NVertices, 4);
5934   PetscCall(PetscLogEventBegin(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
5935   /* Get/check global number of vertices */
5936   {
5937     PetscInt       NVerticesInCells, i;
5938     const PetscInt len = numCells * numCorners;
5939 
5940     /* NVerticesInCells = max(cells) + 1 */
5941     NVerticesInCells = PETSC_INT_MIN;
5942     for (i = 0; i < len; i++)
5943       if (cells[i] > NVerticesInCells) NVerticesInCells = cells[i];
5944     ++NVerticesInCells;
5945     PetscCallMPI(MPIU_Allreduce(MPI_IN_PLACE, &NVerticesInCells, 1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject)dm)));
5946 
5947     if (numVertices == PETSC_DECIDE && NVertices == PETSC_DECIDE) NVertices = NVerticesInCells;
5948     else
5949       PetscCheck(NVertices == PETSC_DECIDE || NVertices >= NVerticesInCells, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Specified global number of vertices %" PetscInt_FMT " must be greater than or equal to the number of vertices in cells %" PetscInt_FMT, NVertices, NVerticesInCells);
5950   }
5951   /* Count locally unique vertices */
5952   {
5953     PetscHSetI vhash;
5954     PetscInt   off = 0;
5955 
5956     PetscCall(PetscHSetICreate(&vhash));
5957     for (c = 0; c < numCells; ++c) {
5958       for (p = 0; p < numCorners; ++p) PetscCall(PetscHSetIAdd(vhash, cells[c * numCorners + p]));
5959     }
5960     PetscCall(PetscHSetIGetSize(vhash, &numVerticesAdj));
5961     if (!verticesAdjSaved) PetscCall(PetscMalloc1(numVerticesAdj, &verticesAdj));
5962     else verticesAdj = *verticesAdjSaved;
5963     PetscCall(PetscHSetIGetElems(vhash, &off, verticesAdj));
5964     PetscCall(PetscHSetIDestroy(&vhash));
5965     PetscCheck(off == numVerticesAdj, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid number of local vertices %" PetscInt_FMT " should be %" PetscInt_FMT, off, numVerticesAdj);
5966   }
5967   PetscCall(PetscSortInt(numVerticesAdj, verticesAdj));
5968   /* Create cones */
5969   PetscCall(DMPlexSetChart(dm, 0, numCells + numVerticesAdj));
5970   for (c = 0; c < numCells; ++c) PetscCall(DMPlexSetConeSize(dm, c, numCorners));
5971   PetscCall(DMSetUp(dm));
5972   PetscCall(DMPlexGetCones(dm, &cones));
5973   for (c = 0; c < numCells; ++c) {
5974     for (p = 0; p < numCorners; ++p) {
5975       const PetscInt gv = cells[c * numCorners + p];
5976       PetscInt       lv;
5977 
5978       /* Positions within verticesAdj form 0-based local vertex numbering;
5979          we need to shift it by numCells to get correct DAG points (cells go first) */
5980       PetscCall(PetscFindInt(gv, numVerticesAdj, verticesAdj, &lv));
5981       PetscCheck(lv >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Could not find global vertex %" PetscInt_FMT " in local connectivity", gv);
5982       cones[c * numCorners + p] = lv + numCells;
5983     }
5984   }
5985   /* Build point sf */
5986   PetscCall(PetscLayoutCreate(PetscObjectComm((PetscObject)dm), &layout));
5987   PetscCall(PetscLayoutSetSize(layout, NVertices));
5988   PetscCall(PetscLayoutSetLocalSize(layout, numVertices));
5989   PetscCall(PetscLayoutSetBlockSize(layout, 1));
5990   PetscCall(PetscSFCreateByMatchingIndices(layout, numVerticesAdj, verticesAdj, NULL, numCells, numVerticesAdj, verticesAdj, NULL, numCells, vertexSF, &sfPoint));
5991   PetscCall(PetscLayoutDestroy(&layout));
5992   if (!verticesAdjSaved) PetscCall(PetscFree(verticesAdj));
5993   PetscCall(PetscObjectSetName((PetscObject)sfPoint, "point SF"));
5994   if (dm->sf) {
5995     const char *prefix;
5996 
5997     PetscCall(PetscObjectGetOptionsPrefix((PetscObject)dm->sf, &prefix));
5998     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)sfPoint, prefix));
5999   }
6000   PetscCall(DMSetPointSF(dm, sfPoint));
6001   PetscCall(PetscSFDestroy(&sfPoint));
6002   if (vertexSF) PetscCall(PetscObjectSetName((PetscObject)*vertexSF, "Vertex Ownership SF"));
6003   /* Fill in the rest of the topology structure */
6004   PetscCall(DMPlexSymmetrize(dm));
6005   PetscCall(DMPlexStratify(dm));
6006   PetscCall(PetscLogEventEnd(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
6007   PetscFunctionReturn(PETSC_SUCCESS);
6008 }
6009 
6010 /*@C
6011   DMPlexBuildFromCellSectionParallel - Build distributed `DMPLEX` topology from a list of vertices for each cell (common mesh generator output) allowing multiple celltypes
6012 
6013   Collective; No Fortran Support
6014 
6015   Input Parameters:
6016 + dm          - The `DM`
6017 . numCells    - The number of cells owned by this process
6018 . numVertices - The number of vertices to be owned by this process, or `PETSC_DECIDE`
6019 . NVertices   - The global number of vertices, or `PETSC_DETERMINE`
6020 . cellSection - The `PetscSection` giving the number of vertices for each cell (layout of cells)
6021 - cells       - An array of the global vertex numbers for each cell
6022 
6023   Output Parameters:
6024 + vertexSF         - (Optional) `PetscSF` describing complete vertex ownership
6025 - verticesAdjSaved - (Optional) vertex adjacency array
6026 
6027   Level: advanced
6028 
6029   Notes:
6030   A triangle and quadrilateral sharing a face
6031 .vb
6032         2----------3
6033       / |          |
6034      /  |          |
6035     /   |          |
6036    0  0 |     1    |
6037     \   |          |
6038      \  |          |
6039       \ |          |
6040         1----------4
6041 .ve
6042   would have input
6043 .vb
6044   numCells = 2, numVertices = 5
6045   cells = [0 1 2  1 4 3 2]
6046 .ve
6047   which would result in the `DMPLEX`
6048 .vb
6049         4----------5
6050       / |          |
6051      /  |          |
6052     /   |          |
6053    2  0 |     1    |
6054     \   |          |
6055      \  |          |
6056       \ |          |
6057         3----------6
6058 .ve
6059 
6060   Vertices are implicitly numbered consecutively 0,...,NVertices.
6061   Each rank owns a chunk of numVertices consecutive vertices.
6062   If numVertices is `PETSC_DECIDE`, PETSc will distribute them as evenly as possible using PetscLayout.
6063   If NVertices is `PETSC_DETERMINE` and numVertices is PETSC_DECIDE, NVertices is computed by PETSc as the maximum vertex index in cells + 1.
6064   If only NVertices is `PETSC_DETERMINE`, it is computed as the sum of numVertices over all ranks.
6065 
6066   The cell distribution is arbitrary non-overlapping, independent of the vertex distribution.
6067 
6068 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexBuildFromCellListParallel()`, `DMPlexCreateFromCellSectionParallel()`, `DMPlexBuildCoordinatesFromCellListParallel()`,
6069           `PetscSF`
6070 @*/
6071 PetscErrorCode DMPlexBuildFromCellSectionParallel(DM dm, PetscInt numCells, PetscInt numVertices, PetscInt NVertices, PetscSection cellSection, const PetscInt cells[], PeOp PetscSF *vertexSF, PeOp PetscInt *verticesAdjSaved[])
6072 {
6073   PetscSF     sfPoint;
6074   PetscLayout layout;
6075   PetscInt    numVerticesAdj, *verticesAdj, *cones, cStart, cEnd, len;
6076 
6077   PetscFunctionBegin;
6078   PetscValidLogicalCollectiveInt(dm, NVertices, 4);
6079   PetscCall(PetscLogEventBegin(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
6080   PetscCall(PetscSectionGetChart(cellSection, &cStart, &cEnd));
6081   PetscCall(PetscSectionGetStorageSize(cellSection, &len));
6082   PetscCheck(cStart == 0 && cEnd == numCells, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Section chart [%" PetscInt_FMT ", %" PetscInt_FMT ") should be [0, %" PetscInt_FMT ")", cStart, cEnd, numCells);
6083   /* Get/check global number of vertices */
6084   {
6085     PetscInt NVerticesInCells;
6086 
6087     /* NVerticesInCells = max(cells) + 1 */
6088     NVerticesInCells = PETSC_MIN_INT;
6089     for (PetscInt i = 0; i < len; i++)
6090       if (cells[i] > NVerticesInCells) NVerticesInCells = cells[i];
6091     ++NVerticesInCells;
6092     PetscCallMPI(MPIU_Allreduce(MPI_IN_PLACE, &NVerticesInCells, 1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject)dm)));
6093 
6094     if (numVertices == PETSC_DECIDE && NVertices == PETSC_DECIDE) NVertices = NVerticesInCells;
6095     else
6096       PetscCheck(NVertices == PETSC_DECIDE || NVertices >= NVerticesInCells, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Specified global number of vertices %" PetscInt_FMT " must be greater than or equal to the number of vertices in cells %" PetscInt_FMT, NVertices, NVerticesInCells);
6097   }
6098   /* Count locally unique vertices */
6099   {
6100     PetscHSetI vhash;
6101     PetscInt   off = 0;
6102 
6103     PetscCall(PetscHSetICreate(&vhash));
6104     for (PetscInt i = 0; i < len; i++) PetscCall(PetscHSetIAdd(vhash, cells[i]));
6105     PetscCall(PetscHSetIGetSize(vhash, &numVerticesAdj));
6106     if (!verticesAdjSaved) PetscCall(PetscMalloc1(numVerticesAdj, &verticesAdj));
6107     else verticesAdj = *verticesAdjSaved;
6108     PetscCall(PetscHSetIGetElems(vhash, &off, verticesAdj));
6109     PetscCall(PetscHSetIDestroy(&vhash));
6110     PetscCheck(off == numVerticesAdj, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Invalid number of local vertices %" PetscInt_FMT " should be %" PetscInt_FMT, off, numVerticesAdj);
6111   }
6112   PetscCall(PetscSortInt(numVerticesAdj, verticesAdj));
6113   /* Create cones */
6114   PetscCall(DMPlexSetChart(dm, 0, numCells + numVerticesAdj));
6115   for (PetscInt c = 0; c < numCells; ++c) {
6116     PetscInt dof;
6117 
6118     PetscCall(PetscSectionGetDof(cellSection, c, &dof));
6119     PetscCall(DMPlexSetConeSize(dm, c, dof));
6120   }
6121   PetscCall(DMSetUp(dm));
6122   PetscCall(DMPlexGetCones(dm, &cones));
6123   for (PetscInt c = 0; c < numCells; ++c) {
6124     PetscInt dof, off;
6125 
6126     PetscCall(PetscSectionGetDof(cellSection, c, &dof));
6127     PetscCall(PetscSectionGetOffset(cellSection, c, &off));
6128     for (PetscInt p = off; p < off + dof; ++p) {
6129       const PetscInt gv = cells[p];
6130       PetscInt       lv;
6131 
6132       /* Positions within verticesAdj form 0-based local vertex numbering;
6133          we need to shift it by numCells to get correct DAG points (cells go first) */
6134       PetscCall(PetscFindInt(gv, numVerticesAdj, verticesAdj, &lv));
6135       PetscCheck(lv >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Could not find global vertex %" PetscInt_FMT " in local connectivity", gv);
6136       cones[p] = lv + numCells;
6137     }
6138   }
6139   /* Build point sf */
6140   PetscCall(PetscLayoutCreate(PetscObjectComm((PetscObject)dm), &layout));
6141   PetscCall(PetscLayoutSetSize(layout, NVertices));
6142   PetscCall(PetscLayoutSetLocalSize(layout, numVertices));
6143   PetscCall(PetscLayoutSetBlockSize(layout, 1));
6144   PetscCall(PetscSFCreateByMatchingIndices(layout, numVerticesAdj, verticesAdj, NULL, numCells, numVerticesAdj, verticesAdj, NULL, numCells, vertexSF, &sfPoint));
6145   PetscCall(PetscLayoutDestroy(&layout));
6146   if (!verticesAdjSaved) PetscCall(PetscFree(verticesAdj));
6147   PetscCall(PetscObjectSetName((PetscObject)sfPoint, "point SF"));
6148   if (dm->sf) {
6149     const char *prefix;
6150 
6151     PetscCall(PetscObjectGetOptionsPrefix((PetscObject)dm->sf, &prefix));
6152     PetscCall(PetscObjectSetOptionsPrefix((PetscObject)sfPoint, prefix));
6153   }
6154   PetscCall(DMSetPointSF(dm, sfPoint));
6155   PetscCall(PetscSFDestroy(&sfPoint));
6156   if (vertexSF) PetscCall(PetscObjectSetName((PetscObject)*vertexSF, "Vertex Ownership SF"));
6157   /* Fill in the rest of the topology structure */
6158   PetscCall(DMPlexSymmetrize(dm));
6159   PetscCall(DMPlexStratify(dm));
6160   PetscCall(PetscLogEventEnd(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
6161   PetscFunctionReturn(PETSC_SUCCESS);
6162 }
6163 
6164 /*@
6165   DMPlexBuildCoordinatesFromCellListParallel - Build `DM` coordinates from a list of coordinates for each owned vertex (common mesh generator output)
6166 
6167   Collective; No Fortran Support
6168 
6169   Input Parameters:
6170 + dm           - The `DM`
6171 . spaceDim     - The spatial dimension used for coordinates
6172 . sfVert       - `PetscSF` describing complete vertex ownership
6173 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
6174 
6175   Level: advanced
6176 
6177 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildFromCellListParallel()`
6178 @*/
6179 PetscErrorCode DMPlexBuildCoordinatesFromCellListParallel(DM dm, PetscInt spaceDim, PetscSF sfVert, const PetscReal vertexCoords[])
6180 {
6181   PetscSection coordSection;
6182   Vec          coordinates;
6183   PetscScalar *coords;
6184   PetscInt     numVertices, numVerticesAdj, coordSize, v, vStart, vEnd;
6185   PetscMPIInt  spaceDimi;
6186 
6187   PetscFunctionBegin;
6188   PetscCall(PetscLogEventBegin(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
6189   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
6190   PetscCheck(vStart >= 0 && vEnd >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM is not set up properly. DMPlexBuildFromCellList() should be called first.");
6191   PetscCall(DMSetCoordinateDim(dm, spaceDim));
6192   PetscCall(PetscSFGetGraph(sfVert, &numVertices, &numVerticesAdj, NULL, NULL));
6193   PetscCheck(vEnd - vStart == numVerticesAdj, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Supplied sfVert has wrong number of leaves = %" PetscInt_FMT " != %" PetscInt_FMT " = vEnd - vStart", numVerticesAdj, vEnd - vStart);
6194   PetscCall(DMGetCoordinateSection(dm, &coordSection));
6195   PetscCall(PetscSectionSetNumFields(coordSection, 1));
6196   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, spaceDim));
6197   PetscCall(PetscSectionSetChart(coordSection, vStart, vEnd));
6198   for (v = vStart; v < vEnd; ++v) {
6199     PetscCall(PetscSectionSetDof(coordSection, v, spaceDim));
6200     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, spaceDim));
6201   }
6202   PetscCall(PetscSectionSetUp(coordSection));
6203   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
6204   PetscCall(VecCreate(PetscObjectComm((PetscObject)dm), &coordinates));
6205   PetscCall(VecSetBlockSize(coordinates, spaceDim));
6206   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
6207   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
6208   PetscCall(VecSetType(coordinates, VECSTANDARD));
6209   PetscCall(VecGetArray(coordinates, &coords));
6210   {
6211     MPI_Datatype coordtype;
6212 
6213     /* Need a temp buffer for coords if we have complex/single */
6214     PetscCall(PetscMPIIntCast(spaceDim, &spaceDimi));
6215     PetscCallMPI(MPI_Type_contiguous(spaceDimi, MPIU_SCALAR, &coordtype));
6216     PetscCallMPI(MPI_Type_commit(&coordtype));
6217 #if defined(PETSC_USE_COMPLEX)
6218     {
6219       PetscScalar *svertexCoords;
6220       PetscInt     i;
6221       PetscCall(PetscMalloc1(numVertices * spaceDim, &svertexCoords));
6222       for (i = 0; i < numVertices * spaceDim; i++) svertexCoords[i] = vertexCoords[i];
6223       PetscCall(PetscSFBcastBegin(sfVert, coordtype, svertexCoords, coords, MPI_REPLACE));
6224       PetscCall(PetscSFBcastEnd(sfVert, coordtype, svertexCoords, coords, MPI_REPLACE));
6225       PetscCall(PetscFree(svertexCoords));
6226     }
6227 #else
6228     PetscCall(PetscSFBcastBegin(sfVert, coordtype, vertexCoords, coords, MPI_REPLACE));
6229     PetscCall(PetscSFBcastEnd(sfVert, coordtype, vertexCoords, coords, MPI_REPLACE));
6230 #endif
6231     PetscCallMPI(MPI_Type_free(&coordtype));
6232   }
6233   PetscCall(VecRestoreArray(coordinates, &coords));
6234   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
6235   PetscCall(VecDestroy(&coordinates));
6236   PetscCall(PetscLogEventEnd(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
6237   PetscFunctionReturn(PETSC_SUCCESS);
6238 }
6239 
6240 /*@
6241   DMPlexCreateFromCellListParallelPetsc - Create distributed `DMPLEX` from a list of vertices for each cell (common mesh generator output) where all cells have the same celltype
6242 
6243   Collective
6244 
6245   Input Parameters:
6246 + comm         - The communicator
6247 . dim          - The topological dimension of the mesh
6248 . numCells     - The number of cells owned by this process
6249 . numVertices  - The number of vertices owned by this process, or `PETSC_DECIDE`
6250 . NVertices    - The global number of vertices, or `PETSC_DECIDE`
6251 . numCorners   - The number of vertices for each cell
6252 . interpolate  - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
6253 . cells        - An array of numCells*numCorners numbers, the global vertex numbers for each cell
6254 . spaceDim     - The spatial dimension used for coordinates
6255 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
6256 
6257   Output Parameters:
6258 + dm          - The `DM`
6259 . vertexSF    - (Optional) `PetscSF` describing complete vertex ownership
6260 - verticesAdj - (Optional) vertex adjacency array
6261 
6262   Level: intermediate
6263 
6264   Notes:
6265   This function is just a convenient sequence of `DMCreate()`, `DMSetType()`, `DMSetDimension()`,
6266   `DMPlexBuildFromCellListParallel()`, `DMPlexInterpolate()`, `DMPlexBuildCoordinatesFromCellListParallel()`
6267 
6268   See `DMPlexBuildFromCellListParallel()` for an example and details about the topology-related parameters.
6269 
6270   See `DMPlexBuildCoordinatesFromCellListParallel()` for details about the geometry-related parameters.
6271 
6272 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromCellListPetsc()`, `DMPlexBuildFromCellListParallel()`, `DMPlexBuildCoordinatesFromCellListParallel()`, `DMPlexCreateFromDAG()`, `DMPlexCreate()`
6273 @*/
6274 PetscErrorCode DMPlexCreateFromCellListParallelPetsc(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt NVertices, PetscInt numCorners, PetscBool interpolate, const PetscInt cells[], PetscInt spaceDim, const PetscReal vertexCoords[], PeOp PetscSF *vertexSF, PeOp PetscInt *verticesAdj[], DM *dm)
6275 {
6276   PetscSF sfVert;
6277 
6278   PetscFunctionBegin;
6279   PetscCall(DMCreate(comm, dm));
6280   PetscCall(DMSetType(*dm, DMPLEX));
6281   PetscValidLogicalCollectiveInt(*dm, dim, 2);
6282   PetscValidLogicalCollectiveInt(*dm, spaceDim, 9);
6283   PetscCall(DMSetDimension(*dm, dim));
6284   PetscCall(DMPlexBuildFromCellListParallel(*dm, numCells, numVertices, NVertices, numCorners, cells, &sfVert, verticesAdj));
6285   if (interpolate) {
6286     DM idm;
6287 
6288     PetscCall(DMPlexInterpolate(*dm, &idm));
6289     PetscCall(DMDestroy(dm));
6290     *dm = idm;
6291   }
6292   PetscCall(DMPlexBuildCoordinatesFromCellListParallel(*dm, spaceDim, sfVert, vertexCoords));
6293   if (vertexSF) *vertexSF = sfVert;
6294   else PetscCall(PetscSFDestroy(&sfVert));
6295   PetscFunctionReturn(PETSC_SUCCESS);
6296 }
6297 
6298 /*@
6299   DMPlexCreateFromCellSectionParallel - Create distributed `DMPLEX` from a list of vertices for each cell (common mesh generator output) and supports multiple celltypes
6300 
6301   Collective
6302 
6303   Input Parameters:
6304 + comm         - The communicator
6305 . dim          - The topological dimension of the mesh
6306 . numCells     - The number of cells owned by this process
6307 . numVertices  - The number of vertices owned by this process, or `PETSC_DECIDE`
6308 . NVertices    - The global number of vertices, or `PETSC_DECIDE`
6309 . cellSection  - The `PetscSection` giving the number of vertices for each cell (layout of cells)
6310 . interpolate  - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
6311 . cells        - An array of the global vertex numbers for each cell
6312 . spaceDim     - The spatial dimension used for coordinates
6313 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
6314 
6315   Output Parameters:
6316 + dm          - The `DM`
6317 . vertexSF    - (Optional) `PetscSF` describing complete vertex ownership
6318 - verticesAdj - (Optional) vertex adjacency array
6319 
6320   Level: intermediate
6321 
6322   Notes:
6323   This function is just a convenient sequence of `DMCreate()`, `DMSetType()`, `DMSetDimension()`,
6324   `DMPlexBuildFromCellSectionParallel()`, `DMPlexInterpolate()`, `DMPlexBuildCoordinatesFromCellListParallel()`
6325 
6326   See `DMPlexBuildFromCellSectionParallel()` for an example and details about the topology-related parameters.
6327 
6328   See `DMPlexBuildCoordinatesFromCellListParallel()` for details about the geometry-related parameters.
6329 
6330 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromCellListPetsc()`, `DMPlexBuildFromCellListParallel()`, `DMPlexBuildCoordinatesFromCellListParallel()`, `DMPlexCreateFromDAG()`, `DMPlexCreate()`
6331 @*/
6332 PetscErrorCode DMPlexCreateFromCellSectionParallel(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt NVertices, PetscSection cellSection, PetscBool interpolate, const PetscInt cells[], PetscInt spaceDim, const PetscReal vertexCoords[], PeOp PetscSF *vertexSF, PeOp PetscInt *verticesAdj[], DM *dm)
6333 {
6334   PetscSF sfVert;
6335 
6336   PetscFunctionBegin;
6337   PetscCall(DMCreate(comm, dm));
6338   PetscCall(DMSetType(*dm, DMPLEX));
6339   PetscValidLogicalCollectiveInt(*dm, dim, 2);
6340   PetscValidLogicalCollectiveInt(*dm, spaceDim, 9);
6341   PetscCall(DMSetDimension(*dm, dim));
6342   PetscCall(DMPlexBuildFromCellSectionParallel(*dm, numCells, numVertices, NVertices, cellSection, cells, &sfVert, verticesAdj));
6343   if (interpolate) {
6344     DM idm;
6345 
6346     PetscCall(DMPlexInterpolate(*dm, &idm));
6347     PetscCall(DMDestroy(dm));
6348     *dm = idm;
6349   }
6350   PetscCall(DMPlexBuildCoordinatesFromCellListParallel(*dm, spaceDim, sfVert, vertexCoords));
6351   if (vertexSF) *vertexSF = sfVert;
6352   else PetscCall(PetscSFDestroy(&sfVert));
6353   PetscFunctionReturn(PETSC_SUCCESS);
6354 }
6355 
6356 /*@
6357   DMPlexBuildFromCellList - Build `DMPLEX` topology from a list of vertices for each cell (common mesh generator output)
6358 
6359   Collective; No Fortran Support
6360 
6361   Input Parameters:
6362 + dm          - The `DM`
6363 . numCells    - The number of cells owned by this process
6364 . numVertices - The number of vertices owned by this process, or `PETSC_DETERMINE`
6365 . numCorners  - The number of vertices for each cell
6366 - cells       - An array of `numCells` x `numCorners` numbers, the global vertex numbers for each cell
6367 
6368   Level: advanced
6369 
6370   Notes:
6371   Two triangles sharing a face
6372 .vb
6373 
6374         2
6375       / | \
6376      /  |  \
6377     /   |   \
6378    0  0 | 1  3
6379     \   |   /
6380      \  |  /
6381       \ | /
6382         1
6383 .ve
6384   would have input
6385 .vb
6386   numCells = 2, numVertices = 4
6387   cells = [0 1 2  1 3 2]
6388 .ve
6389   which would result in the `DMPLEX`
6390 .vb
6391 
6392         4
6393       / | \
6394      /  |  \
6395     /   |   \
6396    2  0 | 1  5
6397     \   |   /
6398      \  |  /
6399       \ | /
6400         3
6401 .ve
6402 
6403   If numVertices is `PETSC_DETERMINE`, it is computed by PETSc as the maximum vertex index in cells + 1.
6404 
6405 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexBuildFromCellListParallel()`, `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromCellListPetsc()`
6406 @*/
6407 PetscErrorCode DMPlexBuildFromCellList(DM dm, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, const PetscInt cells[])
6408 {
6409   PetscInt *cones, c, p, dim;
6410 
6411   PetscFunctionBegin;
6412   PetscCall(PetscLogEventBegin(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
6413   PetscCall(DMGetDimension(dm, &dim));
6414   /* Get/check global number of vertices */
6415   {
6416     PetscInt       NVerticesInCells, i;
6417     const PetscInt len = numCells * numCorners;
6418 
6419     /* NVerticesInCells = max(cells) + 1 */
6420     NVerticesInCells = PETSC_INT_MIN;
6421     for (i = 0; i < len; i++)
6422       if (cells[i] > NVerticesInCells) NVerticesInCells = cells[i];
6423     ++NVerticesInCells;
6424 
6425     if (numVertices == PETSC_DECIDE) numVertices = NVerticesInCells;
6426     else
6427       PetscCheck(numVertices >= NVerticesInCells, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Specified number of vertices %" PetscInt_FMT " must be greater than or equal to the number of vertices in cells %" PetscInt_FMT, numVertices, NVerticesInCells);
6428   }
6429   PetscCall(DMPlexSetChart(dm, 0, numCells + numVertices));
6430   for (c = 0; c < numCells; ++c) PetscCall(DMPlexSetConeSize(dm, c, numCorners));
6431   PetscCall(DMSetUp(dm));
6432   PetscCall(DMPlexGetCones(dm, &cones));
6433   for (c = 0; c < numCells; ++c) {
6434     for (p = 0; p < numCorners; ++p) cones[c * numCorners + p] = cells[c * numCorners + p] + numCells;
6435   }
6436   PetscCall(DMPlexSymmetrize(dm));
6437   PetscCall(DMPlexStratify(dm));
6438   PetscCall(PetscLogEventEnd(DMPLEX_BuildFromCellList, dm, 0, 0, 0));
6439   PetscFunctionReturn(PETSC_SUCCESS);
6440 }
6441 
6442 /*@
6443   DMPlexBuildCoordinatesFromCellList - Build `DM` coordinates from a list of coordinates for each owned vertex (common mesh generator output)
6444 
6445   Collective
6446 
6447   Input Parameters:
6448 + dm           - The `DM`
6449 . spaceDim     - The spatial dimension used for coordinates
6450 - vertexCoords - An array of numVertices*spaceDim numbers, the coordinates of each vertex
6451 
6452   Level: advanced
6453 
6454 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexBuildCoordinatesFromCellListParallel()`, `DMPlexCreateFromCellListPetsc()`, `DMPlexBuildFromCellList()`
6455 @*/
6456 PetscErrorCode DMPlexBuildCoordinatesFromCellList(DM dm, PetscInt spaceDim, const PetscReal vertexCoords[])
6457 {
6458   PetscSection coordSection;
6459   Vec          coordinates;
6460   DM           cdm;
6461   PetscScalar *coords;
6462   PetscInt     v, vStart, vEnd, d;
6463 
6464   PetscFunctionBegin;
6465   PetscCall(PetscLogEventBegin(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
6466   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
6467   PetscCheck(vStart >= 0 && vEnd >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM is not set up properly. DMPlexBuildFromCellList() should be called first.");
6468   PetscCall(DMSetCoordinateDim(dm, spaceDim));
6469   PetscCall(DMGetCoordinateSection(dm, &coordSection));
6470   PetscCall(PetscSectionSetNumFields(coordSection, 1));
6471   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, spaceDim));
6472   PetscCall(PetscSectionSetChart(coordSection, vStart, vEnd));
6473   for (v = vStart; v < vEnd; ++v) {
6474     PetscCall(PetscSectionSetDof(coordSection, v, spaceDim));
6475     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, spaceDim));
6476   }
6477   PetscCall(PetscSectionSetUp(coordSection));
6478 
6479   PetscCall(DMGetCoordinateDM(dm, &cdm));
6480   PetscCall(DMCreateLocalVector(cdm, &coordinates));
6481   PetscCall(VecSetBlockSize(coordinates, spaceDim));
6482   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
6483   PetscCall(VecGetArrayWrite(coordinates, &coords));
6484   for (v = 0; v < vEnd - vStart; ++v) {
6485     for (d = 0; d < spaceDim; ++d) coords[v * spaceDim + d] = vertexCoords[v * spaceDim + d];
6486   }
6487   PetscCall(VecRestoreArrayWrite(coordinates, &coords));
6488   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
6489   PetscCall(VecDestroy(&coordinates));
6490   PetscCall(PetscLogEventEnd(DMPLEX_BuildCoordinatesFromCellList, dm, 0, 0, 0));
6491   PetscFunctionReturn(PETSC_SUCCESS);
6492 }
6493 
6494 /*@
6495   DMPlexCreateFromCellListPetsc - Create `DMPLEX` from a list of vertices for each cell (common mesh generator output), but only process 0 takes in the input
6496 
6497   Collective
6498 
6499   Input Parameters:
6500 + comm         - The communicator
6501 . dim          - The topological dimension of the mesh
6502 . numCells     - The number of cells, only on process 0
6503 . numVertices  - The number of vertices owned by this process, or `PETSC_DECIDE`, only on process 0
6504 . numCorners   - The number of vertices for each cell, only on process 0
6505 . interpolate  - Flag indicating that intermediate mesh entities (faces, edges) should be created automatically
6506 . cells        - An array of $ numCells \times numCorners$ numbers, the vertices for each cell, only on process 0
6507 . spaceDim     - The spatial dimension used for coordinates
6508 - vertexCoords - An array of $ numVertices \times spaceDim$ numbers, the coordinates of each vertex, only on process 0
6509 
6510   Output Parameter:
6511 . dm - The `DM`, which only has points on process 0
6512 
6513   Level: intermediate
6514 
6515   Notes:
6516   This function is just a convenient sequence of `DMCreate()`, `DMSetType()`, `DMSetDimension()`, `DMPlexBuildFromCellList()`,
6517   `DMPlexInterpolate()`, `DMPlexBuildCoordinatesFromCellList()`
6518 
6519   See `DMPlexBuildFromCellList()` for an example and details about the topology-related parameters.
6520   See `DMPlexBuildCoordinatesFromCellList()` for details about the geometry-related parameters.
6521   See `DMPlexCreateFromCellListParallelPetsc()` for parallel input
6522 
6523 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromCellListParallelPetsc()`, `DMPlexBuildFromCellList()`, `DMPlexBuildCoordinatesFromCellList()`, `DMPlexCreateFromDAG()`, `DMPlexCreate()`
6524 @*/
6525 PetscErrorCode DMPlexCreateFromCellListPetsc(MPI_Comm comm, PetscInt dim, PetscInt numCells, PetscInt numVertices, PetscInt numCorners, PetscBool interpolate, const PetscInt cells[], PetscInt spaceDim, const PetscReal vertexCoords[], DM *dm)
6526 {
6527   PetscMPIInt rank;
6528 
6529   PetscFunctionBegin;
6530   PetscCheck(dim, comm, PETSC_ERR_ARG_OUTOFRANGE, "This is not appropriate for 0-dimensional meshes. Consider either creating the DM using DMPlexCreateFromDAG(), by hand, or using DMSwarm.");
6531   PetscCallMPI(MPI_Comm_rank(comm, &rank));
6532   PetscCall(DMCreate(comm, dm));
6533   PetscCall(DMSetType(*dm, DMPLEX));
6534   PetscCall(DMSetDimension(*dm, dim));
6535   if (rank == 0) PetscCall(DMPlexBuildFromCellList(*dm, numCells, numVertices, numCorners, cells));
6536   else PetscCall(DMPlexBuildFromCellList(*dm, 0, 0, 0, NULL));
6537   if (interpolate) {
6538     DM idm;
6539 
6540     PetscCall(DMPlexInterpolate(*dm, &idm));
6541     PetscCall(DMDestroy(dm));
6542     *dm = idm;
6543   }
6544   if (rank == 0) PetscCall(DMPlexBuildCoordinatesFromCellList(*dm, spaceDim, vertexCoords));
6545   else PetscCall(DMPlexBuildCoordinatesFromCellList(*dm, spaceDim, NULL));
6546   PetscFunctionReturn(PETSC_SUCCESS);
6547 }
6548 
6549 /*@
6550   DMPlexCreateFromDAG - This takes as input the adjacency-list representation of the Directed Acyclic Graph (Hasse Diagram) encoding a mesh, and produces a `DM`
6551 
6552   Input Parameters:
6553 + dm               - The empty `DM` object, usually from `DMCreate()` and `DMSetDimension()`
6554 . depth            - The depth of the DAG
6555 . numPoints        - Array of size $ depth + 1 $ containing the number of points at each `depth`
6556 . coneSize         - The cone size of each point
6557 . cones            - The concatenation of the cone points for each point, the cone list must be oriented correctly for each point
6558 . coneOrientations - The orientation of each cone point
6559 - vertexCoords     - An array of $ numPoints[0] \times spacedim $ numbers representing the coordinates of each vertex, with `spacedim` the value set via `DMSetCoordinateDim()`
6560 
6561   Output Parameter:
6562 . dm - The `DM`
6563 
6564   Level: advanced
6565 
6566   Note:
6567   Two triangles sharing a face would have input
6568 .vb
6569   depth = 1, numPoints = [4 2], coneSize = [3 3 0 0 0 0]
6570   cones = [2 3 4  3 5 4], coneOrientations = [0 0 0  0 0 0]
6571  vertexCoords = [-1.0 0.0  0.0 -1.0  0.0 1.0  1.0 0.0]
6572 .ve
6573   which would result in the DMPlex
6574 .vb
6575         4
6576       / | \
6577      /  |  \
6578     /   |   \
6579    2  0 | 1  5
6580     \   |   /
6581      \  |  /
6582       \ | /
6583         3
6584 .ve
6585   Notice that all points are numbered consecutively, unlike `DMPlexCreateFromCellListPetsc()`
6586 
6587   Developer Note:
6588   This does not create anything so should not have create in the name.
6589 
6590 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromCellListPetsc()`, `DMPlexCreate()`
6591 @*/
6592 PetscErrorCode DMPlexCreateFromDAG(DM dm, PetscInt depth, const PetscInt numPoints[], const PetscInt coneSize[], const PetscInt cones[], const PetscInt coneOrientations[], const PetscScalar vertexCoords[])
6593 {
6594   Vec          coordinates;
6595   PetscSection coordSection;
6596   PetscScalar *coords;
6597   PetscInt     coordSize, firstVertex = -1, pStart = 0, pEnd = 0, p, v, dim, dimEmbed, d, off;
6598 
6599   PetscFunctionBegin;
6600   PetscCall(DMGetDimension(dm, &dim));
6601   PetscCall(DMGetCoordinateDim(dm, &dimEmbed));
6602   PetscCheck(dimEmbed >= dim, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Embedding dimension %" PetscInt_FMT " cannot be less than intrinsic dimension %" PetscInt_FMT, dimEmbed, dim);
6603   for (d = 0; d <= depth; ++d) pEnd += numPoints[d];
6604   PetscCall(DMPlexSetChart(dm, pStart, pEnd));
6605   for (p = pStart; p < pEnd; ++p) {
6606     PetscCall(DMPlexSetConeSize(dm, p, coneSize[p - pStart]));
6607     if (firstVertex < 0 && !coneSize[p - pStart]) firstVertex = p - pStart;
6608   }
6609   PetscCheck(firstVertex >= 0 || !numPoints[0], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Expected %" PetscInt_FMT " vertices but could not find any", numPoints[0]);
6610   PetscCall(DMSetUp(dm)); /* Allocate space for cones */
6611   for (p = pStart, off = 0; p < pEnd; off += coneSize[p - pStart], ++p) {
6612     PetscCall(DMPlexSetCone(dm, p, &cones[off]));
6613     PetscCall(DMPlexSetConeOrientation(dm, p, &coneOrientations[off]));
6614   }
6615   PetscCall(DMPlexSymmetrize(dm));
6616   PetscCall(DMPlexStratify(dm));
6617   /* Build coordinates */
6618   PetscCall(DMGetCoordinateSection(dm, &coordSection));
6619   PetscCall(PetscSectionSetNumFields(coordSection, 1));
6620   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, dimEmbed));
6621   PetscCall(PetscSectionSetChart(coordSection, firstVertex, firstVertex + numPoints[0]));
6622   for (v = firstVertex; v < firstVertex + numPoints[0]; ++v) {
6623     PetscCall(PetscSectionSetDof(coordSection, v, dimEmbed));
6624     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, dimEmbed));
6625   }
6626   PetscCall(PetscSectionSetUp(coordSection));
6627   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
6628   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
6629   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
6630   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
6631   PetscCall(VecSetBlockSize(coordinates, PetscMax(dimEmbed, 1)));
6632   PetscCall(VecSetType(coordinates, VECSTANDARD));
6633   if (vertexCoords) {
6634     PetscCall(VecGetArray(coordinates, &coords));
6635     for (v = 0; v < numPoints[0]; ++v) {
6636       PetscInt off;
6637 
6638       PetscCall(PetscSectionGetOffset(coordSection, v + firstVertex, &off));
6639       for (d = 0; d < dimEmbed; ++d) coords[off + d] = vertexCoords[v * dimEmbed + d];
6640     }
6641   }
6642   PetscCall(VecRestoreArray(coordinates, &coords));
6643   PetscCall(DMSetCoordinatesLocal(dm, coordinates));
6644   PetscCall(VecDestroy(&coordinates));
6645   PetscFunctionReturn(PETSC_SUCCESS);
6646 }
6647 
6648 /*
6649   DMPlexCreateCellVertexFromFile - Create a `DMPLEX` mesh from a simple cell-vertex file.
6650 
6651   Collective
6652 
6653 + comm        - The MPI communicator
6654 . filename    - Name of the .dat file
6655 - interpolate - Create faces and edges in the mesh
6656 
6657   Output Parameter:
6658 . dm  - The `DM` object representing the mesh
6659 
6660   Level: beginner
6661 
6662   Note:
6663   The format is the simplest possible:
6664 .vb
6665   dim Ne Nv Nc Nl
6666   v_1 v_2 ... v_Nc
6667   ...
6668   x y z marker_1 ... marker_Nl
6669 .ve
6670 
6671   Developer Note:
6672   Should use a `PetscViewer` not a filename
6673 
6674 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromFile()`, `DMPlexCreateGmsh()`, `DMPlexCreate()`
6675 */
6676 static PetscErrorCode DMPlexCreateCellVertexFromFile(MPI_Comm comm, const char filename[], PetscBool interpolate, DM *dm)
6677 {
6678   DMLabel      marker;
6679   PetscViewer  viewer;
6680   Vec          coordinates;
6681   PetscSection coordSection;
6682   PetscScalar *coords;
6683   char         line[PETSC_MAX_PATH_LEN];
6684   PetscInt     cdim, coordSize, v, c, d;
6685   PetscMPIInt  rank;
6686   int          snum, dim, Nv, Nc, Ncn, Nl;
6687 
6688   PetscFunctionBegin;
6689   PetscCallMPI(MPI_Comm_rank(comm, &rank));
6690   PetscCall(PetscViewerCreate(comm, &viewer));
6691   PetscCall(PetscViewerSetType(viewer, PETSCVIEWERASCII));
6692   PetscCall(PetscViewerFileSetMode(viewer, FILE_MODE_READ));
6693   PetscCall(PetscViewerFileSetName(viewer, filename));
6694   if (rank == 0) {
6695     PetscCall(PetscViewerRead(viewer, line, 5, NULL, PETSC_STRING));
6696     snum = sscanf(line, "%d %d %d %d %d", &dim, &Nc, &Nv, &Ncn, &Nl);
6697     PetscCheck(snum == 5, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
6698   } else {
6699     Nc = Nv = Ncn = Nl = 0;
6700   }
6701   PetscCallMPI(MPI_Bcast(&dim, 1, MPI_INT, 0, comm));
6702   cdim = dim;
6703   PetscCall(DMCreate(comm, dm));
6704   PetscCall(DMSetType(*dm, DMPLEX));
6705   PetscCall(DMPlexSetChart(*dm, 0, Nc + Nv));
6706   PetscCall(DMSetDimension(*dm, dim));
6707   PetscCall(DMSetCoordinateDim(*dm, cdim));
6708   /* Read topology */
6709   if (rank == 0) {
6710     char     format[PETSC_MAX_PATH_LEN];
6711     PetscInt cone[8];
6712     int      vbuf[8], v;
6713 
6714     for (c = 0; c < Ncn; ++c) {
6715       format[c * 3 + 0] = '%';
6716       format[c * 3 + 1] = 'd';
6717       format[c * 3 + 2] = ' ';
6718     }
6719     format[Ncn * 3 - 1] = '\0';
6720     for (c = 0; c < Nc; ++c) PetscCall(DMPlexSetConeSize(*dm, c, Ncn));
6721     PetscCall(DMSetUp(*dm));
6722     for (c = 0; c < Nc; ++c) {
6723       PetscCall(PetscViewerRead(viewer, line, Ncn, NULL, PETSC_STRING));
6724       switch (Ncn) {
6725       case 2:
6726         snum = sscanf(line, format, &vbuf[0], &vbuf[1]);
6727         break;
6728       case 3:
6729         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2]);
6730         break;
6731       case 4:
6732         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3]);
6733         break;
6734       case 6:
6735         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3], &vbuf[4], &vbuf[5]);
6736         break;
6737       case 8:
6738         snum = sscanf(line, format, &vbuf[0], &vbuf[1], &vbuf[2], &vbuf[3], &vbuf[4], &vbuf[5], &vbuf[6], &vbuf[7]);
6739         break;
6740       default:
6741         SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "No cell shape with %d vertices", Ncn);
6742       }
6743       PetscCheck(snum == Ncn, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
6744       for (v = 0; v < Ncn; ++v) cone[v] = vbuf[v] + Nc;
6745       /* Hexahedra are inverted */
6746       if (Ncn == 8) {
6747         PetscInt tmp = cone[1];
6748         cone[1]      = cone[3];
6749         cone[3]      = tmp;
6750       }
6751       PetscCall(DMPlexSetCone(*dm, c, cone));
6752     }
6753   }
6754   PetscCall(DMPlexSymmetrize(*dm));
6755   PetscCall(DMPlexStratify(*dm));
6756   /* Read coordinates */
6757   PetscCall(DMGetCoordinateSection(*dm, &coordSection));
6758   PetscCall(PetscSectionSetNumFields(coordSection, 1));
6759   PetscCall(PetscSectionSetFieldComponents(coordSection, 0, cdim));
6760   PetscCall(PetscSectionSetChart(coordSection, Nc, Nc + Nv));
6761   for (v = Nc; v < Nc + Nv; ++v) {
6762     PetscCall(PetscSectionSetDof(coordSection, v, cdim));
6763     PetscCall(PetscSectionSetFieldDof(coordSection, v, 0, cdim));
6764   }
6765   PetscCall(PetscSectionSetUp(coordSection));
6766   PetscCall(PetscSectionGetStorageSize(coordSection, &coordSize));
6767   PetscCall(VecCreate(PETSC_COMM_SELF, &coordinates));
6768   PetscCall(PetscObjectSetName((PetscObject)coordinates, "coordinates"));
6769   PetscCall(VecSetSizes(coordinates, coordSize, PETSC_DETERMINE));
6770   PetscCall(VecSetBlockSize(coordinates, cdim));
6771   PetscCall(VecSetType(coordinates, VECSTANDARD));
6772   PetscCall(VecGetArray(coordinates, &coords));
6773   if (rank == 0) {
6774     char   format[PETSC_MAX_PATH_LEN];
6775     double x[3];
6776     int    l, val[3];
6777 
6778     if (Nl) {
6779       for (l = 0; l < Nl; ++l) {
6780         format[l * 3 + 0] = '%';
6781         format[l * 3 + 1] = 'd';
6782         format[l * 3 + 2] = ' ';
6783       }
6784       format[Nl * 3 - 1] = '\0';
6785       PetscCall(DMCreateLabel(*dm, "marker"));
6786       PetscCall(DMGetLabel(*dm, "marker", &marker));
6787     }
6788     for (v = 0; v < Nv; ++v) {
6789       PetscCall(PetscViewerRead(viewer, line, 3 + Nl, NULL, PETSC_STRING));
6790       snum = sscanf(line, "%lg %lg %lg", &x[0], &x[1], &x[2]);
6791       PetscCheck(snum == 3, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
6792       switch (Nl) {
6793       case 0:
6794         snum = 0;
6795         break;
6796       case 1:
6797         snum = sscanf(line, format, &val[0]);
6798         break;
6799       case 2:
6800         snum = sscanf(line, format, &val[0], &val[1]);
6801         break;
6802       case 3:
6803         snum = sscanf(line, format, &val[0], &val[1], &val[2]);
6804         break;
6805       default:
6806         SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Request support for %d labels", Nl);
6807       }
6808       PetscCheck(snum == Nl, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unable to parse cell-vertex file: %s", line);
6809       for (d = 0; d < cdim; ++d) coords[v * cdim + d] = x[d];
6810       for (l = 0; l < Nl; ++l) PetscCall(DMLabelSetValue(marker, v + Nc, val[l]));
6811     }
6812   }
6813   PetscCall(VecRestoreArray(coordinates, &coords));
6814   PetscCall(DMSetCoordinatesLocal(*dm, coordinates));
6815   PetscCall(VecDestroy(&coordinates));
6816   PetscCall(PetscViewerDestroy(&viewer));
6817   if (interpolate) {
6818     DM      idm;
6819     DMLabel bdlabel;
6820 
6821     PetscCall(DMPlexInterpolate(*dm, &idm));
6822     PetscCall(DMDestroy(dm));
6823     *dm = idm;
6824 
6825     if (!Nl) {
6826       PetscCall(DMCreateLabel(*dm, "marker"));
6827       PetscCall(DMGetLabel(*dm, "marker", &bdlabel));
6828       PetscCall(DMPlexMarkBoundaryFaces(*dm, PETSC_DETERMINE, bdlabel));
6829       PetscCall(DMPlexLabelComplete(*dm, bdlabel));
6830     }
6831   }
6832   PetscFunctionReturn(PETSC_SUCCESS);
6833 }
6834 
6835 /*@
6836   DMPlexCreateFromFile - This takes a filename and produces a `DM`
6837 
6838   Collective
6839 
6840   Input Parameters:
6841 + comm        - The communicator
6842 . filename    - A file name
6843 . plexname    - The object name of the resulting `DM`, also used for intra-datafile lookup by some formats
6844 - interpolate - Flag to create intermediate mesh pieces (edges, faces)
6845 
6846   Output Parameter:
6847 . dm - The `DM`
6848 
6849   Options Database Key:
6850 . -dm_plex_create_from_hdf5_xdmf - use the `PETSC_VIEWER_HDF5_XDMF` format for reading HDF5
6851 
6852   Use `-dm_plex_create_ prefix` to pass options to the internal `PetscViewer`, e.g.
6853 $ -dm_plex_create_viewer_hdf5_collective
6854 
6855   Level: beginner
6856 
6857   Notes:
6858   Using `PETSCVIEWERHDF5` type with `PETSC_VIEWER_HDF5_PETSC` format, one can save multiple `DMPLEX`
6859   meshes in a single HDF5 file. This in turn requires one to name the `DMPLEX` object with `PetscObjectSetName()`
6860   before saving it with `DMView()` and before loading it with `DMLoad()` for identification of the mesh object.
6861   The input parameter name is thus used to name the `DMPLEX` object when `DMPlexCreateFromFile()` internally
6862   calls `DMLoad()`. Currently, name is ignored for other viewer types and/or formats.
6863 
6864 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateFromDAG()`, `DMPlexCreateFromCellListPetsc()`, `DMPlexCreate()`, `PetscObjectSetName()`, `DMView()`, `DMLoad()`
6865 @*/
6866 PetscErrorCode DMPlexCreateFromFile(MPI_Comm comm, const char filename[], const char plexname[], PetscBool interpolate, DM *dm)
6867 {
6868   const char  extGmsh[]      = ".msh";
6869   const char  extGmsh2[]     = ".msh2";
6870   const char  extGmsh4[]     = ".msh4";
6871   const char  extCGNS[]      = ".cgns";
6872   const char  extExodus[]    = ".exo";
6873   const char  extExodus_e[]  = ".e";
6874   const char  extGenesis[]   = ".gen";
6875   const char  extFluent[]    = ".cas";
6876   const char  extHDF5[]      = ".h5";
6877   const char  extXDMFHDF5[]  = ".xdmf.h5";
6878   const char  extPLY[]       = ".ply";
6879   const char  extEGADSlite[] = ".egadslite";
6880   const char  extEGADS[]     = ".egads";
6881   const char  extIGES[]      = ".igs";
6882   const char  extIGES2[]     = ".iges";
6883   const char  extSTEP[]      = ".stp";
6884   const char  extSTEP2[]     = ".step";
6885   const char  extBREP[]      = ".brep";
6886   const char  extCV[]        = ".dat";
6887   size_t      len;
6888   PetscBool   isGmsh, isGmsh2, isGmsh4, isCGNS, isExodus, isGenesis, isFluent, isHDF5, isPLY, isEGADSlite, isEGADS, isIGES, isIGES2, isSTEP, isSTEP2, isBREP, isCV, isXDMFHDF5;
6889   PetscMPIInt rank;
6890 
6891   PetscFunctionBegin;
6892   PetscAssertPointer(filename, 2);
6893   if (plexname) PetscAssertPointer(plexname, 3);
6894   PetscAssertPointer(dm, 5);
6895   PetscCall(DMInitializePackage());
6896   PetscCall(PetscLogEventBegin(DMPLEX_CreateFromFile, 0, 0, 0, 0));
6897   PetscCallMPI(MPI_Comm_rank(comm, &rank));
6898   PetscCall(PetscStrlen(filename, &len));
6899   PetscCheck(len, comm, PETSC_ERR_ARG_WRONG, "Filename must be a valid path");
6900 
6901 #define CheckExtension(extension__, is_extension__) \
6902   do { \
6903     PetscAssert(sizeof(extension__), comm, PETSC_ERR_PLIB, "Zero-size extension: %s", extension__); \
6904     /* don't count the null-terminator at the end */ \
6905     const size_t ext_len = sizeof(extension__) - 1; \
6906     if (len < ext_len) { \
6907       is_extension__ = PETSC_FALSE; \
6908     } else { \
6909       PetscCall(PetscStrncmp(filename + len - ext_len, extension__, ext_len, &is_extension__)); \
6910     } \
6911   } while (0)
6912 
6913   CheckExtension(extGmsh, isGmsh);
6914   CheckExtension(extGmsh2, isGmsh2);
6915   CheckExtension(extGmsh4, isGmsh4);
6916   CheckExtension(extCGNS, isCGNS);
6917   CheckExtension(extExodus, isExodus);
6918   if (!isExodus) CheckExtension(extExodus_e, isExodus);
6919   CheckExtension(extGenesis, isGenesis);
6920   CheckExtension(extFluent, isFluent);
6921   CheckExtension(extHDF5, isHDF5);
6922   CheckExtension(extPLY, isPLY);
6923   CheckExtension(extEGADSlite, isEGADSlite);
6924   CheckExtension(extEGADS, isEGADS);
6925   CheckExtension(extIGES, isIGES);
6926   CheckExtension(extIGES2, isIGES2);
6927   CheckExtension(extSTEP, isSTEP);
6928   CheckExtension(extSTEP2, isSTEP2);
6929   CheckExtension(extBREP, isBREP);
6930   CheckExtension(extCV, isCV);
6931   CheckExtension(extXDMFHDF5, isXDMFHDF5);
6932 
6933 #undef CheckExtension
6934 
6935   if (isGmsh || isGmsh2 || isGmsh4) {
6936     PetscCall(DMPlexCreateGmshFromFile(comm, filename, interpolate, dm));
6937   } else if (isCGNS) {
6938     PetscCall(DMPlexCreateCGNSFromFile(comm, filename, interpolate, dm));
6939   } else if (isExodus || isGenesis) {
6940     PetscCall(DMPlexCreateExodusFromFile(comm, filename, interpolate, dm));
6941   } else if (isFluent) {
6942     PetscCall(DMPlexCreateFluentFromFile(comm, filename, interpolate, dm));
6943   } else if (isHDF5) {
6944     PetscViewer viewer;
6945 
6946     /* PETSC_VIEWER_HDF5_XDMF is used if the filename ends with .xdmf.h5, or if -dm_plex_create_from_hdf5_xdmf option is present */
6947     PetscCall(PetscOptionsGetBool(NULL, NULL, "-dm_plex_create_from_hdf5_xdmf", &isXDMFHDF5, NULL));
6948     PetscCall(PetscViewerCreate(comm, &viewer));
6949     PetscCall(PetscViewerSetType(viewer, PETSCVIEWERHDF5));
6950     PetscCall(PetscViewerSetOptionsPrefix(viewer, "dm_plex_create_"));
6951     PetscCall(PetscViewerSetFromOptions(viewer));
6952     PetscCall(PetscViewerFileSetMode(viewer, FILE_MODE_READ));
6953     PetscCall(PetscViewerFileSetName(viewer, filename));
6954 
6955     PetscCall(DMCreate(comm, dm));
6956     PetscCall(PetscObjectSetName((PetscObject)*dm, plexname));
6957     PetscCall(DMSetType(*dm, DMPLEX));
6958     if (isXDMFHDF5) PetscCall(PetscViewerPushFormat(viewer, PETSC_VIEWER_HDF5_XDMF));
6959     PetscCall(DMLoad(*dm, viewer));
6960     if (isXDMFHDF5) PetscCall(PetscViewerPopFormat(viewer));
6961     PetscCall(PetscViewerDestroy(&viewer));
6962 
6963     if (interpolate) {
6964       DM idm;
6965 
6966       PetscCall(DMPlexInterpolate(*dm, &idm));
6967       PetscCall(DMDestroy(dm));
6968       *dm = idm;
6969     }
6970   } else if (isPLY) {
6971     PetscCall(DMPlexCreatePLYFromFile(comm, filename, interpolate, dm));
6972   } else if (isEGADSlite || isEGADS || isIGES || isIGES2 || isSTEP || isSTEP2 || isBREP) {
6973     PetscCall(DMPlexCreateGeomFromFile(comm, filename, dm, isEGADSlite));
6974 
6975     if (!interpolate) {
6976       DM udm;
6977 
6978       PetscCall(DMPlexUninterpolate(*dm, &udm));
6979       PetscCall(DMDestroy(dm));
6980       *dm = udm;
6981     }
6982   } else if (isCV) {
6983     PetscCall(DMPlexCreateCellVertexFromFile(comm, filename, interpolate, dm));
6984   } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cannot load file %s: unrecognized extension", filename);
6985   PetscCall(PetscStrlen(plexname, &len));
6986   if (len) PetscCall(PetscObjectSetName((PetscObject)*dm, plexname));
6987   PetscCall(PetscLogEventEnd(DMPLEX_CreateFromFile, 0, 0, 0, 0));
6988   PetscFunctionReturn(PETSC_SUCCESS);
6989 }
6990 
6991 /*@
6992   DMPlexCreateEphemeral - This takes a `DMPlexTransform` and a base `DMPlex` and produces an ephemeral `DM`, meaning one that is created on the fly in response to queries.
6993 
6994   Input Parameters:
6995 + tr     - The `DMPlexTransform`
6996 - prefix - An options prefix, or NULL
6997 
6998   Output Parameter:
6999 . dm - The `DM`
7000 
7001   Level: beginner
7002 
7003   Notes:
7004   An emphemeral mesh is one that is not stored concretely, as in the default `DMPLEX` implementation, but rather is produced on the fly in response to queries, using information from the transform and the base mesh.
7005 
7006 .seealso: `DMPlexCreateFromFile`, `DMPlexCreateFromDAG()`, `DMPlexCreateFromCellListPetsc()`, `DMPlexCreate()`
7007 @*/
7008 PetscErrorCode DMPlexCreateEphemeral(DMPlexTransform tr, const char prefix[], DM *dm)
7009 {
7010   DM           bdm, bcdm, cdm;
7011   Vec          coordinates, coordinatesNew;
7012   PetscSection cs;
7013   PetscInt     cdim, Nl;
7014 
7015   PetscFunctionBegin;
7016   PetscCall(DMCreate(PetscObjectComm((PetscObject)tr), dm));
7017   PetscCall(DMSetType(*dm, DMPLEX));
7018   ((DM_Plex *)(*dm)->data)->interpolated = DMPLEX_INTERPOLATED_FULL;
7019   // Handle coordinates
7020   PetscCall(DMPlexTransformGetDM(tr, &bdm));
7021   PetscCall(DMPlexTransformSetDimensions(tr, bdm, *dm));
7022   PetscCall(DMGetCoordinateDim(*dm, &cdim));
7023   PetscCall(DMGetCoordinateDM(bdm, &bcdm));
7024   PetscCall(DMGetCoordinateDM(*dm, &cdm));
7025   PetscCall(DMCopyDisc(bcdm, cdm));
7026   PetscCall(DMGetLocalSection(cdm, &cs));
7027   PetscCall(PetscSectionSetNumFields(cs, 1));
7028   PetscCall(PetscSectionSetFieldComponents(cs, 0, cdim));
7029   PetscCall(DMGetCoordinatesLocal(bdm, &coordinates));
7030   PetscCall(VecDuplicate(coordinates, &coordinatesNew));
7031   PetscCall(VecCopy(coordinates, coordinatesNew));
7032   PetscCall(DMSetCoordinatesLocal(*dm, coordinatesNew));
7033   PetscCall(VecDestroy(&coordinatesNew));
7034 
7035   PetscCall(PetscObjectReference((PetscObject)tr));
7036   PetscCall(DMPlexTransformDestroy(&((DM_Plex *)(*dm)->data)->tr));
7037   ((DM_Plex *)(*dm)->data)->tr = tr;
7038   PetscCall(DMPlexDistributeSetDefault(*dm, PETSC_FALSE));
7039   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)*dm, prefix));
7040   PetscCall(DMSetFromOptions(*dm));
7041 
7042   PetscCall(DMGetNumLabels(bdm, &Nl));
7043   for (PetscInt l = 0; l < Nl; ++l) {
7044     DMLabel     label, labelNew;
7045     const char *lname;
7046     PetscBool   isDepth, isCellType;
7047 
7048     PetscCall(DMGetLabelName(bdm, l, &lname));
7049     PetscCall(PetscStrcmp(lname, "depth", &isDepth));
7050     if (isDepth) continue;
7051     PetscCall(PetscStrcmp(lname, "celltype", &isCellType));
7052     if (isCellType) continue;
7053     PetscCall(DMCreateLabel(*dm, lname));
7054     PetscCall(DMGetLabel(bdm, lname, &label));
7055     PetscCall(DMGetLabel(*dm, lname, &labelNew));
7056     PetscCall(DMLabelSetType(labelNew, DMLABELEPHEMERAL));
7057     PetscCall(DMLabelEphemeralSetLabel(labelNew, label));
7058     PetscCall(DMLabelEphemeralSetTransform(labelNew, tr));
7059     PetscCall(DMLabelSetUp(labelNew));
7060   }
7061   PetscFunctionReturn(PETSC_SUCCESS);
7062 }
7063