xref: /petsc/src/dm/impls/plex/plex.c (revision d7c1f4409a34685d8dcd545a97d161d483d89f66)
1 #include <petsc/private/dmpleximpl.h> /*I      "petscdmplex.h"   I*/
2 #include <petsc/private/dmlabelimpl.h>
3 #include <petsc/private/isimpl.h>
4 #include <petsc/private/vecimpl.h>
5 #include <petsc/private/glvisvecimpl.h>
6 #include <petscsf.h>
7 #include <petscds.h>
8 #include <petscdraw.h>
9 #include <petscdmfield.h>
10 #include <petscdmplextransform.h>
11 #include <petscblaslapack.h>
12 
13 /* Logging support */
14 PetscLogEvent DMPLEX_Interpolate, DMPLEX_Partition, DMPLEX_Distribute, DMPLEX_DistributeCones, DMPLEX_DistributeLabels, DMPLEX_DistributeSF, DMPLEX_DistributeOverlap, DMPLEX_DistributeField, DMPLEX_DistributeData, DMPLEX_Migrate, DMPLEX_InterpolateSF, DMPLEX_GlobalToNaturalBegin, DMPLEX_GlobalToNaturalEnd, DMPLEX_NaturalToGlobalBegin, DMPLEX_NaturalToGlobalEnd, DMPLEX_Stratify, DMPLEX_Symmetrize, DMPLEX_Preallocate, DMPLEX_ResidualFEM, DMPLEX_JacobianFEM, DMPLEX_InterpolatorFEM, DMPLEX_InjectorFEM, DMPLEX_IntegralFEM, DMPLEX_CreateGmsh, DMPLEX_RebalanceSharedPoints, DMPLEX_PartSelf, DMPLEX_PartLabelInvert, DMPLEX_PartLabelCreateSF, DMPLEX_PartStratSF, DMPLEX_CreatePointSF, DMPLEX_LocatePoints, DMPLEX_TopologyView, DMPLEX_LabelsView, DMPLEX_CoordinatesView, DMPLEX_SectionView, DMPLEX_GlobalVectorView, DMPLEX_LocalVectorView, DMPLEX_TopologyLoad, DMPLEX_LabelsLoad, DMPLEX_CoordinatesLoad, DMPLEX_SectionLoad, DMPLEX_GlobalVectorLoad, DMPLEX_LocalVectorLoad;
15 PetscLogEvent DMPLEX_RebalBuildGraph, DMPLEX_RebalRewriteSF, DMPLEX_RebalGatherGraph, DMPLEX_RebalPartition, DMPLEX_RebalScatterPart, DMPLEX_Generate, DMPLEX_Transform, DMPLEX_GetLocalOffsets, DMPLEX_Uninterpolate;
16 
17 PetscBool  Plexcite       = PETSC_FALSE;
18 const char PlexCitation[] = "@article{LangeMitchellKnepleyGorman2015,\n"
19                             "title     = {Efficient mesh management in {Firedrake} using {PETSc-DMPlex}},\n"
20                             "author    = {Michael Lange and Lawrence Mitchell and Matthew G. Knepley and Gerard J. Gorman},\n"
21                             "journal   = {SIAM Journal on Scientific Computing},\n"
22                             "volume    = {38},\n"
23                             "number    = {5},\n"
24                             "pages     = {S143--S155},\n"
25                             "eprint    = {http://arxiv.org/abs/1506.07749},\n"
26                             "doi       = {10.1137/15M1026092},\n"
27                             "year      = {2016},\n"
28                             "petsc_uses={DMPlex},\n}\n";
29 
30 PETSC_EXTERN PetscErrorCode VecView_MPI(Vec, PetscViewer);
31 
32 /*@
33   DMPlexIsSimplex - Is the first cell in this mesh a simplex?
34 
35   Input Parameter:
36 . dm - The `DMPLEX` object
37 
38   Output Parameter:
39 . simplex - Flag checking for a simplex
40 
41   Level: intermediate
42 
43   Note:
44   This just gives the first range of cells found. If the mesh has several cell types, it will only give the first.
45   If the mesh has no cells, this returns `PETSC_FALSE`.
46 
47 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetSimplexOrBoxCells()`, `DMPlexGetCellType()`, `DMPlexGetHeightStratum()`, `DMPolytopeTypeGetNumVertices()`
48 @*/
49 PetscErrorCode DMPlexIsSimplex(DM dm, PetscBool *simplex)
50 {
51   DMPolytopeType ct;
52   PetscInt       cStart, cEnd;
53 
54   PetscFunctionBegin;
55   PetscCall(DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd));
56   if (cEnd <= cStart) {
57     *simplex = PETSC_FALSE;
58     PetscFunctionReturn(PETSC_SUCCESS);
59   }
60   PetscCall(DMPlexGetCellType(dm, cStart, &ct));
61   *simplex = DMPolytopeTypeGetNumVertices(ct) == DMPolytopeTypeGetDim(ct) + 1 ? PETSC_TRUE : PETSC_FALSE;
62   PetscFunctionReturn(PETSC_SUCCESS);
63 }
64 
65 /*@
66   DMPlexGetSimplexOrBoxCells - Get the range of cells which are neither prisms nor ghost FV cells
67 
68   Input Parameters:
69 + dm     - The `DMPLEX` object
70 - height - The cell height in the Plex, 0 is the default
71 
72   Output Parameters:
73 + cStart - The first "normal" cell
74 - cEnd   - The upper bound on "normal" cells
75 
76   Level: developer
77 
78   Note:
79   This function requires that tensor cells are ordered last.
80 
81 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexConstructGhostCells()`, `DMPlexGetCellTypeStratum()`
82 @*/
83 PetscErrorCode DMPlexGetSimplexOrBoxCells(DM dm, PetscInt height, PetscInt *cStart, PetscInt *cEnd)
84 {
85   DMLabel         ctLabel;
86   IS              valueIS;
87   const PetscInt *ctypes;
88   PetscBool       found = PETSC_FALSE;
89   PetscInt        Nct, cS = PETSC_INT_MAX, cE = 0;
90 
91   PetscFunctionBegin;
92   PetscCall(DMPlexGetCellTypeLabel(dm, &ctLabel));
93   PetscCall(DMLabelGetValueIS(ctLabel, &valueIS));
94   PetscCall(ISGetLocalSize(valueIS, &Nct));
95   PetscCall(ISGetIndices(valueIS, &ctypes));
96   for (PetscInt t = 0; t < Nct; ++t) {
97     const DMPolytopeType ct = (DMPolytopeType)ctypes[t];
98     PetscInt             ctS, ctE, ht;
99 
100     if (ct == DM_POLYTOPE_UNKNOWN) {
101       // If any cells are not typed, just use all cells
102       PetscCall(DMPlexGetHeightStratum(dm, PetscMax(height, 0), cStart, cEnd));
103       break;
104     }
105     if (DMPolytopeTypeIsHybrid(ct) || ct == DM_POLYTOPE_FV_GHOST) continue;
106     PetscCall(DMLabelGetStratumBounds(ctLabel, ct, &ctS, &ctE));
107     if (ctS >= ctE) continue;
108     // Check that a point has the right height
109     PetscCall(DMPlexGetPointHeight(dm, ctS, &ht));
110     if (ht != height) continue;
111     cS    = PetscMin(cS, ctS);
112     cE    = PetscMax(cE, ctE);
113     found = PETSC_TRUE;
114   }
115   if (!Nct || !found) cS = cE = 0;
116   PetscCall(ISDestroy(&valueIS));
117   // Reset label for fast lookup
118   PetscCall(DMLabelMakeAllInvalid_Internal(ctLabel));
119   if (cStart) *cStart = cS;
120   if (cEnd) *cEnd = cE;
121   PetscFunctionReturn(PETSC_SUCCESS);
122 }
123 
124 PetscErrorCode DMPlexGetFieldType_Internal(DM dm, PetscSection section, PetscInt field, PetscInt *sStart, PetscInt *sEnd, PetscViewerVTKFieldType *ft)
125 {
126   PetscInt cdim, pStart, pEnd, vStart, vEnd, cStart, cEnd;
127   PetscInt vcdof[2] = {0, 0}, globalvcdof[2];
128 
129   PetscFunctionBegin;
130   *ft = PETSC_VTK_INVALID;
131   PetscCall(DMGetCoordinateDim(dm, &cdim));
132   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
133   PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd));
134   PetscCall(PetscSectionGetChart(section, &pStart, &pEnd));
135   if (field >= 0) {
136     if ((vStart >= pStart) && (vStart < pEnd)) PetscCall(PetscSectionGetFieldDof(section, vStart, field, &vcdof[0]));
137     if ((cStart >= pStart) && (cStart < pEnd)) PetscCall(PetscSectionGetFieldDof(section, cStart, field, &vcdof[1]));
138   } else {
139     if ((vStart >= pStart) && (vStart < pEnd)) PetscCall(PetscSectionGetDof(section, vStart, &vcdof[0]));
140     if ((cStart >= pStart) && (cStart < pEnd)) PetscCall(PetscSectionGetDof(section, cStart, &vcdof[1]));
141   }
142   PetscCallMPI(MPIU_Allreduce(vcdof, globalvcdof, 2, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject)dm)));
143   if (globalvcdof[0]) {
144     *sStart = vStart;
145     *sEnd   = vEnd;
146     if (globalvcdof[0] == cdim) *ft = PETSC_VTK_POINT_VECTOR_FIELD;
147     else *ft = PETSC_VTK_POINT_FIELD;
148   } else if (globalvcdof[1]) {
149     *sStart = cStart;
150     *sEnd   = cEnd;
151     if (globalvcdof[1] == cdim) *ft = PETSC_VTK_CELL_VECTOR_FIELD;
152     else *ft = PETSC_VTK_CELL_FIELD;
153   } else {
154     if (field >= 0) {
155       const char *fieldname;
156 
157       PetscCall(PetscSectionGetFieldName(section, field, &fieldname));
158       PetscCall(PetscInfo((PetscObject)dm, "Could not classify VTK output type of section field %" PetscInt_FMT " \"%s\"\n", field, fieldname));
159     } else {
160       PetscCall(PetscInfo((PetscObject)dm, "Could not classify VTK output type of section\n"));
161     }
162   }
163   PetscFunctionReturn(PETSC_SUCCESS);
164 }
165 
166 /*@
167   DMPlexVecView1D - Plot many 1D solutions on the same line graph
168 
169   Collective
170 
171   Input Parameters:
172 + dm     - The `DMPLEX` object
173 . n      - The number of vectors
174 . u      - The array of local vectors
175 - viewer - The `PetscViewer`
176 
177   Level: advanced
178 
179 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `VecViewFromOptions()`, `VecView()`
180 @*/
181 PetscErrorCode DMPlexVecView1D(DM dm, PetscInt n, Vec u[], PetscViewer viewer)
182 {
183   PetscDS            ds;
184   PetscDraw          draw = NULL;
185   PetscDrawLG        lg;
186   Vec                coordinates;
187   const PetscScalar *coords, **sol;
188   PetscReal         *vals;
189   PetscInt          *Nc;
190   PetscInt           Nf, f, c, Nl, l, i, vStart, vEnd, v;
191   char             **names;
192 
193   PetscFunctionBegin;
194   PetscCall(DMGetDS(dm, &ds));
195   PetscCall(PetscDSGetNumFields(ds, &Nf));
196   PetscCall(PetscDSGetTotalComponents(ds, &Nl));
197   PetscCall(PetscDSGetComponents(ds, &Nc));
198 
199   PetscCall(PetscViewerDrawGetDraw(viewer, 0, &draw));
200   if (!draw) PetscFunctionReturn(PETSC_SUCCESS);
201   PetscCall(PetscDrawLGCreate(draw, n * Nl, &lg));
202 
203   PetscCall(PetscMalloc3(n, &sol, n * Nl, &names, n * Nl, &vals));
204   for (i = 0, l = 0; i < n; ++i) {
205     const char *vname;
206 
207     PetscCall(PetscObjectGetName((PetscObject)u[i], &vname));
208     for (f = 0; f < Nf; ++f) {
209       PetscObject disc;
210       const char *fname;
211       char        tmpname[PETSC_MAX_PATH_LEN];
212 
213       PetscCall(PetscDSGetDiscretization(ds, f, &disc));
214       /* TODO Create names for components */
215       for (c = 0; c < Nc[f]; ++c, ++l) {
216         PetscCall(PetscObjectGetName(disc, &fname));
217         PetscCall(PetscStrncpy(tmpname, vname, sizeof(tmpname)));
218         PetscCall(PetscStrlcat(tmpname, ":", sizeof(tmpname)));
219         PetscCall(PetscStrlcat(tmpname, fname, sizeof(tmpname)));
220         PetscCall(PetscStrallocpy(tmpname, &names[l]));
221       }
222     }
223   }
224   PetscCall(PetscDrawLGSetLegend(lg, (const char *const *)names));
225   /* Just add P_1 support for now */
226   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
227   PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
228   PetscCall(VecGetArrayRead(coordinates, &coords));
229   for (i = 0; i < n; ++i) PetscCall(VecGetArrayRead(u[i], &sol[i]));
230   for (v = vStart; v < vEnd; ++v) {
231     PetscScalar *x, *svals;
232 
233     PetscCall(DMPlexPointLocalRead(dm, v, coords, &x));
234     for (i = 0; i < n; ++i) {
235       PetscCall(DMPlexPointLocalRead(dm, v, sol[i], &svals));
236       for (l = 0; l < Nl; ++l) vals[i * Nl + l] = PetscRealPart(svals[l]);
237     }
238     PetscCall(PetscDrawLGAddCommonPoint(lg, PetscRealPart(x[0]), vals));
239   }
240   PetscCall(VecRestoreArrayRead(coordinates, &coords));
241   for (i = 0; i < n; ++i) PetscCall(VecRestoreArrayRead(u[i], &sol[i]));
242   for (l = 0; l < n * Nl; ++l) PetscCall(PetscFree(names[l]));
243   PetscCall(PetscFree3(sol, names, vals));
244 
245   PetscCall(PetscDrawLGDraw(lg));
246   PetscCall(PetscDrawLGDestroy(&lg));
247   PetscFunctionReturn(PETSC_SUCCESS);
248 }
249 
250 static PetscErrorCode VecView_Plex_Local_Draw_1D(Vec u, PetscViewer viewer)
251 {
252   DM dm;
253 
254   PetscFunctionBegin;
255   PetscCall(VecGetDM(u, &dm));
256   PetscCall(DMPlexVecView1D(dm, 1, &u, viewer));
257   PetscFunctionReturn(PETSC_SUCCESS);
258 }
259 
260 static PetscErrorCode VecView_Plex_Local_Draw_2D(Vec v, PetscViewer viewer)
261 {
262   DM                 dm;
263   PetscSection       s;
264   PetscDraw          draw, popup;
265   DM                 cdm;
266   PetscSection       coordSection;
267   Vec                coordinates;
268   const PetscScalar *array;
269   PetscReal          lbound[3], ubound[3];
270   PetscReal          vbound[2], time;
271   PetscBool          flg;
272   PetscInt           dim, Nf, f, Nc, comp, vStart, vEnd, cStart, cEnd, c, N, level, step, w = 0;
273   const char        *name;
274   char               title[PETSC_MAX_PATH_LEN];
275 
276   PetscFunctionBegin;
277   PetscCall(PetscViewerDrawGetDraw(viewer, 0, &draw));
278   PetscCall(VecGetDM(v, &dm));
279   PetscCall(DMGetCoordinateDim(dm, &dim));
280   PetscCall(DMGetLocalSection(dm, &s));
281   PetscCall(PetscSectionGetNumFields(s, &Nf));
282   PetscCall(DMGetCoarsenLevel(dm, &level));
283   PetscCall(DMGetCoordinateDM(dm, &cdm));
284   PetscCall(DMGetLocalSection(cdm, &coordSection));
285   PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
286   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
287   PetscCall(DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd));
288 
289   PetscCall(PetscObjectGetName((PetscObject)v, &name));
290   PetscCall(DMGetOutputSequenceNumber(dm, &step, &time));
291 
292   PetscCall(VecGetLocalSize(coordinates, &N));
293   PetscCall(DMGetBoundingBox(dm, lbound, ubound));
294   PetscCall(PetscDrawClear(draw));
295 
296   /* Could implement something like DMDASelectFields() */
297   for (f = 0; f < Nf; ++f) {
298     DM          fdm = dm;
299     Vec         fv  = v;
300     IS          fis;
301     char        prefix[PETSC_MAX_PATH_LEN];
302     const char *fname;
303 
304     PetscCall(PetscSectionGetFieldComponents(s, f, &Nc));
305     PetscCall(PetscSectionGetFieldName(s, f, &fname));
306 
307     if (v->hdr.prefix) PetscCall(PetscStrncpy(prefix, v->hdr.prefix, sizeof(prefix)));
308     else prefix[0] = '\0';
309     if (Nf > 1) {
310       PetscCall(DMCreateSubDM(dm, 1, &f, &fis, &fdm));
311       PetscCall(VecGetSubVector(v, fis, &fv));
312       PetscCall(PetscStrlcat(prefix, fname, sizeof(prefix)));
313       PetscCall(PetscStrlcat(prefix, "_", sizeof(prefix)));
314     }
315     for (comp = 0; comp < Nc; ++comp, ++w) {
316       PetscInt nmax = 2;
317 
318       PetscCall(PetscViewerDrawGetDraw(viewer, w, &draw));
319       if (Nc > 1) PetscCall(PetscSNPrintf(title, sizeof(title), "%s:%s_%" PetscInt_FMT " Step: %" PetscInt_FMT " Time: %.4g", name, fname, comp, step, (double)time));
320       else PetscCall(PetscSNPrintf(title, sizeof(title), "%s:%s Step: %" PetscInt_FMT " Time: %.4g", name, fname, step, (double)time));
321       PetscCall(PetscDrawSetTitle(draw, title));
322 
323       /* TODO Get max and min only for this component */
324       PetscCall(PetscOptionsGetRealArray(NULL, prefix, "-vec_view_bounds", vbound, &nmax, &flg));
325       if (!flg) {
326         PetscCall(VecMin(fv, NULL, &vbound[0]));
327         PetscCall(VecMax(fv, NULL, &vbound[1]));
328         if (vbound[1] <= vbound[0]) vbound[1] = vbound[0] + 1.0;
329       }
330 
331       PetscCall(PetscDrawGetPopup(draw, &popup));
332       PetscCall(PetscDrawScalePopup(popup, vbound[0], vbound[1]));
333       PetscCall(PetscDrawSetCoordinates(draw, lbound[0], lbound[1], ubound[0], ubound[1]));
334       PetscCall(VecGetArrayRead(fv, &array));
335       for (c = cStart; c < cEnd; ++c) {
336         DMPolytopeType     ct;
337         PetscScalar       *coords = NULL, *a = NULL;
338         const PetscScalar *coords_arr;
339         PetscBool          isDG;
340         PetscInt           numCoords;
341         int                color[4] = {-1, -1, -1, -1};
342 
343         PetscCall(DMPlexGetCellType(dm, c, &ct));
344         PetscCall(DMPlexPointLocalRead(fdm, c, array, &a));
345         if (a) {
346           color[0] = PetscDrawRealToColor(PetscRealPart(a[comp]), vbound[0], vbound[1]);
347           color[1] = color[2] = color[3] = color[0];
348         } else {
349           PetscScalar *vals = NULL;
350           PetscInt     numVals, va;
351 
352           PetscCall(DMPlexVecGetClosure(fdm, NULL, fv, c, &numVals, &vals));
353           if (!numVals) {
354             PetscCall(DMPlexVecRestoreClosure(fdm, NULL, fv, c, &numVals, &vals));
355             continue;
356           }
357           PetscCheck(numVals % Nc == 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "The number of components %" PetscInt_FMT " does not divide the number of values in the closure %" PetscInt_FMT, Nc, numVals);
358           switch (numVals / Nc) {
359           case 1: /* P1 Clamped Segment Prism */
360           case 2: /* P1 Segment Prism, P2 Clamped Segment Prism */
361             PetscCheck(ct == DM_POLYTOPE_SEG_PRISM_TENSOR, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell should be a tensor segment, but it is a %s", DMPolytopeTypes[ct]);
362             for (va = 0; va < numVals / Nc; ++va) color[va] = PetscDrawRealToColor(PetscRealPart(vals[va * Nc + comp]), vbound[0], vbound[1]);
363             break;
364           case 3: /* P1 Triangle */
365           case 4: /* P1 Quadrangle */
366             PetscCheck(ct == DM_POLYTOPE_TRIANGLE || ct == DM_POLYTOPE_QUADRILATERAL || ct == DM_POLYTOPE_SEG_PRISM_TENSOR, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell should be a triangle or quad, but it is a %s", DMPolytopeTypes[ct]);
367             for (va = 0; va < numVals / Nc; ++va) color[va] = PetscDrawRealToColor(PetscRealPart(vals[va * Nc + comp]), vbound[0], vbound[1]);
368             break;
369           case 6: /* P2 Triangle */
370           case 8: /* P2 Quadrangle */
371             PetscCheck(ct == DM_POLYTOPE_TRIANGLE || ct == DM_POLYTOPE_QUADRILATERAL || ct == DM_POLYTOPE_SEG_PRISM_TENSOR, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell should be a triangle or quad, but it is a %s", DMPolytopeTypes[ct]);
372             for (va = 0; va < numVals / (Nc * 2); ++va) color[va] = PetscDrawRealToColor(PetscRealPart(vals[va * Nc + comp + numVals / (Nc * 2)]), vbound[0], vbound[1]);
373             break;
374           default:
375             SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of values for cell closure %" PetscInt_FMT " cannot be handled", numVals / Nc);
376           }
377           PetscCall(DMPlexVecRestoreClosure(fdm, NULL, fv, c, &numVals, &vals));
378         }
379         PetscCall(DMPlexGetCellCoordinates(dm, c, &isDG, &numCoords, &coords_arr, &coords));
380         switch (numCoords) {
381         case 6:
382         case 12: /* Localized triangle */
383           PetscCall(PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), color[0], color[1], color[2]));
384           break;
385         case 8:
386         case 16: /* Localized quadrilateral */
387           if (ct == DM_POLYTOPE_SEG_PRISM_TENSOR) {
388             PetscCall(PetscDrawLine(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscMax(color[0], color[1])));
389           } else {
390             PetscCall(PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), color[0], color[1], color[2]));
391             PetscCall(PetscDrawTriangle(draw, PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[6]), PetscRealPart(coords[7]), PetscRealPart(coords[0]), PetscRealPart(coords[1]), color[2], color[3], color[0]));
392           }
393           break;
394         default:
395           SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot draw cells with %" PetscInt_FMT " coordinates", numCoords);
396         }
397         PetscCall(DMPlexRestoreCellCoordinates(dm, c, &isDG, &numCoords, &coords_arr, &coords));
398       }
399       PetscCall(VecRestoreArrayRead(fv, &array));
400       PetscCall(PetscDrawFlush(draw));
401       PetscCall(PetscDrawPause(draw));
402       PetscCall(PetscDrawSave(draw));
403     }
404     if (Nf > 1) {
405       PetscCall(VecRestoreSubVector(v, fis, &fv));
406       PetscCall(ISDestroy(&fis));
407       PetscCall(DMDestroy(&fdm));
408     }
409   }
410   PetscFunctionReturn(PETSC_SUCCESS);
411 }
412 
413 static PetscErrorCode VecView_Plex_Local_Draw(Vec v, PetscViewer viewer)
414 {
415   DM        dm;
416   PetscDraw draw;
417   PetscInt  dim;
418   PetscBool isnull;
419 
420   PetscFunctionBegin;
421   PetscCall(PetscViewerDrawGetDraw(viewer, 0, &draw));
422   PetscCall(PetscDrawIsNull(draw, &isnull));
423   if (isnull) PetscFunctionReturn(PETSC_SUCCESS);
424 
425   PetscCall(VecGetDM(v, &dm));
426   PetscCall(DMGetCoordinateDim(dm, &dim));
427   switch (dim) {
428   case 1:
429     PetscCall(VecView_Plex_Local_Draw_1D(v, viewer));
430     break;
431   case 2:
432     PetscCall(VecView_Plex_Local_Draw_2D(v, viewer));
433     break;
434   default:
435     SETERRQ(PetscObjectComm((PetscObject)v), PETSC_ERR_SUP, "Cannot draw meshes of dimension %" PetscInt_FMT ". Try PETSCVIEWERGLVIS", dim);
436   }
437   PetscFunctionReturn(PETSC_SUCCESS);
438 }
439 
440 static PetscErrorCode VecView_Plex_Local_VTK(Vec v, PetscViewer viewer)
441 {
442   DM                      dm;
443   Vec                     locv;
444   const char             *name;
445   PetscSection            section;
446   PetscInt                pStart, pEnd;
447   PetscInt                numFields;
448   PetscViewerVTKFieldType ft;
449 
450   PetscFunctionBegin;
451   PetscCall(VecGetDM(v, &dm));
452   PetscCall(DMCreateLocalVector(dm, &locv)); /* VTK viewer requires exclusive ownership of the vector */
453   PetscCall(PetscObjectGetName((PetscObject)v, &name));
454   PetscCall(PetscObjectSetName((PetscObject)locv, name));
455   PetscCall(VecCopy(v, locv));
456   PetscCall(DMGetLocalSection(dm, &section));
457   PetscCall(PetscSectionGetNumFields(section, &numFields));
458   if (!numFields) {
459     PetscCall(DMPlexGetFieldType_Internal(dm, section, PETSC_DETERMINE, &pStart, &pEnd, &ft));
460     PetscCall(PetscViewerVTKAddField(viewer, (PetscObject)dm, DMPlexVTKWriteAll, PETSC_DEFAULT, ft, PETSC_TRUE, (PetscObject)locv));
461   } else {
462     PetscInt f;
463 
464     for (f = 0; f < numFields; f++) {
465       PetscCall(DMPlexGetFieldType_Internal(dm, section, f, &pStart, &pEnd, &ft));
466       if (ft == PETSC_VTK_INVALID) continue;
467       PetscCall(PetscObjectReference((PetscObject)locv));
468       PetscCall(PetscViewerVTKAddField(viewer, (PetscObject)dm, DMPlexVTKWriteAll, f, ft, PETSC_TRUE, (PetscObject)locv));
469     }
470     PetscCall(VecDestroy(&locv));
471   }
472   PetscFunctionReturn(PETSC_SUCCESS);
473 }
474 
475 PetscErrorCode VecView_Plex_Local(Vec v, PetscViewer viewer)
476 {
477   DM        dm;
478   PetscBool isvtk, ishdf5, isdraw, isglvis, iscgns;
479 
480   PetscFunctionBegin;
481   PetscCall(VecGetDM(v, &dm));
482   PetscCheck(dm, PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
483   PetscCall(PetscObjectTypeCompare((PetscObject)viewer, PETSCVIEWERVTK, &isvtk));
484   PetscCall(PetscObjectTypeCompare((PetscObject)viewer, PETSCVIEWERHDF5, &ishdf5));
485   PetscCall(PetscObjectTypeCompare((PetscObject)viewer, PETSCVIEWERDRAW, &isdraw));
486   PetscCall(PetscObjectTypeCompare((PetscObject)viewer, PETSCVIEWERGLVIS, &isglvis));
487   PetscCall(PetscObjectTypeCompare((PetscObject)viewer, PETSCVIEWERCGNS, &iscgns));
488   if (isvtk || ishdf5 || isdraw || isglvis || iscgns) {
489     PetscInt    i, numFields;
490     PetscObject fe;
491     PetscBool   fem  = PETSC_FALSE;
492     Vec         locv = v;
493     const char *name;
494     PetscInt    step;
495     PetscReal   time;
496 
497     PetscCall(DMGetNumFields(dm, &numFields));
498     for (i = 0; i < numFields; i++) {
499       PetscCall(DMGetField(dm, i, NULL, &fe));
500       if (fe->classid == PETSCFE_CLASSID) {
501         fem = PETSC_TRUE;
502         break;
503       }
504     }
505     if (fem) {
506       PetscObject isZero;
507 
508       PetscCall(DMGetLocalVector(dm, &locv));
509       PetscCall(PetscObjectGetName((PetscObject)v, &name));
510       PetscCall(PetscObjectSetName((PetscObject)locv, name));
511       PetscCall(PetscObjectQuery((PetscObject)v, "__Vec_bc_zero__", &isZero));
512       PetscCall(PetscObjectCompose((PetscObject)locv, "__Vec_bc_zero__", isZero));
513       PetscCall(VecCopy(v, locv));
514       PetscCall(DMGetOutputSequenceNumber(dm, NULL, &time));
515       PetscCall(DMPlexInsertBoundaryValues(dm, PETSC_TRUE, locv, time, NULL, NULL, NULL));
516     }
517     if (isvtk) {
518       PetscCall(VecView_Plex_Local_VTK(locv, viewer));
519     } else if (ishdf5) {
520 #if defined(PETSC_HAVE_HDF5)
521       PetscCall(VecView_Plex_Local_HDF5_Internal(locv, viewer));
522 #else
523       SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
524 #endif
525     } else if (isdraw) {
526       PetscCall(VecView_Plex_Local_Draw(locv, viewer));
527     } else if (isglvis) {
528       PetscCall(DMGetOutputSequenceNumber(dm, &step, NULL));
529       PetscCall(PetscViewerGLVisSetSnapId(viewer, step));
530       PetscCall(VecView_GLVis(locv, viewer));
531     } else if (iscgns) {
532 #if defined(PETSC_HAVE_CGNS)
533       PetscCall(VecView_Plex_Local_CGNS(locv, viewer));
534 #else
535       SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "CGNS not supported in this build.\nPlease reconfigure using --download-cgns");
536 #endif
537     }
538     if (fem) {
539       PetscCall(PetscObjectCompose((PetscObject)locv, "__Vec_bc_zero__", NULL));
540       PetscCall(DMRestoreLocalVector(dm, &locv));
541     }
542   } else {
543     PetscBool isseq;
544 
545     PetscCall(PetscObjectTypeCompare((PetscObject)v, VECSEQ, &isseq));
546     if (isseq) PetscCall(VecView_Seq(v, viewer));
547     else PetscCall(VecView_MPI(v, viewer));
548   }
549   PetscFunctionReturn(PETSC_SUCCESS);
550 }
551 
552 PetscErrorCode VecView_Plex(Vec v, PetscViewer viewer)
553 {
554   DM        dm;
555   PetscBool isvtk, ishdf5, isdraw, isglvis, isexodusii, iscgns;
556 
557   PetscFunctionBegin;
558   PetscCall(VecGetDM(v, &dm));
559   PetscCheck(dm, PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
560   PetscCall(PetscObjectTypeCompare((PetscObject)viewer, PETSCVIEWERVTK, &isvtk));
561   PetscCall(PetscObjectTypeCompare((PetscObject)viewer, PETSCVIEWERHDF5, &ishdf5));
562   PetscCall(PetscObjectTypeCompare((PetscObject)viewer, PETSCVIEWERDRAW, &isdraw));
563   PetscCall(PetscObjectTypeCompare((PetscObject)viewer, PETSCVIEWERGLVIS, &isglvis));
564   PetscCall(PetscObjectTypeCompare((PetscObject)viewer, PETSCVIEWERCGNS, &iscgns));
565   PetscCall(PetscObjectTypeCompare((PetscObject)viewer, PETSCVIEWEREXODUSII, &isexodusii));
566   if (isvtk || isdraw || isglvis || iscgns) {
567     Vec         locv;
568     PetscObject isZero;
569     const char *name;
570 
571     PetscCall(DMGetLocalVector(dm, &locv));
572     PetscCall(PetscObjectGetName((PetscObject)v, &name));
573     PetscCall(PetscObjectSetName((PetscObject)locv, name));
574     PetscCall(DMGlobalToLocalBegin(dm, v, INSERT_VALUES, locv));
575     PetscCall(DMGlobalToLocalEnd(dm, v, INSERT_VALUES, locv));
576     PetscCall(PetscObjectQuery((PetscObject)v, "__Vec_bc_zero__", &isZero));
577     PetscCall(PetscObjectCompose((PetscObject)locv, "__Vec_bc_zero__", isZero));
578     PetscCall(VecView_Plex_Local(locv, viewer));
579     PetscCall(PetscObjectCompose((PetscObject)locv, "__Vec_bc_zero__", NULL));
580     PetscCall(DMRestoreLocalVector(dm, &locv));
581     /* Call flush for proper logging of VecView timings */
582     if (isvtk) PetscCall(PetscViewerFlush(viewer));
583   } else if (ishdf5) {
584 #if defined(PETSC_HAVE_HDF5)
585     PetscCall(VecView_Plex_HDF5_Internal(v, viewer));
586 #else
587     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
588 #endif
589   } else if (isexodusii) {
590 #if defined(PETSC_HAVE_EXODUSII)
591     PetscCall(VecView_PlexExodusII_Internal(v, viewer));
592 #else
593     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "ExodusII not supported in this build.\nPlease reconfigure using --download-exodusii");
594 #endif
595   } else {
596     PetscBool isseq;
597 
598     PetscCall(PetscObjectTypeCompare((PetscObject)v, VECSEQ, &isseq));
599     if (isseq) PetscCall(VecView_Seq(v, viewer));
600     else PetscCall(VecView_MPI(v, viewer));
601   }
602   PetscFunctionReturn(PETSC_SUCCESS);
603 }
604 
605 PetscErrorCode VecView_Plex_Native(Vec originalv, PetscViewer viewer)
606 {
607   DM                dm;
608   MPI_Comm          comm;
609   PetscViewerFormat format;
610   Vec               v;
611   PetscBool         isvtk, ishdf5;
612 
613   PetscFunctionBegin;
614   PetscCall(VecGetDM(originalv, &dm));
615   PetscCall(PetscObjectGetComm((PetscObject)originalv, &comm));
616   PetscCheck(dm, comm, PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
617   PetscCall(PetscViewerGetFormat(viewer, &format));
618   PetscCall(PetscObjectTypeCompare((PetscObject)viewer, PETSCVIEWERHDF5, &ishdf5));
619   PetscCall(PetscObjectTypeCompare((PetscObject)viewer, PETSCVIEWERVTK, &isvtk));
620   if (format == PETSC_VIEWER_NATIVE) {
621     /* Natural ordering is the common case for DMDA, NATIVE means plain vector, for PLEX is the opposite */
622     /* this need a better fix */
623     if (dm->useNatural) {
624       if (dm->sfNatural) {
625         const char *vecname;
626         PetscInt    n, nroots;
627 
628         PetscCall(VecGetLocalSize(originalv, &n));
629         PetscCall(PetscSFGetGraph(dm->sfNatural, &nroots, NULL, NULL, NULL));
630         if (n == nroots) {
631           PetscCall(DMPlexCreateNaturalVector(dm, &v));
632           PetscCall(DMPlexGlobalToNaturalBegin(dm, originalv, v));
633           PetscCall(DMPlexGlobalToNaturalEnd(dm, originalv, v));
634           PetscCall(PetscObjectGetName((PetscObject)originalv, &vecname));
635           PetscCall(PetscObjectSetName((PetscObject)v, vecname));
636         } else SETERRQ(comm, PETSC_ERR_ARG_WRONG, "DM global to natural SF only handles global vectors");
637       } else SETERRQ(comm, PETSC_ERR_ARG_WRONGSTATE, "DM global to natural SF was not created");
638     } else v = originalv;
639   } else v = originalv;
640 
641   if (ishdf5) {
642 #if defined(PETSC_HAVE_HDF5)
643     PetscCall(VecView_Plex_HDF5_Native_Internal(v, viewer));
644 #else
645     SETERRQ(comm, PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
646 #endif
647   } else if (isvtk) {
648     SETERRQ(comm, PETSC_ERR_SUP, "VTK format does not support viewing in natural order. Please switch to HDF5.");
649   } else {
650     PetscBool isseq;
651 
652     PetscCall(PetscObjectTypeCompare((PetscObject)v, VECSEQ, &isseq));
653     if (isseq) PetscCall(VecView_Seq(v, viewer));
654     else PetscCall(VecView_MPI(v, viewer));
655   }
656   if (v != originalv) PetscCall(VecDestroy(&v));
657   PetscFunctionReturn(PETSC_SUCCESS);
658 }
659 
660 PetscErrorCode VecLoad_Plex_Local(Vec v, PetscViewer viewer)
661 {
662   DM        dm;
663   PetscBool ishdf5;
664 
665   PetscFunctionBegin;
666   PetscCall(VecGetDM(v, &dm));
667   PetscCheck(dm, PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
668   PetscCall(PetscObjectTypeCompare((PetscObject)viewer, PETSCVIEWERHDF5, &ishdf5));
669   if (ishdf5) {
670     DM          dmBC;
671     Vec         gv;
672     const char *name;
673 
674     PetscCall(DMGetOutputDM(dm, &dmBC));
675     PetscCall(DMGetGlobalVector(dmBC, &gv));
676     PetscCall(PetscObjectGetName((PetscObject)v, &name));
677     PetscCall(PetscObjectSetName((PetscObject)gv, name));
678     PetscCall(VecLoad_Default(gv, viewer));
679     PetscCall(DMGlobalToLocalBegin(dmBC, gv, INSERT_VALUES, v));
680     PetscCall(DMGlobalToLocalEnd(dmBC, gv, INSERT_VALUES, v));
681     PetscCall(DMRestoreGlobalVector(dmBC, &gv));
682   } else PetscCall(VecLoad_Default(v, viewer));
683   PetscFunctionReturn(PETSC_SUCCESS);
684 }
685 
686 PetscErrorCode VecLoad_Plex(Vec v, PetscViewer viewer)
687 {
688   DM        dm;
689   PetscBool ishdf5, isexodusii, iscgns;
690 
691   PetscFunctionBegin;
692   PetscCall(VecGetDM(v, &dm));
693   PetscCheck(dm, PetscObjectComm((PetscObject)v), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
694   PetscCall(PetscObjectTypeCompare((PetscObject)viewer, PETSCVIEWERHDF5, &ishdf5));
695   PetscCall(PetscObjectTypeCompare((PetscObject)viewer, PETSCVIEWEREXODUSII, &isexodusii));
696   PetscCall(PetscObjectTypeCompare((PetscObject)viewer, PETSCVIEWERCGNS, &iscgns));
697   if (ishdf5) {
698 #if defined(PETSC_HAVE_HDF5)
699     PetscCall(VecLoad_Plex_HDF5_Internal(v, viewer));
700 #else
701     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
702 #endif
703   } else if (isexodusii) {
704 #if defined(PETSC_HAVE_EXODUSII)
705     PetscCall(VecLoad_PlexExodusII_Internal(v, viewer));
706 #else
707     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "ExodusII not supported in this build.\nPlease reconfigure using --download-exodusii");
708 #endif
709   } else if (iscgns) {
710 #if defined(PETSC_HAVE_CGNS)
711     PetscCall(VecLoad_Plex_CGNS_Internal(v, viewer));
712 #else
713     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "CGNS not supported in this build.\nPlease reconfigure using --download-cgns");
714 #endif
715   } else PetscCall(VecLoad_Default(v, viewer));
716   PetscFunctionReturn(PETSC_SUCCESS);
717 }
718 
719 PetscErrorCode VecLoad_Plex_Native(Vec originalv, PetscViewer viewer)
720 {
721   DM                dm;
722   PetscViewerFormat format;
723   PetscBool         ishdf5;
724 
725   PetscFunctionBegin;
726   PetscCall(VecGetDM(originalv, &dm));
727   PetscCheck(dm, PetscObjectComm((PetscObject)originalv), PETSC_ERR_ARG_WRONG, "Vector not generated from a DM");
728   PetscCall(PetscViewerGetFormat(viewer, &format));
729   PetscCall(PetscObjectTypeCompare((PetscObject)viewer, PETSCVIEWERHDF5, &ishdf5));
730   if (format == PETSC_VIEWER_NATIVE) {
731     if (dm->useNatural) {
732       if (dm->sfNatural) {
733         if (ishdf5) {
734 #if defined(PETSC_HAVE_HDF5)
735           Vec         v;
736           const char *vecname;
737 
738           PetscCall(DMPlexCreateNaturalVector(dm, &v));
739           PetscCall(PetscObjectGetName((PetscObject)originalv, &vecname));
740           PetscCall(PetscObjectSetName((PetscObject)v, vecname));
741           PetscCall(VecLoad_Plex_HDF5_Native_Internal(v, viewer));
742           PetscCall(DMPlexNaturalToGlobalBegin(dm, v, originalv));
743           PetscCall(DMPlexNaturalToGlobalEnd(dm, v, originalv));
744           PetscCall(VecDestroy(&v));
745 #else
746           SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
747 #endif
748         } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Reading in natural order is not supported for anything but HDF5.");
749       }
750     } else PetscCall(VecLoad_Default(originalv, viewer));
751   }
752   PetscFunctionReturn(PETSC_SUCCESS);
753 }
754 
755 PETSC_UNUSED static PetscErrorCode DMPlexView_Ascii_Geometry(DM dm, PetscViewer viewer)
756 {
757   PetscSection       coordSection;
758   Vec                coordinates;
759   DMLabel            depthLabel, celltypeLabel;
760   const char        *name[4];
761   const PetscScalar *a;
762   PetscInt           dim, pStart, pEnd, cStart, cEnd, c;
763 
764   PetscFunctionBegin;
765   PetscCall(DMGetDimension(dm, &dim));
766   PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
767   PetscCall(DMGetCoordinateSection(dm, &coordSection));
768   PetscCall(DMPlexGetDepthLabel(dm, &depthLabel));
769   PetscCall(DMPlexGetCellTypeLabel(dm, &celltypeLabel));
770   PetscCall(DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd));
771   PetscCall(PetscSectionGetChart(coordSection, &pStart, &pEnd));
772   PetscCall(VecGetArrayRead(coordinates, &a));
773   name[0]       = "vertex";
774   name[1]       = "edge";
775   name[dim - 1] = "face";
776   name[dim]     = "cell";
777   for (c = cStart; c < cEnd; ++c) {
778     PetscInt *closure = NULL;
779     PetscInt  closureSize, cl, ct;
780 
781     PetscCall(DMLabelGetValue(celltypeLabel, c, &ct));
782     PetscCall(PetscViewerASCIIPrintf(viewer, "Geometry for cell %" PetscInt_FMT " polytope type %s:\n", c, DMPolytopeTypes[ct]));
783     PetscCall(DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure));
784     PetscCall(PetscViewerASCIIPushTab(viewer));
785     for (cl = 0; cl < closureSize * 2; cl += 2) {
786       PetscInt point = closure[cl], depth, dof, off, d, p;
787 
788       if ((point < pStart) || (point >= pEnd)) continue;
789       PetscCall(PetscSectionGetDof(coordSection, point, &dof));
790       if (!dof) continue;
791       PetscCall(DMLabelGetValue(depthLabel, point, &depth));
792       PetscCall(PetscSectionGetOffset(coordSection, point, &off));
793       PetscCall(PetscViewerASCIIPrintf(viewer, "%s %" PetscInt_FMT " coords:", name[depth], point));
794       for (p = 0; p < dof / dim; ++p) {
795         PetscCall(PetscViewerASCIIPrintf(viewer, " ("));
796         for (d = 0; d < dim; ++d) {
797           if (d > 0) PetscCall(PetscViewerASCIIPrintf(viewer, ", "));
798           PetscCall(PetscViewerASCIIPrintf(viewer, "%g", (double)PetscRealPart(a[off + p * dim + d])));
799         }
800         PetscCall(PetscViewerASCIIPrintf(viewer, ")"));
801       }
802       PetscCall(PetscViewerASCIIPrintf(viewer, "\n"));
803     }
804     PetscCall(DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure));
805     PetscCall(PetscViewerASCIIPopTab(viewer));
806   }
807   PetscCall(VecRestoreArrayRead(coordinates, &a));
808   PetscFunctionReturn(PETSC_SUCCESS);
809 }
810 
811 typedef enum {
812   CS_CARTESIAN,
813   CS_POLAR,
814   CS_CYLINDRICAL,
815   CS_SPHERICAL
816 } CoordSystem;
817 const char *CoordSystems[] = {"cartesian", "polar", "cylindrical", "spherical", "CoordSystem", "CS_", NULL};
818 
819 static PetscErrorCode DMPlexView_Ascii_Coordinates(PetscViewer viewer, CoordSystem cs, PetscInt dim, const PetscScalar x[])
820 {
821   PetscInt i;
822 
823   PetscFunctionBegin;
824   if (dim > 3) {
825     for (i = 0; i < dim; ++i) PetscCall(PetscViewerASCIISynchronizedPrintf(viewer, " %g", (double)PetscRealPart(x[i])));
826   } else {
827     PetscReal coords[3], trcoords[3] = {0., 0., 0.};
828 
829     for (i = 0; i < dim; ++i) coords[i] = PetscRealPart(x[i]);
830     switch (cs) {
831     case CS_CARTESIAN:
832       for (i = 0; i < dim; ++i) trcoords[i] = coords[i];
833       break;
834     case CS_POLAR:
835       PetscCheck(dim == 2, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Polar coordinates are for 2 dimension, not %" PetscInt_FMT, dim);
836       trcoords[0] = PetscSqrtReal(PetscSqr(coords[0]) + PetscSqr(coords[1]));
837       trcoords[1] = PetscAtan2Real(coords[1], coords[0]);
838       break;
839     case CS_CYLINDRICAL:
840       PetscCheck(dim == 3, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cylindrical coordinates are for 3 dimension, not %" PetscInt_FMT, dim);
841       trcoords[0] = PetscSqrtReal(PetscSqr(coords[0]) + PetscSqr(coords[1]));
842       trcoords[1] = PetscAtan2Real(coords[1], coords[0]);
843       trcoords[2] = coords[2];
844       break;
845     case CS_SPHERICAL:
846       PetscCheck(dim == 3, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Spherical coordinates are for 3 dimension, not %" PetscInt_FMT, dim);
847       trcoords[0] = PetscSqrtReal(PetscSqr(coords[0]) + PetscSqr(coords[1]) + PetscSqr(coords[2]));
848       trcoords[1] = PetscAtan2Real(PetscSqrtReal(PetscSqr(coords[0]) + PetscSqr(coords[1])), coords[2]);
849       trcoords[2] = PetscAtan2Real(coords[1], coords[0]);
850       break;
851     }
852     for (i = 0; i < dim; ++i) PetscCall(PetscViewerASCIISynchronizedPrintf(viewer, " %g", (double)trcoords[i]));
853   }
854   PetscFunctionReturn(PETSC_SUCCESS);
855 }
856 
857 static PetscErrorCode DMPlexView_Ascii(DM dm, PetscViewer viewer)
858 {
859   DM_Plex          *mesh = (DM_Plex *)dm->data;
860   DM                cdm, cdmCell;
861   PetscSection      coordSection, coordSectionCell;
862   Vec               coordinates, coordinatesCell;
863   PetscViewerFormat format;
864 
865   PetscFunctionBegin;
866   PetscCall(PetscViewerGetFormat(viewer, &format));
867   if (format == PETSC_VIEWER_ASCII_INFO_DETAIL) {
868     const char *name;
869     PetscInt    dim, cellHeight, maxConeSize, maxSupportSize;
870     PetscInt    pStart, pEnd, p, numLabels, l;
871     PetscMPIInt rank, size;
872 
873     PetscCall(DMGetCoordinateDM(dm, &cdm));
874     PetscCall(DMGetCoordinateSection(dm, &coordSection));
875     PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
876     PetscCall(DMGetCellCoordinateDM(dm, &cdmCell));
877     PetscCall(DMGetCellCoordinateSection(dm, &coordSectionCell));
878     PetscCall(DMGetCellCoordinatesLocal(dm, &coordinatesCell));
879     PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
880     PetscCallMPI(MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size));
881     PetscCall(PetscObjectGetName((PetscObject)dm, &name));
882     PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
883     PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize));
884     PetscCall(DMGetDimension(dm, &dim));
885     PetscCall(DMPlexGetVTKCellHeight(dm, &cellHeight));
886     if (name) PetscCall(PetscViewerASCIIPrintf(viewer, "%s in %" PetscInt_FMT " dimension%s:\n", name, dim, dim == 1 ? "" : "s"));
887     else PetscCall(PetscViewerASCIIPrintf(viewer, "Mesh in %" PetscInt_FMT " dimension%s:\n", dim, dim == 1 ? "" : "s"));
888     if (cellHeight) PetscCall(PetscViewerASCIIPrintf(viewer, "  Cells are at height %" PetscInt_FMT "\n", cellHeight));
889     PetscCall(PetscViewerASCIIPrintf(viewer, "Supports:\n"));
890     PetscCall(PetscViewerASCIIPushSynchronized(viewer));
891     PetscCall(PetscViewerASCIISynchronizedPrintf(viewer, "[%d] Max support size: %" PetscInt_FMT "\n", rank, maxSupportSize));
892     for (p = pStart; p < pEnd; ++p) {
893       PetscInt dof, off, s;
894 
895       PetscCall(PetscSectionGetDof(mesh->supportSection, p, &dof));
896       PetscCall(PetscSectionGetOffset(mesh->supportSection, p, &off));
897       for (s = off; s < off + dof; ++s) PetscCall(PetscViewerASCIISynchronizedPrintf(viewer, "[%d]: %" PetscInt_FMT " ----> %" PetscInt_FMT "\n", rank, p, mesh->supports[s]));
898     }
899     PetscCall(PetscViewerFlush(viewer));
900     PetscCall(PetscViewerASCIIPrintf(viewer, "Cones:\n"));
901     PetscCall(PetscViewerASCIISynchronizedPrintf(viewer, "[%d] Max cone size: %" PetscInt_FMT "\n", rank, maxConeSize));
902     for (p = pStart; p < pEnd; ++p) {
903       PetscInt dof, off, c;
904 
905       PetscCall(PetscSectionGetDof(mesh->coneSection, p, &dof));
906       PetscCall(PetscSectionGetOffset(mesh->coneSection, p, &off));
907       for (c = off; c < off + dof; ++c) PetscCall(PetscViewerASCIISynchronizedPrintf(viewer, "[%d]: %" PetscInt_FMT " <---- %" PetscInt_FMT " (%" PetscInt_FMT ")\n", rank, p, mesh->cones[c], mesh->coneOrientations[c]));
908     }
909     PetscCall(PetscViewerFlush(viewer));
910     PetscCall(PetscViewerASCIIPopSynchronized(viewer));
911     if (coordSection && coordinates) {
912       CoordSystem        cs = CS_CARTESIAN;
913       const PetscScalar *array, *arrayCell = NULL;
914       PetscInt           Nf, Nc, pvStart, pvEnd, pcStart = PETSC_INT_MAX, pcEnd = PETSC_INT_MIN, pStart, pEnd, p;
915       PetscMPIInt        rank;
916       const char        *name;
917 
918       PetscCall(PetscOptionsGetEnum(((PetscObject)viewer)->options, ((PetscObject)viewer)->prefix, "-dm_plex_view_coord_system", CoordSystems, (PetscEnum *)&cs, NULL));
919       PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)viewer), &rank));
920       PetscCall(PetscSectionGetNumFields(coordSection, &Nf));
921       PetscCheck(Nf == 1, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Coordinate section should have 1 field, not %" PetscInt_FMT, Nf);
922       PetscCall(PetscSectionGetFieldComponents(coordSection, 0, &Nc));
923       PetscCall(PetscSectionGetChart(coordSection, &pvStart, &pvEnd));
924       if (coordSectionCell) PetscCall(PetscSectionGetChart(coordSectionCell, &pcStart, &pcEnd));
925       pStart = PetscMin(pvStart, pcStart);
926       pEnd   = PetscMax(pvEnd, pcEnd);
927       PetscCall(PetscObjectGetName((PetscObject)coordinates, &name));
928       PetscCall(PetscViewerASCIIPrintf(viewer, "%s with %" PetscInt_FMT " fields\n", name, Nf));
929       PetscCall(PetscViewerASCIIPrintf(viewer, "  field 0 with %" PetscInt_FMT " components\n", Nc));
930       if (cs != CS_CARTESIAN) PetscCall(PetscViewerASCIIPrintf(viewer, "  output coordinate system: %s\n", CoordSystems[cs]));
931 
932       PetscCall(VecGetArrayRead(coordinates, &array));
933       if (coordinatesCell) PetscCall(VecGetArrayRead(coordinatesCell, &arrayCell));
934       PetscCall(PetscViewerASCIIPushSynchronized(viewer));
935       PetscCall(PetscViewerASCIISynchronizedPrintf(viewer, "Process %d:\n", rank));
936       for (p = pStart; p < pEnd; ++p) {
937         PetscInt dof, off;
938 
939         if (p >= pvStart && p < pvEnd) {
940           PetscCall(PetscSectionGetDof(coordSection, p, &dof));
941           PetscCall(PetscSectionGetOffset(coordSection, p, &off));
942           if (dof) {
943             PetscCall(PetscViewerASCIISynchronizedPrintf(viewer, "  (%4" PetscInt_FMT ") dof %2" PetscInt_FMT " offset %3" PetscInt_FMT, p, dof, off));
944             PetscCall(DMPlexView_Ascii_Coordinates(viewer, cs, dof, &array[off]));
945             PetscCall(PetscViewerASCIISynchronizedPrintf(viewer, "\n"));
946           }
947         }
948         if (cdmCell && p >= pcStart && p < pcEnd) {
949           PetscCall(PetscSectionGetDof(coordSectionCell, p, &dof));
950           PetscCall(PetscSectionGetOffset(coordSectionCell, p, &off));
951           if (dof) {
952             PetscCall(PetscViewerASCIISynchronizedPrintf(viewer, "  (%4" PetscInt_FMT ") dof %2" PetscInt_FMT " offset %3" PetscInt_FMT, p, dof, off));
953             PetscCall(DMPlexView_Ascii_Coordinates(viewer, cs, dof, &arrayCell[off]));
954             PetscCall(PetscViewerASCIISynchronizedPrintf(viewer, "\n"));
955           }
956         }
957       }
958       PetscCall(PetscViewerFlush(viewer));
959       PetscCall(PetscViewerASCIIPopSynchronized(viewer));
960       PetscCall(VecRestoreArrayRead(coordinates, &array));
961       if (coordinatesCell) PetscCall(VecRestoreArrayRead(coordinatesCell, &arrayCell));
962     }
963     PetscCall(DMGetNumLabels(dm, &numLabels));
964     if (numLabels) PetscCall(PetscViewerASCIIPrintf(viewer, "Labels:\n"));
965     for (l = 0; l < numLabels; ++l) {
966       DMLabel     label;
967       PetscBool   isdepth;
968       const char *name;
969 
970       PetscCall(DMGetLabelName(dm, l, &name));
971       PetscCall(PetscStrcmp(name, "depth", &isdepth));
972       if (isdepth) continue;
973       PetscCall(DMGetLabel(dm, name, &label));
974       PetscCall(DMLabelView(label, viewer));
975     }
976     if (size > 1) {
977       PetscSF sf;
978 
979       PetscCall(DMGetPointSF(dm, &sf));
980       PetscCall(PetscSFView(sf, viewer));
981     }
982     if (mesh->periodic.face_sfs)
983       for (PetscInt i = 0; i < mesh->periodic.num_face_sfs; i++) PetscCall(PetscSFView(mesh->periodic.face_sfs[i], viewer));
984     PetscCall(PetscViewerFlush(viewer));
985   } else if (format == PETSC_VIEWER_ASCII_LATEX) {
986     const char  *name, *color;
987     const char  *defcolors[3]  = {"gray", "orange", "green"};
988     const char  *deflcolors[4] = {"blue", "cyan", "red", "magenta"};
989     char         lname[PETSC_MAX_PATH_LEN];
990     PetscReal    scale      = 2.0;
991     PetscReal    tikzscale  = 1.0;
992     PetscBool    useNumbers = PETSC_TRUE, drawNumbers[4], drawColors[4], useLabels, useColors, plotEdges, drawHasse = PETSC_FALSE;
993     double       tcoords[3];
994     PetscScalar *coords;
995     PetscInt     numLabels, l, numColors, numLColors, dim, d, depth, cStart, cEnd, c, vStart, vEnd, v, eStart = 0, eEnd = 0, fStart = 0, fEnd = 0, e, p, n;
996     PetscMPIInt  rank, size;
997     char       **names, **colors, **lcolors;
998     PetscBool    flg, lflg;
999     PetscBT      wp = NULL;
1000     PetscInt     pEnd, pStart;
1001 
1002     PetscCall(DMGetCoordinateDM(dm, &cdm));
1003     PetscCall(DMGetCoordinateSection(dm, &coordSection));
1004     PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
1005     PetscCall(DMGetCellCoordinateDM(dm, &cdmCell));
1006     PetscCall(DMGetCellCoordinateSection(dm, &coordSectionCell));
1007     PetscCall(DMGetCellCoordinatesLocal(dm, &coordinatesCell));
1008     PetscCall(DMGetDimension(dm, &dim));
1009     PetscCall(DMPlexGetDepth(dm, &depth));
1010     PetscCall(DMGetNumLabels(dm, &numLabels));
1011     numLabels  = PetscMax(numLabels, 10);
1012     numColors  = 10;
1013     numLColors = 10;
1014     PetscCall(PetscCalloc3(numLabels, &names, numColors, &colors, numLColors, &lcolors));
1015     PetscCall(PetscOptionsGetReal(((PetscObject)viewer)->options, ((PetscObject)viewer)->prefix, "-dm_plex_view_scale", &scale, NULL));
1016     PetscCall(PetscOptionsGetReal(((PetscObject)viewer)->options, ((PetscObject)viewer)->prefix, "-dm_plex_view_tikzscale", &tikzscale, NULL));
1017     PetscCall(PetscOptionsGetBool(((PetscObject)viewer)->options, ((PetscObject)viewer)->prefix, "-dm_plex_view_numbers", &useNumbers, NULL));
1018     for (d = 0; d < 4; ++d) drawNumbers[d] = useNumbers;
1019     for (d = 0; d < 4; ++d) drawColors[d] = PETSC_TRUE;
1020     n = 4;
1021     PetscCall(PetscOptionsGetBoolArray(((PetscObject)viewer)->options, ((PetscObject)viewer)->prefix, "-dm_plex_view_numbers_depth", drawNumbers, &n, &flg));
1022     PetscCheck(!flg || n == dim + 1, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_SIZ, "Number of flags %" PetscInt_FMT " != %" PetscInt_FMT " dim+1", n, dim + 1);
1023     n = 4;
1024     PetscCall(PetscOptionsGetBoolArray(((PetscObject)viewer)->options, ((PetscObject)viewer)->prefix, "-dm_plex_view_colors_depth", drawColors, &n, &flg));
1025     PetscCheck(!flg || n == dim + 1, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_SIZ, "Number of flags %" PetscInt_FMT " != %" PetscInt_FMT " dim+1", n, dim + 1);
1026     PetscCall(PetscOptionsGetStringArray(((PetscObject)viewer)->options, ((PetscObject)viewer)->prefix, "-dm_plex_view_labels", names, &numLabels, &useLabels));
1027     if (!useLabels) numLabels = 0;
1028     PetscCall(PetscOptionsGetStringArray(((PetscObject)viewer)->options, ((PetscObject)viewer)->prefix, "-dm_plex_view_colors", colors, &numColors, &useColors));
1029     if (!useColors) {
1030       numColors = 3;
1031       for (c = 0; c < numColors; ++c) PetscCall(PetscStrallocpy(defcolors[c], &colors[c]));
1032     }
1033     PetscCall(PetscOptionsGetStringArray(((PetscObject)viewer)->options, ((PetscObject)viewer)->prefix, "-dm_plex_view_lcolors", lcolors, &numLColors, &useColors));
1034     if (!useColors) {
1035       numLColors = 4;
1036       for (c = 0; c < numLColors; ++c) PetscCall(PetscStrallocpy(deflcolors[c], &lcolors[c]));
1037     }
1038     PetscCall(PetscOptionsGetString(((PetscObject)viewer)->options, ((PetscObject)viewer)->prefix, "-dm_plex_view_label_filter", lname, sizeof(lname), &lflg));
1039     plotEdges = (PetscBool)(depth > 1 && drawNumbers[1] && dim < 3);
1040     PetscCall(PetscOptionsGetBool(((PetscObject)viewer)->options, ((PetscObject)viewer)->prefix, "-dm_plex_view_edges", &plotEdges, &flg));
1041     PetscCheck(!flg || !plotEdges || depth >= dim, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Mesh must be interpolated");
1042     if (depth < dim) plotEdges = PETSC_FALSE;
1043     PetscCall(PetscOptionsGetBool(((PetscObject)viewer)->options, ((PetscObject)viewer)->prefix, "-dm_plex_view_hasse", &drawHasse, NULL));
1044 
1045     /* filter points with labelvalue != labeldefaultvalue */
1046     PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
1047     PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
1048     PetscCall(DMPlexGetDepthStratum(dm, 1, &eStart, &eEnd));
1049     PetscCall(DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd));
1050     PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
1051     if (lflg) {
1052       DMLabel lbl;
1053 
1054       PetscCall(DMGetLabel(dm, lname, &lbl));
1055       if (lbl) {
1056         PetscInt val, defval;
1057 
1058         PetscCall(DMLabelGetDefaultValue(lbl, &defval));
1059         PetscCall(PetscBTCreate(pEnd - pStart, &wp));
1060         for (c = pStart; c < pEnd; c++) {
1061           PetscInt *closure = NULL;
1062           PetscInt  closureSize;
1063 
1064           PetscCall(DMLabelGetValue(lbl, c, &val));
1065           if (val == defval) continue;
1066 
1067           PetscCall(DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure));
1068           for (p = 0; p < closureSize * 2; p += 2) PetscCall(PetscBTSet(wp, closure[p] - pStart));
1069           PetscCall(DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure));
1070         }
1071       }
1072     }
1073 
1074     PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
1075     PetscCallMPI(MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size));
1076     PetscCall(PetscObjectGetName((PetscObject)dm, &name));
1077     PetscCall(PetscViewerASCIIPrintf(viewer, "\
1078 \\documentclass[tikz]{standalone}\n\n\
1079 \\usepackage{pgflibraryshapes}\n\
1080 \\usetikzlibrary{backgrounds}\n\
1081 \\usetikzlibrary{arrows}\n\
1082 \\begin{document}\n"));
1083     if (size > 1) {
1084       PetscCall(PetscViewerASCIIPrintf(viewer, "%s for process ", name));
1085       for (p = 0; p < size; ++p) {
1086         if (p) PetscCall(PetscViewerASCIIPrintf(viewer, (p == size - 1) ? ", and " : ", "));
1087         PetscCall(PetscViewerASCIIPrintf(viewer, "{\\textcolor{%s}%" PetscInt_FMT "}", colors[p % numColors], p));
1088       }
1089       PetscCall(PetscViewerASCIIPrintf(viewer, ".\n\n\n"));
1090     }
1091     if (drawHasse) {
1092       PetscInt maxStratum = PetscMax(vEnd - vStart, PetscMax(eEnd - eStart, PetscMax(fEnd - fStart, cEnd - cStart)));
1093 
1094       PetscCall(PetscViewerASCIIPrintf(viewer, "\\newcommand{\\vStart}{%" PetscInt_FMT "}\n", vStart));
1095       PetscCall(PetscViewerASCIIPrintf(viewer, "\\newcommand{\\vEnd}{%" PetscInt_FMT "}\n", vEnd - 1));
1096       PetscCall(PetscViewerASCIIPrintf(viewer, "\\newcommand{\\numVertices}{%" PetscInt_FMT "}\n", vEnd - vStart));
1097       PetscCall(PetscViewerASCIIPrintf(viewer, "\\newcommand{\\vShift}{%.2f}\n", 3 + (maxStratum - (vEnd - vStart)) / 2.));
1098       PetscCall(PetscViewerASCIIPrintf(viewer, "\\newcommand{\\eStart}{%" PetscInt_FMT "}\n", eStart));
1099       PetscCall(PetscViewerASCIIPrintf(viewer, "\\newcommand{\\eEnd}{%" PetscInt_FMT "}\n", eEnd - 1));
1100       PetscCall(PetscViewerASCIIPrintf(viewer, "\\newcommand{\\eShift}{%.2f}\n", 3 + (maxStratum - (eEnd - eStart)) / 2.));
1101       PetscCall(PetscViewerASCIIPrintf(viewer, "\\newcommand{\\numEdges}{%" PetscInt_FMT "}\n", eEnd - eStart));
1102       PetscCall(PetscViewerASCIIPrintf(viewer, "\\newcommand{\\fStart}{%" PetscInt_FMT "}\n", fStart));
1103       PetscCall(PetscViewerASCIIPrintf(viewer, "\\newcommand{\\fEnd}{%" PetscInt_FMT "}\n", fEnd - 1));
1104       PetscCall(PetscViewerASCIIPrintf(viewer, "\\newcommand{\\fShift}{%.2f}\n", 3 + (maxStratum - (fEnd - fStart)) / 2.));
1105       PetscCall(PetscViewerASCIIPrintf(viewer, "\\newcommand{\\numFaces}{%" PetscInt_FMT "}\n", fEnd - fStart));
1106       PetscCall(PetscViewerASCIIPrintf(viewer, "\\newcommand{\\cStart}{%" PetscInt_FMT "}\n", cStart));
1107       PetscCall(PetscViewerASCIIPrintf(viewer, "\\newcommand{\\cEnd}{%" PetscInt_FMT "}\n", cEnd - 1));
1108       PetscCall(PetscViewerASCIIPrintf(viewer, "\\newcommand{\\numCells}{%" PetscInt_FMT "}\n", cEnd - cStart));
1109       PetscCall(PetscViewerASCIIPrintf(viewer, "\\newcommand{\\cShift}{%.2f}\n", 3 + (maxStratum - (cEnd - cStart)) / 2.));
1110     }
1111     PetscCall(PetscViewerASCIIPrintf(viewer, "\\begin{tikzpicture}[scale = %g,font=\\fontsize{8}{8}\\selectfont]\n", (double)tikzscale));
1112 
1113     /* Plot vertices */
1114     PetscCall(VecGetArray(coordinates, &coords));
1115     PetscCall(PetscViewerASCIIPushSynchronized(viewer));
1116     for (v = vStart; v < vEnd; ++v) {
1117       PetscInt  off, dof, d;
1118       PetscBool isLabeled = PETSC_FALSE;
1119 
1120       if (wp && !PetscBTLookup(wp, v - pStart)) continue;
1121       PetscCall(PetscSectionGetDof(coordSection, v, &dof));
1122       PetscCall(PetscSectionGetOffset(coordSection, v, &off));
1123       PetscCall(PetscViewerASCIISynchronizedPrintf(viewer, "\\path ("));
1124       PetscCheck(dof <= 3, PETSC_COMM_SELF, PETSC_ERR_PLIB, "coordSection vertex %" PetscInt_FMT " has dof %" PetscInt_FMT " > 3", v, dof);
1125       for (d = 0; d < dof; ++d) {
1126         tcoords[d] = (double)(scale * PetscRealPart(coords[off + d]));
1127         tcoords[d] = PetscAbs(tcoords[d]) < 1e-10 ? 0.0 : tcoords[d];
1128       }
1129       /* Rotate coordinates since PGF makes z point out of the page instead of up */
1130       if (dim == 3) {
1131         PetscReal tmp = tcoords[1];
1132         tcoords[1]    = tcoords[2];
1133         tcoords[2]    = -tmp;
1134       }
1135       for (d = 0; d < dof; ++d) {
1136         if (d > 0) PetscCall(PetscViewerASCIISynchronizedPrintf(viewer, ","));
1137         PetscCall(PetscViewerASCIISynchronizedPrintf(viewer, "%g", (double)tcoords[d]));
1138       }
1139       if (drawHasse) color = colors[0 % numColors];
1140       else color = colors[rank % numColors];
1141       for (l = 0; l < numLabels; ++l) {
1142         PetscInt val;
1143         PetscCall(DMGetLabelValue(dm, names[l], v, &val));
1144         if (val >= 0) {
1145           color     = lcolors[l % numLColors];
1146           isLabeled = PETSC_TRUE;
1147           break;
1148         }
1149       }
1150       if (drawNumbers[0]) {
1151         PetscCall(PetscViewerASCIISynchronizedPrintf(viewer, ") node(%" PetscInt_FMT "_%d) [draw,shape=circle,color=%s] {%" PetscInt_FMT "};\n", v, rank, color, v));
1152       } else if (drawColors[0]) {
1153         PetscCall(PetscViewerASCIISynchronizedPrintf(viewer, ") node(%" PetscInt_FMT "_%d) [fill,inner sep=%dpt,shape=circle,color=%s] {};\n", v, rank, !isLabeled ? 1 : 2, color));
1154       } else PetscCall(PetscViewerASCIISynchronizedPrintf(viewer, ") node(%" PetscInt_FMT "_%d) [] {};\n", v, rank));
1155     }
1156     PetscCall(VecRestoreArray(coordinates, &coords));
1157     PetscCall(PetscViewerFlush(viewer));
1158     /* Plot edges */
1159     if (plotEdges) {
1160       PetscCall(VecGetArray(coordinates, &coords));
1161       PetscCall(PetscViewerASCIIPrintf(viewer, "\\path\n"));
1162       for (e = eStart; e < eEnd; ++e) {
1163         const PetscInt *cone;
1164         PetscInt        coneSize, offA, offB, dof, d;
1165 
1166         if (wp && !PetscBTLookup(wp, e - pStart)) continue;
1167         PetscCall(DMPlexGetConeSize(dm, e, &coneSize));
1168         PetscCheck(coneSize == 2, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Edge %" PetscInt_FMT " cone should have two vertices, not %" PetscInt_FMT, e, coneSize);
1169         PetscCall(DMPlexGetCone(dm, e, &cone));
1170         PetscCall(PetscSectionGetDof(coordSection, cone[0], &dof));
1171         PetscCall(PetscSectionGetOffset(coordSection, cone[0], &offA));
1172         PetscCall(PetscSectionGetOffset(coordSection, cone[1], &offB));
1173         PetscCall(PetscViewerASCIISynchronizedPrintf(viewer, "("));
1174         for (d = 0; d < dof; ++d) {
1175           tcoords[d] = (double)(0.5 * scale * PetscRealPart(coords[offA + d] + coords[offB + d]));
1176           tcoords[d] = PetscAbs(tcoords[d]) < 1e-10 ? 0.0 : tcoords[d];
1177         }
1178         /* Rotate coordinates since PGF makes z point out of the page instead of up */
1179         if (dim == 3) {
1180           PetscReal tmp = tcoords[1];
1181           tcoords[1]    = tcoords[2];
1182           tcoords[2]    = -tmp;
1183         }
1184         for (d = 0; d < dof; ++d) {
1185           if (d > 0) PetscCall(PetscViewerASCIISynchronizedPrintf(viewer, ","));
1186           PetscCall(PetscViewerASCIISynchronizedPrintf(viewer, "%g", (double)tcoords[d]));
1187         }
1188         if (drawHasse) color = colors[1 % numColors];
1189         else color = colors[rank % numColors];
1190         for (l = 0; l < numLabels; ++l) {
1191           PetscInt val;
1192           PetscCall(DMGetLabelValue(dm, names[l], e, &val));
1193           if (val >= 0) {
1194             color = lcolors[l % numLColors];
1195             break;
1196           }
1197         }
1198         PetscCall(PetscViewerASCIISynchronizedPrintf(viewer, ") node(%" PetscInt_FMT "_%d) [draw,shape=circle,color=%s] {%" PetscInt_FMT "} --\n", e, rank, color, e));
1199       }
1200       PetscCall(VecRestoreArray(coordinates, &coords));
1201       PetscCall(PetscViewerFlush(viewer));
1202       PetscCall(PetscViewerASCIIPrintf(viewer, "(0,0);\n"));
1203     }
1204     /* Plot cells */
1205     if (dim == 3 || !drawNumbers[1]) {
1206       for (e = eStart; e < eEnd; ++e) {
1207         const PetscInt *cone;
1208 
1209         if (wp && !PetscBTLookup(wp, e - pStart)) continue;
1210         color = colors[rank % numColors];
1211         for (l = 0; l < numLabels; ++l) {
1212           PetscInt val;
1213           PetscCall(DMGetLabelValue(dm, names[l], e, &val));
1214           if (val >= 0) {
1215             color = lcolors[l % numLColors];
1216             break;
1217           }
1218         }
1219         PetscCall(DMPlexGetCone(dm, e, &cone));
1220         PetscCall(PetscViewerASCIISynchronizedPrintf(viewer, "\\draw[color=%s] (%" PetscInt_FMT "_%d) -- (%" PetscInt_FMT "_%d);\n", color, cone[0], rank, cone[1], rank));
1221       }
1222     } else {
1223       DMPolytopeType ct;
1224 
1225       /* Drawing a 2D polygon */
1226       for (c = cStart; c < cEnd; ++c) {
1227         if (wp && !PetscBTLookup(wp, c - pStart)) continue;
1228         PetscCall(DMPlexGetCellType(dm, c, &ct));
1229         if (DMPolytopeTypeIsHybrid(ct)) {
1230           const PetscInt *cone;
1231           PetscInt        coneSize, e;
1232 
1233           PetscCall(DMPlexGetCone(dm, c, &cone));
1234           PetscCall(DMPlexGetConeSize(dm, c, &coneSize));
1235           for (e = 0; e < coneSize; ++e) {
1236             const PetscInt *econe;
1237 
1238             PetscCall(DMPlexGetCone(dm, cone[e], &econe));
1239             PetscCall(PetscViewerASCIISynchronizedPrintf(viewer, "\\draw[color=%s] (%" PetscInt_FMT "_%d) -- (%" PetscInt_FMT "_%d) -- (%" PetscInt_FMT "_%d);\n", colors[rank % numColors], econe[0], rank, cone[e], rank, econe[1], rank));
1240           }
1241         } else {
1242           PetscInt *closure = NULL;
1243           PetscInt  closureSize, Nv = 0, v;
1244 
1245           PetscCall(DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure));
1246           for (p = 0; p < closureSize * 2; p += 2) {
1247             const PetscInt point = closure[p];
1248 
1249             if ((point >= vStart) && (point < vEnd)) closure[Nv++] = point;
1250           }
1251           PetscCall(PetscViewerASCIISynchronizedPrintf(viewer, "\\draw[color=%s] ", colors[rank % numColors]));
1252           for (v = 0; v <= Nv; ++v) {
1253             const PetscInt vertex = closure[v % Nv];
1254 
1255             if (v > 0) {
1256               if (plotEdges) {
1257                 const PetscInt *edge;
1258                 PetscInt        endpoints[2], ne;
1259 
1260                 endpoints[0] = closure[v - 1];
1261                 endpoints[1] = vertex;
1262                 PetscCall(DMPlexGetJoin(dm, 2, endpoints, &ne, &edge));
1263                 PetscCheck(ne == 1, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Could not find edge for vertices %" PetscInt_FMT ", %" PetscInt_FMT, endpoints[0], endpoints[1]);
1264                 PetscCall(PetscViewerASCIISynchronizedPrintf(viewer, " -- (%" PetscInt_FMT "_%d) -- ", edge[0], rank));
1265                 PetscCall(DMPlexRestoreJoin(dm, 2, endpoints, &ne, &edge));
1266               } else PetscCall(PetscViewerASCIISynchronizedPrintf(viewer, " -- "));
1267             }
1268             PetscCall(PetscViewerASCIISynchronizedPrintf(viewer, "(%" PetscInt_FMT "_%d)", vertex, rank));
1269           }
1270           PetscCall(PetscViewerASCIISynchronizedPrintf(viewer, ";\n"));
1271           PetscCall(DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure));
1272         }
1273       }
1274     }
1275     for (c = cStart; c < cEnd; ++c) {
1276       double             ccoords[3] = {0.0, 0.0, 0.0};
1277       PetscBool          isLabeled  = PETSC_FALSE;
1278       PetscScalar       *cellCoords = NULL;
1279       const PetscScalar *array;
1280       PetscInt           numCoords, cdim, d;
1281       PetscBool          isDG;
1282 
1283       if (wp && !PetscBTLookup(wp, c - pStart)) continue;
1284       PetscCall(DMGetCoordinateDim(dm, &cdim));
1285       PetscCall(DMPlexGetCellCoordinates(dm, c, &isDG, &numCoords, &array, &cellCoords));
1286       PetscCheck(!(numCoords % cdim), PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "coordinate dim %" PetscInt_FMT " does not divide numCoords %" PetscInt_FMT, cdim, numCoords);
1287       PetscCall(PetscViewerASCIISynchronizedPrintf(viewer, "\\path ("));
1288       for (p = 0; p < numCoords / cdim; ++p) {
1289         for (d = 0; d < cdim; ++d) {
1290           tcoords[d] = (double)(scale * PetscRealPart(cellCoords[p * cdim + d]));
1291           tcoords[d] = PetscAbs(tcoords[d]) < 1e-10 ? 0.0 : tcoords[d];
1292         }
1293         /* Rotate coordinates since PGF makes z point out of the page instead of up */
1294         if (cdim == 3) {
1295           PetscReal tmp = tcoords[1];
1296           tcoords[1]    = tcoords[2];
1297           tcoords[2]    = -tmp;
1298         }
1299         for (d = 0; d < dim; ++d) ccoords[d] += tcoords[d];
1300       }
1301       for (d = 0; d < cdim; ++d) ccoords[d] /= (numCoords / cdim);
1302       PetscCall(DMPlexRestoreCellCoordinates(dm, c, &isDG, &numCoords, &array, &cellCoords));
1303       for (d = 0; d < cdim; ++d) {
1304         if (d > 0) PetscCall(PetscViewerASCIISynchronizedPrintf(viewer, ","));
1305         PetscCall(PetscViewerASCIISynchronizedPrintf(viewer, "%g", (double)ccoords[d]));
1306       }
1307       if (drawHasse) color = colors[depth % numColors];
1308       else color = colors[rank % numColors];
1309       for (l = 0; l < numLabels; ++l) {
1310         PetscInt val;
1311         PetscCall(DMGetLabelValue(dm, names[l], c, &val));
1312         if (val >= 0) {
1313           color     = lcolors[l % numLColors];
1314           isLabeled = PETSC_TRUE;
1315           break;
1316         }
1317       }
1318       if (drawNumbers[dim]) {
1319         PetscCall(PetscViewerASCIISynchronizedPrintf(viewer, ") node(%" PetscInt_FMT "_%d) [draw,shape=circle,color=%s] {%" PetscInt_FMT "};\n", c, rank, color, c));
1320       } else if (drawColors[dim]) {
1321         PetscCall(PetscViewerASCIISynchronizedPrintf(viewer, ") node(%" PetscInt_FMT "_%d) [fill,inner sep=%dpt,shape=circle,color=%s] {};\n", c, rank, !isLabeled ? 1 : 2, color));
1322       } else PetscCall(PetscViewerASCIISynchronizedPrintf(viewer, ") node(%" PetscInt_FMT "_%d) [] {};\n", c, rank));
1323     }
1324     if (drawHasse) {
1325       int height = 0;
1326 
1327       color = colors[depth % numColors];
1328       PetscCall(PetscViewerASCIIPrintf(viewer, "%% Cells\n"));
1329       PetscCall(PetscViewerASCIIPrintf(viewer, "\\foreach \\c in {\\cStart,...,\\cEnd}\n"));
1330       PetscCall(PetscViewerASCIIPrintf(viewer, "{\n"));
1331       PetscCall(PetscViewerASCIIPrintf(viewer, "  \\node(\\c_%d) [draw,shape=circle,color=%s,minimum size = 6mm] at (\\cShift+\\c-\\cStart,%d) {\\c};\n", rank, color, height++));
1332       PetscCall(PetscViewerASCIIPrintf(viewer, "}\n"));
1333 
1334       if (depth > 2) {
1335         color = colors[1 % numColors];
1336         PetscCall(PetscViewerASCIIPrintf(viewer, "%% Faces\n"));
1337         PetscCall(PetscViewerASCIIPrintf(viewer, "\\foreach \\f in {\\fStart,...,\\fEnd}\n"));
1338         PetscCall(PetscViewerASCIIPrintf(viewer, "{\n"));
1339         PetscCall(PetscViewerASCIIPrintf(viewer, "  \\node(\\f_%d) [draw,shape=circle,color=%s,minimum size = 6mm] at (\\fShift+\\f-\\fStart,%d) {\\f};\n", rank, color, height++));
1340         PetscCall(PetscViewerASCIIPrintf(viewer, "}\n"));
1341       }
1342 
1343       color = colors[1 % numColors];
1344       PetscCall(PetscViewerASCIIPrintf(viewer, "%% Edges\n"));
1345       PetscCall(PetscViewerASCIIPrintf(viewer, "\\foreach \\e in {\\eStart,...,\\eEnd}\n"));
1346       PetscCall(PetscViewerASCIIPrintf(viewer, "{\n"));
1347       PetscCall(PetscViewerASCIIPrintf(viewer, "  \\node(\\e_%d) [draw,shape=circle,color=%s,minimum size = 6mm] at (\\eShift+\\e-\\eStart,%d) {\\e};\n", rank, color, height++));
1348       PetscCall(PetscViewerASCIIPrintf(viewer, "}\n"));
1349 
1350       color = colors[0 % numColors];
1351       PetscCall(PetscViewerASCIIPrintf(viewer, "%% Vertices\n"));
1352       PetscCall(PetscViewerASCIIPrintf(viewer, "\\foreach \\v in {\\vStart,...,\\vEnd}\n"));
1353       PetscCall(PetscViewerASCIIPrintf(viewer, "{\n"));
1354       PetscCall(PetscViewerASCIIPrintf(viewer, "  \\node(\\v_%d) [draw,shape=circle,color=%s,minimum size = 6mm] at (\\vShift+\\v-\\vStart,%d) {\\v};\n", rank, color, height++));
1355       PetscCall(PetscViewerASCIIPrintf(viewer, "}\n"));
1356 
1357       for (p = pStart; p < pEnd; ++p) {
1358         const PetscInt *cone;
1359         PetscInt        coneSize, cp;
1360 
1361         PetscCall(DMPlexGetCone(dm, p, &cone));
1362         PetscCall(DMPlexGetConeSize(dm, p, &coneSize));
1363         for (cp = 0; cp < coneSize; ++cp) PetscCall(PetscViewerASCIIPrintf(viewer, "\\draw[->, shorten >=1pt] (%" PetscInt_FMT "_%d) -- (%" PetscInt_FMT "_%d);\n", cone[cp], rank, p, rank));
1364       }
1365     }
1366     PetscCall(PetscViewerFlush(viewer));
1367     PetscCall(PetscViewerASCIIPopSynchronized(viewer));
1368     PetscCall(PetscViewerASCIIPrintf(viewer, "\\end{tikzpicture}\n"));
1369     PetscCall(PetscViewerASCIIPrintf(viewer, "\\end{document}\n"));
1370     for (l = 0; l < numLabels; ++l) PetscCall(PetscFree(names[l]));
1371     for (c = 0; c < numColors; ++c) PetscCall(PetscFree(colors[c]));
1372     for (c = 0; c < numLColors; ++c) PetscCall(PetscFree(lcolors[c]));
1373     PetscCall(PetscFree3(names, colors, lcolors));
1374     PetscCall(PetscBTDestroy(&wp));
1375   } else if (format == PETSC_VIEWER_LOAD_BALANCE) {
1376     Vec                    cown, acown;
1377     VecScatter             sct;
1378     ISLocalToGlobalMapping g2l;
1379     IS                     gid, acis;
1380     MPI_Comm               comm, ncomm = MPI_COMM_NULL;
1381     MPI_Group              ggroup, ngroup;
1382     PetscScalar           *array, nid;
1383     const PetscInt        *idxs;
1384     PetscInt              *idxs2, *start, *adjacency, *work;
1385     PetscInt64             lm[3], gm[3];
1386     PetscInt               i, c, cStart, cEnd, cum, numVertices, ect, ectn, cellHeight;
1387     PetscMPIInt            d1, d2, rank;
1388 
1389     PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
1390     PetscCallMPI(MPI_Comm_rank(comm, &rank));
1391 #if defined(PETSC_HAVE_MPI_PROCESS_SHARED_MEMORY)
1392     PetscCallMPI(MPI_Comm_split_type(comm, MPI_COMM_TYPE_SHARED, rank, MPI_INFO_NULL, &ncomm));
1393 #endif
1394     if (ncomm != MPI_COMM_NULL) {
1395       PetscCallMPI(MPI_Comm_group(comm, &ggroup));
1396       PetscCallMPI(MPI_Comm_group(ncomm, &ngroup));
1397       d1 = 0;
1398       PetscCallMPI(MPI_Group_translate_ranks(ngroup, 1, &d1, ggroup, &d2));
1399       nid = d2;
1400       PetscCallMPI(MPI_Group_free(&ggroup));
1401       PetscCallMPI(MPI_Group_free(&ngroup));
1402       PetscCallMPI(MPI_Comm_free(&ncomm));
1403     } else nid = 0.0;
1404 
1405     /* Get connectivity */
1406     PetscCall(DMPlexGetVTKCellHeight(dm, &cellHeight));
1407     PetscCall(DMPlexCreatePartitionerGraph(dm, cellHeight, &numVertices, &start, &adjacency, &gid));
1408 
1409     /* filter overlapped local cells */
1410     PetscCall(DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd));
1411     PetscCall(ISGetIndices(gid, &idxs));
1412     PetscCall(ISGetLocalSize(gid, &cum));
1413     PetscCall(PetscMalloc1(cum, &idxs2));
1414     for (c = cStart, cum = 0; c < cEnd; c++) {
1415       if (idxs[c - cStart] < 0) continue;
1416       idxs2[cum++] = idxs[c - cStart];
1417     }
1418     PetscCall(ISRestoreIndices(gid, &idxs));
1419     PetscCheck(numVertices == cum, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Unexpected %" PetscInt_FMT " != %" PetscInt_FMT, numVertices, cum);
1420     PetscCall(ISDestroy(&gid));
1421     PetscCall(ISCreateGeneral(comm, numVertices, idxs2, PETSC_OWN_POINTER, &gid));
1422 
1423     /* support for node-aware cell locality */
1424     PetscCall(ISCreateGeneral(comm, start[numVertices], adjacency, PETSC_USE_POINTER, &acis));
1425     PetscCall(VecCreateSeq(PETSC_COMM_SELF, start[numVertices], &acown));
1426     PetscCall(VecCreateMPI(comm, numVertices, PETSC_DECIDE, &cown));
1427     PetscCall(VecGetArray(cown, &array));
1428     for (c = 0; c < numVertices; c++) array[c] = nid;
1429     PetscCall(VecRestoreArray(cown, &array));
1430     PetscCall(VecScatterCreate(cown, acis, acown, NULL, &sct));
1431     PetscCall(VecScatterBegin(sct, cown, acown, INSERT_VALUES, SCATTER_FORWARD));
1432     PetscCall(VecScatterEnd(sct, cown, acown, INSERT_VALUES, SCATTER_FORWARD));
1433     PetscCall(ISDestroy(&acis));
1434     PetscCall(VecScatterDestroy(&sct));
1435     PetscCall(VecDestroy(&cown));
1436 
1437     /* compute edgeCut */
1438     for (c = 0, cum = 0; c < numVertices; c++) cum = PetscMax(cum, start[c + 1] - start[c]);
1439     PetscCall(PetscMalloc1(cum, &work));
1440     PetscCall(ISLocalToGlobalMappingCreateIS(gid, &g2l));
1441     PetscCall(ISLocalToGlobalMappingSetType(g2l, ISLOCALTOGLOBALMAPPINGHASH));
1442     PetscCall(ISDestroy(&gid));
1443     PetscCall(VecGetArray(acown, &array));
1444     for (c = 0, ect = 0, ectn = 0; c < numVertices; c++) {
1445       PetscInt totl;
1446 
1447       totl = start[c + 1] - start[c];
1448       PetscCall(ISGlobalToLocalMappingApply(g2l, IS_GTOLM_MASK, totl, adjacency + start[c], NULL, work));
1449       for (i = 0; i < totl; i++) {
1450         if (work[i] < 0) {
1451           ect += 1;
1452           ectn += (array[i + start[c]] != nid) ? 0 : 1;
1453         }
1454       }
1455     }
1456     PetscCall(PetscFree(work));
1457     PetscCall(VecRestoreArray(acown, &array));
1458     lm[0] = numVertices > 0 ? numVertices : PETSC_INT_MAX;
1459     lm[1] = -numVertices;
1460     PetscCallMPI(MPIU_Allreduce(lm, gm, 2, MPIU_INT64, MPI_MIN, comm));
1461     PetscCall(PetscViewerASCIIPrintf(viewer, "  Cell balance: %.2f (max %" PetscInt_FMT ", min %" PetscInt_FMT, -((double)gm[1]) / ((double)gm[0]), -(PetscInt)gm[1], (PetscInt)gm[0]));
1462     lm[0] = ect;                     /* edgeCut */
1463     lm[1] = ectn;                    /* node-aware edgeCut */
1464     lm[2] = numVertices > 0 ? 0 : 1; /* empty processes */
1465     PetscCallMPI(MPIU_Allreduce(lm, gm, 3, MPIU_INT64, MPI_SUM, comm));
1466     PetscCall(PetscViewerASCIIPrintf(viewer, ", empty %" PetscInt_FMT ")\n", (PetscInt)gm[2]));
1467 #if defined(PETSC_HAVE_MPI_PROCESS_SHARED_MEMORY)
1468     PetscCall(PetscViewerASCIIPrintf(viewer, "  Edge Cut: %" PetscInt_FMT " (on node %.3f)\n", (PetscInt)(gm[0] / 2), gm[0] ? ((double)gm[1]) / ((double)gm[0]) : 1.));
1469 #else
1470     PetscCall(PetscViewerASCIIPrintf(viewer, "  Edge Cut: %" PetscInt_FMT " (on node %.3f)\n", (PetscInt)(gm[0] / 2), 0.0));
1471 #endif
1472     PetscCall(ISLocalToGlobalMappingDestroy(&g2l));
1473     PetscCall(PetscFree(start));
1474     PetscCall(PetscFree(adjacency));
1475     PetscCall(VecDestroy(&acown));
1476   } else {
1477     const char    *name;
1478     PetscInt      *sizes, *hybsizes, *ghostsizes;
1479     PetscInt       locDepth, depth, cellHeight, dim, d;
1480     PetscInt       pStart, pEnd, p, gcStart, gcEnd, gcNum;
1481     PetscInt       numLabels, l, maxSize = 17;
1482     DMPolytopeType ct0 = DM_POLYTOPE_UNKNOWN;
1483     MPI_Comm       comm;
1484     PetscMPIInt    size, rank;
1485 
1486     PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
1487     PetscCallMPI(MPI_Comm_size(comm, &size));
1488     PetscCallMPI(MPI_Comm_rank(comm, &rank));
1489     PetscCall(DMGetDimension(dm, &dim));
1490     PetscCall(DMPlexGetVTKCellHeight(dm, &cellHeight));
1491     PetscCall(PetscObjectGetName((PetscObject)dm, &name));
1492     if (name) PetscCall(PetscViewerASCIIPrintf(viewer, "%s in %" PetscInt_FMT " dimension%s:\n", name, dim, dim == 1 ? "" : "s"));
1493     else PetscCall(PetscViewerASCIIPrintf(viewer, "Mesh in %" PetscInt_FMT " dimension%s:\n", dim, dim == 1 ? "" : "s"));
1494     if (cellHeight) PetscCall(PetscViewerASCIIPrintf(viewer, "  Cells are at height %" PetscInt_FMT "\n", cellHeight));
1495     PetscCall(DMPlexGetDepth(dm, &locDepth));
1496     PetscCallMPI(MPIU_Allreduce(&locDepth, &depth, 1, MPIU_INT, MPI_MAX, comm));
1497     PetscCall(DMPlexGetCellTypeStratum(dm, DM_POLYTOPE_FV_GHOST, &gcStart, &gcEnd));
1498     gcNum = gcEnd - gcStart;
1499     if (size < maxSize) PetscCall(PetscCalloc3(size, &sizes, size, &hybsizes, size, &ghostsizes));
1500     else PetscCall(PetscCalloc3(3, &sizes, 3, &hybsizes, 3, &ghostsizes));
1501     for (d = 0; d <= depth; d++) {
1502       PetscInt Nc[2] = {0, 0}, ict;
1503 
1504       PetscCall(DMPlexGetDepthStratum(dm, d, &pStart, &pEnd));
1505       if (pStart < pEnd) PetscCall(DMPlexGetCellType(dm, pStart, &ct0));
1506       ict = ct0;
1507       PetscCallMPI(MPI_Bcast(&ict, 1, MPIU_INT, 0, comm));
1508       ct0 = (DMPolytopeType)ict;
1509       for (p = pStart; p < pEnd; ++p) {
1510         DMPolytopeType ct;
1511 
1512         PetscCall(DMPlexGetCellType(dm, p, &ct));
1513         if (ct == ct0) ++Nc[0];
1514         else ++Nc[1];
1515       }
1516       if (size < maxSize) {
1517         PetscCallMPI(MPI_Gather(&Nc[0], 1, MPIU_INT, sizes, 1, MPIU_INT, 0, comm));
1518         PetscCallMPI(MPI_Gather(&Nc[1], 1, MPIU_INT, hybsizes, 1, MPIU_INT, 0, comm));
1519         if (d == depth) PetscCallMPI(MPI_Gather(&gcNum, 1, MPIU_INT, ghostsizes, 1, MPIU_INT, 0, comm));
1520         PetscCall(PetscViewerASCIIPrintf(viewer, "  Number of %" PetscInt_FMT "-cells per rank:", (depth == 1) && d ? dim : d));
1521         for (p = 0; p < size; ++p) {
1522           if (rank == 0) {
1523             PetscCall(PetscViewerASCIIPrintf(viewer, " %" PetscInt_FMT, sizes[p] + hybsizes[p]));
1524             if (hybsizes[p] > 0) PetscCall(PetscViewerASCIIPrintf(viewer, " (%" PetscInt_FMT ")", hybsizes[p]));
1525             if (ghostsizes[p] > 0) PetscCall(PetscViewerASCIIPrintf(viewer, " [%" PetscInt_FMT "]", ghostsizes[p]));
1526           }
1527         }
1528       } else {
1529         PetscInt locMinMax[2];
1530 
1531         locMinMax[0] = Nc[0] + Nc[1];
1532         locMinMax[1] = Nc[0] + Nc[1];
1533         PetscCall(PetscGlobalMinMaxInt(comm, locMinMax, sizes));
1534         locMinMax[0] = Nc[1];
1535         locMinMax[1] = Nc[1];
1536         PetscCall(PetscGlobalMinMaxInt(comm, locMinMax, hybsizes));
1537         if (d == depth) {
1538           locMinMax[0] = gcNum;
1539           locMinMax[1] = gcNum;
1540           PetscCall(PetscGlobalMinMaxInt(comm, locMinMax, ghostsizes));
1541         }
1542         PetscCall(PetscViewerASCIIPrintf(viewer, "  Min/Max of %" PetscInt_FMT "-cells per rank:", (depth == 1) && d ? dim : d));
1543         PetscCall(PetscViewerASCIIPrintf(viewer, " %" PetscInt_FMT "/%" PetscInt_FMT, sizes[0], sizes[1]));
1544         if (hybsizes[0] > 0) PetscCall(PetscViewerASCIIPrintf(viewer, " (%" PetscInt_FMT "/%" PetscInt_FMT ")", hybsizes[0], hybsizes[1]));
1545         if (ghostsizes[0] > 0) PetscCall(PetscViewerASCIIPrintf(viewer, " [%" PetscInt_FMT "/%" PetscInt_FMT "]", ghostsizes[0], ghostsizes[1]));
1546       }
1547       PetscCall(PetscViewerASCIIPrintf(viewer, "\n"));
1548     }
1549     PetscCall(PetscFree3(sizes, hybsizes, ghostsizes));
1550     {
1551       const PetscReal *maxCell;
1552       const PetscReal *L;
1553       PetscBool        localized;
1554 
1555       PetscCall(DMGetPeriodicity(dm, &maxCell, NULL, &L));
1556       PetscCall(DMGetCoordinatesLocalized(dm, &localized));
1557       if (L || localized) {
1558         PetscCall(PetscViewerASCIIPrintf(viewer, "Periodic mesh"));
1559         PetscCall(PetscViewerASCIIUseTabs(viewer, PETSC_FALSE));
1560         if (L) {
1561           PetscCall(PetscViewerASCIIPrintf(viewer, " ("));
1562           for (d = 0; d < dim; ++d) {
1563             if (d > 0) PetscCall(PetscViewerASCIIPrintf(viewer, ", "));
1564             PetscCall(PetscViewerASCIIPrintf(viewer, "%s", L[d] > 0.0 ? "PERIODIC" : "NONE"));
1565           }
1566           PetscCall(PetscViewerASCIIPrintf(viewer, ")"));
1567         }
1568         PetscCall(PetscViewerASCIIPrintf(viewer, " coordinates %s\n", localized ? "localized" : "not localized"));
1569         PetscCall(PetscViewerASCIIUseTabs(viewer, PETSC_TRUE));
1570       }
1571     }
1572     PetscCall(DMGetNumLabels(dm, &numLabels));
1573     if (numLabels) PetscCall(PetscViewerASCIIPrintf(viewer, "Labels:\n"));
1574     for (l = 0; l < numLabels; ++l) {
1575       DMLabel     label;
1576       const char *name;
1577       PetscInt   *values;
1578       PetscInt    numValues, v;
1579 
1580       PetscCall(DMGetLabelName(dm, l, &name));
1581       PetscCall(DMGetLabel(dm, name, &label));
1582       PetscCall(DMLabelGetNumValues(label, &numValues));
1583       PetscCall(PetscViewerASCIIPrintf(viewer, "  %s: %" PetscInt_FMT " strata with value/size (", name, numValues));
1584 
1585       { // Extract array of DMLabel values so it can be sorted
1586         IS              is_values;
1587         const PetscInt *is_values_local = NULL;
1588 
1589         PetscCall(DMLabelGetValueIS(label, &is_values));
1590         PetscCall(ISGetIndices(is_values, &is_values_local));
1591         PetscCall(PetscMalloc1(numValues, &values));
1592         PetscCall(PetscArraycpy(values, is_values_local, numValues));
1593         PetscCall(PetscSortInt(numValues, values));
1594         PetscCall(ISRestoreIndices(is_values, &is_values_local));
1595         PetscCall(ISDestroy(&is_values));
1596       }
1597       PetscCall(PetscViewerASCIIUseTabs(viewer, PETSC_FALSE));
1598       for (v = 0; v < numValues; ++v) {
1599         PetscInt size;
1600 
1601         PetscCall(DMLabelGetStratumSize(label, values[v], &size));
1602         if (v > 0) PetscCall(PetscViewerASCIIPrintf(viewer, ", "));
1603         PetscCall(PetscViewerASCIIPrintf(viewer, "%" PetscInt_FMT " (%" PetscInt_FMT ")", values[v], size));
1604       }
1605       PetscCall(PetscViewerASCIIPrintf(viewer, ")\n"));
1606       PetscCall(PetscViewerASCIIUseTabs(viewer, PETSC_TRUE));
1607       PetscCall(PetscFree(values));
1608     }
1609     {
1610       char    **labelNames;
1611       PetscInt  Nl = numLabels;
1612       PetscBool flg;
1613 
1614       PetscCall(PetscMalloc1(Nl, &labelNames));
1615       PetscCall(PetscOptionsGetStringArray(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_plex_view_labels", labelNames, &Nl, &flg));
1616       for (l = 0; l < Nl; ++l) {
1617         DMLabel label;
1618 
1619         PetscCall(DMHasLabel(dm, labelNames[l], &flg));
1620         if (flg) {
1621           PetscCall(DMGetLabel(dm, labelNames[l], &label));
1622           PetscCall(DMLabelView(label, viewer));
1623         }
1624         PetscCall(PetscFree(labelNames[l]));
1625       }
1626       PetscCall(PetscFree(labelNames));
1627     }
1628     /* If no fields are specified, people do not want to see adjacency */
1629     if (dm->Nf) {
1630       PetscInt f;
1631 
1632       for (f = 0; f < dm->Nf; ++f) {
1633         const char *name;
1634 
1635         PetscCall(PetscObjectGetName(dm->fields[f].disc, &name));
1636         if (numLabels) PetscCall(PetscViewerASCIIPrintf(viewer, "Field %s:\n", name));
1637         PetscCall(PetscViewerASCIIPushTab(viewer));
1638         if (dm->fields[f].label) PetscCall(DMLabelView(dm->fields[f].label, viewer));
1639         if (dm->fields[f].adjacency[0]) {
1640           if (dm->fields[f].adjacency[1]) PetscCall(PetscViewerASCIIPrintf(viewer, "adjacency FVM++\n"));
1641           else PetscCall(PetscViewerASCIIPrintf(viewer, "adjacency FVM\n"));
1642         } else {
1643           if (dm->fields[f].adjacency[1]) PetscCall(PetscViewerASCIIPrintf(viewer, "adjacency FEM\n"));
1644           else PetscCall(PetscViewerASCIIPrintf(viewer, "adjacency FUNKY\n"));
1645         }
1646         PetscCall(PetscViewerASCIIPopTab(viewer));
1647       }
1648     }
1649     PetscCall(DMGetCoarseDM(dm, &cdm));
1650     if (cdm) {
1651       PetscCall(PetscViewerASCIIPushTab(viewer));
1652       PetscCall(PetscViewerASCIIPrintf(viewer, "Defined by transform from:\n"));
1653       PetscCall(DMPlexView_Ascii(cdm, viewer));
1654       PetscCall(PetscViewerASCIIPopTab(viewer));
1655     }
1656   }
1657   PetscFunctionReturn(PETSC_SUCCESS);
1658 }
1659 
1660 static PetscErrorCode DMPlexDrawCell(DM dm, PetscDraw draw, PetscInt cell, const PetscScalar coords[])
1661 {
1662   DMPolytopeType ct;
1663   PetscMPIInt    rank;
1664   PetscInt       cdim;
1665 
1666   PetscFunctionBegin;
1667   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
1668   PetscCall(DMPlexGetCellType(dm, cell, &ct));
1669   PetscCall(DMGetCoordinateDim(dm, &cdim));
1670   switch (ct) {
1671   case DM_POLYTOPE_SEGMENT:
1672   case DM_POLYTOPE_POINT_PRISM_TENSOR:
1673     switch (cdim) {
1674     case 1: {
1675       const PetscReal y  = 0.5;  /* TODO Put it in the middle of the viewport */
1676       const PetscReal dy = 0.05; /* TODO Make it a fraction of the total length */
1677 
1678       PetscCall(PetscDrawLine(draw, PetscRealPart(coords[0]), y, PetscRealPart(coords[1]), y, PETSC_DRAW_BLACK));
1679       PetscCall(PetscDrawLine(draw, PetscRealPart(coords[0]), y + dy, PetscRealPart(coords[0]), y - dy, PETSC_DRAW_BLACK));
1680       PetscCall(PetscDrawLine(draw, PetscRealPart(coords[1]), y + dy, PetscRealPart(coords[1]), y - dy, PETSC_DRAW_BLACK));
1681     } break;
1682     case 2: {
1683       const PetscReal dx = (PetscRealPart(coords[3]) - PetscRealPart(coords[1]));
1684       const PetscReal dy = (PetscRealPart(coords[2]) - PetscRealPart(coords[0]));
1685       const PetscReal l  = 0.1 / PetscSqrtReal(dx * dx + dy * dy);
1686 
1687       PetscCall(PetscDrawLine(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PETSC_DRAW_BLACK));
1688       PetscCall(PetscDrawLine(draw, PetscRealPart(coords[0]) + l * dx, PetscRealPart(coords[1]) + l * dy, PetscRealPart(coords[0]) - l * dx, PetscRealPart(coords[1]) - l * dy, PETSC_DRAW_BLACK));
1689       PetscCall(PetscDrawLine(draw, PetscRealPart(coords[2]) + l * dx, PetscRealPart(coords[3]) + l * dy, PetscRealPart(coords[2]) - l * dx, PetscRealPart(coords[3]) - l * dy, PETSC_DRAW_BLACK));
1690     } break;
1691     default:
1692       SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot draw cells of dimension %" PetscInt_FMT, cdim);
1693     }
1694     break;
1695   case DM_POLYTOPE_TRIANGLE:
1696     PetscCall(PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS - 2) + 2, PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS - 2) + 2, PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS - 2) + 2));
1697     PetscCall(PetscDrawLine(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PETSC_DRAW_BLACK));
1698     PetscCall(PetscDrawLine(draw, PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), PETSC_DRAW_BLACK));
1699     PetscCall(PetscDrawLine(draw, PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[0]), PetscRealPart(coords[1]), PETSC_DRAW_BLACK));
1700     break;
1701   case DM_POLYTOPE_QUADRILATERAL:
1702     PetscCall(PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS - 2) + 2, PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS - 2) + 2, PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS - 2) + 2));
1703     PetscCall(PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[6]), PetscRealPart(coords[7]), PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS - 2) + 2, PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS - 2) + 2, PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS - 2) + 2));
1704     PetscCall(PetscDrawLine(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PETSC_DRAW_BLACK));
1705     PetscCall(PetscDrawLine(draw, PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), PETSC_DRAW_BLACK));
1706     PetscCall(PetscDrawLine(draw, PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[6]), PetscRealPart(coords[7]), PETSC_DRAW_BLACK));
1707     PetscCall(PetscDrawLine(draw, PetscRealPart(coords[6]), PetscRealPart(coords[7]), PetscRealPart(coords[0]), PetscRealPart(coords[1]), PETSC_DRAW_BLACK));
1708     break;
1709   case DM_POLYTOPE_SEG_PRISM_TENSOR:
1710     PetscCall(PetscDrawTriangle(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS - 2) + 2, PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS - 2) + 2, PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS - 2) + 2));
1711     PetscCall(PetscDrawTriangle(draw, PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[6]), PetscRealPart(coords[7]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS - 2) + 2, PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS - 2) + 2, PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS - 2) + 2));
1712     PetscCall(PetscDrawLine(draw, PetscRealPart(coords[0]), PetscRealPart(coords[1]), PetscRealPart(coords[2]), PetscRealPart(coords[3]), PETSC_DRAW_BLACK));
1713     PetscCall(PetscDrawLine(draw, PetscRealPart(coords[2]), PetscRealPart(coords[3]), PetscRealPart(coords[6]), PetscRealPart(coords[7]), PETSC_DRAW_BLACK));
1714     PetscCall(PetscDrawLine(draw, PetscRealPart(coords[6]), PetscRealPart(coords[7]), PetscRealPart(coords[4]), PetscRealPart(coords[5]), PETSC_DRAW_BLACK));
1715     PetscCall(PetscDrawLine(draw, PetscRealPart(coords[4]), PetscRealPart(coords[5]), PetscRealPart(coords[0]), PetscRealPart(coords[1]), PETSC_DRAW_BLACK));
1716     break;
1717   case DM_POLYTOPE_FV_GHOST:
1718     break;
1719   default:
1720     SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot draw cells of type %s", DMPolytopeTypes[ct]);
1721   }
1722   PetscFunctionReturn(PETSC_SUCCESS);
1723 }
1724 
1725 static PetscErrorCode DrawPolygon_Private(DM dm, PetscDraw draw, PetscInt cell, PetscInt Nv, const PetscReal refVertices[], const PetscScalar coords[], PetscInt edgeDiv, PetscReal refCoords[], PetscReal edgeCoords[])
1726 {
1727   PetscReal   centroid[2] = {0., 0.};
1728   PetscMPIInt rank;
1729   PetscMPIInt fillColor;
1730 
1731   PetscFunctionBegin;
1732   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
1733   fillColor = PETSC_DRAW_WHITE + rank % (PETSC_DRAW_BASIC_COLORS - 2) + 2;
1734   for (PetscInt v = 0; v < Nv; ++v) {
1735     centroid[0] += PetscRealPart(coords[v * 2 + 0]) / Nv;
1736     centroid[1] += PetscRealPart(coords[v * 2 + 1]) / Nv;
1737   }
1738   for (PetscInt e = 0; e < Nv; ++e) {
1739     refCoords[0] = refVertices[e * 2 + 0];
1740     refCoords[1] = refVertices[e * 2 + 1];
1741     for (PetscInt d = 1; d <= edgeDiv; ++d) {
1742       refCoords[d * 2 + 0] = refCoords[0] + (refVertices[(e + 1) % Nv * 2 + 0] - refCoords[0]) * d / edgeDiv;
1743       refCoords[d * 2 + 1] = refCoords[1] + (refVertices[(e + 1) % Nv * 2 + 1] - refCoords[1]) * d / edgeDiv;
1744     }
1745     PetscCall(DMPlexReferenceToCoordinates(dm, cell, edgeDiv + 1, refCoords, edgeCoords));
1746     for (PetscInt d = 0; d < edgeDiv; ++d) {
1747       PetscCall(PetscDrawTriangle(draw, centroid[0], centroid[1], edgeCoords[d * 2 + 0], edgeCoords[d * 2 + 1], edgeCoords[(d + 1) * 2 + 0], edgeCoords[(d + 1) * 2 + 1], fillColor, fillColor, fillColor));
1748       PetscCall(PetscDrawLine(draw, edgeCoords[d * 2 + 0], edgeCoords[d * 2 + 1], edgeCoords[(d + 1) * 2 + 0], edgeCoords[(d + 1) * 2 + 1], PETSC_DRAW_BLACK));
1749     }
1750   }
1751   PetscFunctionReturn(PETSC_SUCCESS);
1752 }
1753 
1754 static PetscErrorCode DMPlexDrawCellHighOrder(DM dm, PetscDraw draw, PetscInt cell, const PetscScalar coords[], PetscInt edgeDiv, PetscReal refCoords[], PetscReal edgeCoords[])
1755 {
1756   DMPolytopeType ct;
1757 
1758   PetscFunctionBegin;
1759   PetscCall(DMPlexGetCellType(dm, cell, &ct));
1760   switch (ct) {
1761   case DM_POLYTOPE_TRIANGLE: {
1762     PetscReal refVertices[6] = {-1., -1., 1., -1., -1., 1.};
1763 
1764     PetscCall(DrawPolygon_Private(dm, draw, cell, 3, refVertices, coords, edgeDiv, refCoords, edgeCoords));
1765   } break;
1766   case DM_POLYTOPE_QUADRILATERAL: {
1767     PetscReal refVertices[8] = {-1., -1., 1., -1., 1., 1., -1., 1.};
1768 
1769     PetscCall(DrawPolygon_Private(dm, draw, cell, 4, refVertices, coords, edgeDiv, refCoords, edgeCoords));
1770   } break;
1771   default:
1772     SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Cannot draw cells of type %s", DMPolytopeTypes[ct]);
1773   }
1774   PetscFunctionReturn(PETSC_SUCCESS);
1775 }
1776 
1777 static PetscErrorCode DMPlexView_Draw(DM dm, PetscViewer viewer)
1778 {
1779   PetscDraw    draw;
1780   DM           cdm;
1781   PetscSection coordSection;
1782   Vec          coordinates;
1783   PetscReal    xyl[3], xyr[3];
1784   PetscReal   *refCoords, *edgeCoords;
1785   PetscBool    isnull, drawAffine;
1786   PetscInt     dim, vStart, vEnd, cStart, cEnd, c, cDegree, edgeDiv;
1787 
1788   PetscFunctionBegin;
1789   PetscCall(DMGetCoordinateDim(dm, &dim));
1790   PetscCheck(dim <= 2, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Cannot draw meshes of dimension %" PetscInt_FMT, dim);
1791   PetscCall(DMGetCoordinateDegree_Internal(dm, &cDegree));
1792   drawAffine = cDegree > 1 ? PETSC_FALSE : PETSC_TRUE;
1793   edgeDiv    = cDegree + 1;
1794   PetscCall(PetscOptionsGetBool(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_view_draw_affine", &drawAffine, NULL));
1795   if (!drawAffine) PetscCall(PetscMalloc2((edgeDiv + 1) * dim, &refCoords, (edgeDiv + 1) * dim, &edgeCoords));
1796   PetscCall(DMGetCoordinateDM(dm, &cdm));
1797   PetscCall(DMGetLocalSection(cdm, &coordSection));
1798   PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
1799   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
1800   PetscCall(DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd));
1801 
1802   PetscCall(PetscViewerDrawGetDraw(viewer, 0, &draw));
1803   PetscCall(PetscDrawIsNull(draw, &isnull));
1804   if (isnull) PetscFunctionReturn(PETSC_SUCCESS);
1805   PetscCall(PetscDrawSetTitle(draw, "Mesh"));
1806 
1807   PetscCall(DMGetBoundingBox(dm, xyl, xyr));
1808   PetscCall(PetscDrawSetCoordinates(draw, xyl[0], xyl[1], xyr[0], xyr[1]));
1809   PetscCall(PetscDrawClear(draw));
1810 
1811   for (c = cStart; c < cEnd; ++c) {
1812     PetscScalar       *coords = NULL;
1813     const PetscScalar *coords_arr;
1814     PetscInt           numCoords;
1815     PetscBool          isDG;
1816 
1817     PetscCall(DMPlexGetCellCoordinates(dm, c, &isDG, &numCoords, &coords_arr, &coords));
1818     if (drawAffine) PetscCall(DMPlexDrawCell(dm, draw, c, coords));
1819     else PetscCall(DMPlexDrawCellHighOrder(dm, draw, c, coords, edgeDiv, refCoords, edgeCoords));
1820     PetscCall(DMPlexRestoreCellCoordinates(dm, c, &isDG, &numCoords, &coords_arr, &coords));
1821   }
1822   if (!drawAffine) PetscCall(PetscFree2(refCoords, edgeCoords));
1823   PetscCall(PetscDrawFlush(draw));
1824   PetscCall(PetscDrawPause(draw));
1825   PetscCall(PetscDrawSave(draw));
1826   PetscFunctionReturn(PETSC_SUCCESS);
1827 }
1828 
1829 static PetscErrorCode DMPlexCreateHighOrderSurrogate_Internal(DM dm, DM *hdm)
1830 {
1831   DM           odm = dm, rdm = dm, cdm;
1832   PetscFE      fe;
1833   PetscSpace   sp;
1834   PetscClassId id;
1835   PetscInt     degree;
1836   PetscBool    hoView = PETSC_TRUE;
1837 
1838   PetscFunctionBegin;
1839   PetscObjectOptionsBegin((PetscObject)dm);
1840   PetscCall(PetscOptionsBool("-dm_plex_high_order_view", "Subsample to view meshes with high order coordinates", "DMPlexCreateHighOrderSurrogate_Internal", hoView, &hoView, NULL));
1841   PetscOptionsEnd();
1842   PetscCall(PetscObjectReference((PetscObject)dm));
1843   *hdm = dm;
1844   if (!hoView) PetscFunctionReturn(PETSC_SUCCESS);
1845   PetscCall(DMGetCoordinateDM(dm, &cdm));
1846   PetscCall(DMGetField(cdm, 0, NULL, (PetscObject *)&fe));
1847   PetscCall(PetscObjectGetClassId((PetscObject)fe, &id));
1848   if (id != PETSCFE_CLASSID) PetscFunctionReturn(PETSC_SUCCESS);
1849   PetscCall(PetscFEGetBasisSpace(fe, &sp));
1850   PetscCall(PetscSpaceGetDegree(sp, &degree, NULL));
1851   for (PetscInt r = 0, rd = PetscCeilReal(((PetscReal)degree) / 2.); r < (PetscInt)PetscCeilReal(PetscLog2Real(degree)); ++r, rd = PetscCeilReal(((PetscReal)rd) / 2.)) {
1852     DM  cdm, rcdm;
1853     Mat In;
1854     Vec cl, rcl;
1855 
1856     PetscCall(DMRefine(odm, PetscObjectComm((PetscObject)odm), &rdm));
1857     PetscCall(DMPlexCreateCoordinateSpace(rdm, rd, PETSC_FALSE, NULL));
1858     PetscCall(PetscObjectSetName((PetscObject)rdm, "Refined Mesh with Linear Coordinates"));
1859     PetscCall(DMGetCoordinateDM(odm, &cdm));
1860     PetscCall(DMGetCoordinateDM(rdm, &rcdm));
1861     PetscCall(DMGetCoordinatesLocal(odm, &cl));
1862     PetscCall(DMGetCoordinatesLocal(rdm, &rcl));
1863     PetscCall(DMSetCoarseDM(rcdm, cdm));
1864     PetscCall(DMCreateInterpolation(cdm, rcdm, &In, NULL));
1865     PetscCall(MatMult(In, cl, rcl));
1866     PetscCall(MatDestroy(&In));
1867     PetscCall(DMSetCoordinatesLocal(rdm, rcl));
1868     PetscCall(DMDestroy(&odm));
1869     odm = rdm;
1870   }
1871   *hdm = rdm;
1872   PetscFunctionReturn(PETSC_SUCCESS);
1873 }
1874 
1875 #if defined(PETSC_HAVE_EXODUSII)
1876   #include <exodusII.h>
1877   #include <petscviewerexodusii.h>
1878 #endif
1879 
1880 PetscErrorCode DMView_Plex(DM dm, PetscViewer viewer)
1881 {
1882   PetscBool iascii, ishdf5, isvtk, isdraw, flg, isglvis, isexodus, iscgns;
1883   char      name[PETSC_MAX_PATH_LEN];
1884 
1885   PetscFunctionBegin;
1886   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1887   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1888   PetscCall(PetscObjectTypeCompare((PetscObject)viewer, PETSCVIEWERASCII, &iascii));
1889   PetscCall(PetscObjectTypeCompare((PetscObject)viewer, PETSCVIEWERVTK, &isvtk));
1890   PetscCall(PetscObjectTypeCompare((PetscObject)viewer, PETSCVIEWERHDF5, &ishdf5));
1891   PetscCall(PetscObjectTypeCompare((PetscObject)viewer, PETSCVIEWERDRAW, &isdraw));
1892   PetscCall(PetscObjectTypeCompare((PetscObject)viewer, PETSCVIEWERGLVIS, &isglvis));
1893   PetscCall(PetscObjectTypeCompare((PetscObject)viewer, PETSCVIEWEREXODUSII, &isexodus));
1894   PetscCall(PetscObjectTypeCompare((PetscObject)viewer, PETSCVIEWERCGNS, &iscgns));
1895   if (iascii) {
1896     PetscViewerFormat format;
1897     PetscCall(PetscViewerGetFormat(viewer, &format));
1898     if (format == PETSC_VIEWER_ASCII_GLVIS) PetscCall(DMPlexView_GLVis(dm, viewer));
1899     else PetscCall(DMPlexView_Ascii(dm, viewer));
1900   } else if (ishdf5) {
1901 #if defined(PETSC_HAVE_HDF5)
1902     PetscCall(DMPlexView_HDF5_Internal(dm, viewer));
1903 #else
1904     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1905 #endif
1906   } else if (isvtk) {
1907     PetscCall(DMPlexVTKWriteAll((PetscObject)dm, viewer));
1908   } else if (isdraw) {
1909     DM hdm;
1910 
1911     PetscCall(DMPlexCreateHighOrderSurrogate_Internal(dm, &hdm));
1912     PetscCall(DMPlexView_Draw(hdm, viewer));
1913     PetscCall(DMDestroy(&hdm));
1914   } else if (isglvis) {
1915     PetscCall(DMPlexView_GLVis(dm, viewer));
1916 #if defined(PETSC_HAVE_EXODUSII)
1917   } else if (isexodus) {
1918     /*
1919       exodusII requires that all sets be part of exactly one cell set.
1920       If the dm does not have a "Cell Sets" label defined, we create one
1921       with ID 1, containing all cells.
1922       Note that if the Cell Sets label is defined but does not cover all cells,
1923       we may still have a problem. This should probably be checked here or in the viewer;
1924     */
1925     PetscInt numCS;
1926     PetscCall(DMGetLabelSize(dm, "Cell Sets", &numCS));
1927     if (!numCS) {
1928       PetscInt cStart, cEnd, c;
1929       PetscCall(DMCreateLabel(dm, "Cell Sets"));
1930       PetscCall(DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd));
1931       for (c = cStart; c < cEnd; ++c) PetscCall(DMSetLabelValue(dm, "Cell Sets", c, 1));
1932     }
1933     PetscCall(DMView_PlexExodusII(dm, viewer));
1934 #endif
1935 #if defined(PETSC_HAVE_CGNS)
1936   } else if (iscgns) {
1937     PetscCall(DMView_PlexCGNS(dm, viewer));
1938 #endif
1939   } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Viewer type %s not yet supported for DMPlex writing", ((PetscObject)viewer)->type_name);
1940   /* Optionally view the partition */
1941   PetscCall(PetscOptionsHasName(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_partition_view", &flg));
1942   if (flg) {
1943     Vec ranks;
1944     PetscCall(DMPlexCreateRankField(dm, &ranks));
1945     PetscCall(VecView(ranks, viewer));
1946     PetscCall(VecDestroy(&ranks));
1947   }
1948   /* Optionally view a label */
1949   PetscCall(PetscOptionsGetString(((PetscObject)dm)->options, ((PetscObject)dm)->prefix, "-dm_label_view", name, sizeof(name), &flg));
1950   if (flg) {
1951     DMLabel label;
1952     Vec     val;
1953 
1954     PetscCall(DMGetLabel(dm, name, &label));
1955     PetscCheck(label, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Label %s provided to -dm_label_view does not exist in this DM", name);
1956     PetscCall(DMPlexCreateLabelField(dm, label, &val));
1957     PetscCall(VecView(val, viewer));
1958     PetscCall(VecDestroy(&val));
1959   }
1960   PetscFunctionReturn(PETSC_SUCCESS);
1961 }
1962 
1963 /*@
1964   DMPlexTopologyView - Saves a `DMPLEX` topology into a file
1965 
1966   Collective
1967 
1968   Input Parameters:
1969 + dm     - The `DM` whose topology is to be saved
1970 - viewer - The `PetscViewer` to save it in
1971 
1972   Level: advanced
1973 
1974 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMView()`, `DMPlexCoordinatesView()`, `DMPlexLabelsView()`, `DMPlexTopologyLoad()`, `PetscViewer`
1975 @*/
1976 PetscErrorCode DMPlexTopologyView(DM dm, PetscViewer viewer)
1977 {
1978   PetscBool ishdf5;
1979 
1980   PetscFunctionBegin;
1981   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1982   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
1983   PetscCall(PetscObjectTypeCompare((PetscObject)viewer, PETSCVIEWERHDF5, &ishdf5));
1984   PetscCall(PetscLogEventBegin(DMPLEX_TopologyView, viewer, 0, 0, 0));
1985   if (ishdf5) {
1986 #if defined(PETSC_HAVE_HDF5)
1987     PetscViewerFormat format;
1988     PetscCall(PetscViewerGetFormat(viewer, &format));
1989     if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
1990       IS globalPointNumbering;
1991 
1992       PetscCall(DMPlexCreatePointNumbering(dm, &globalPointNumbering));
1993       PetscCall(DMPlexTopologyView_HDF5_Internal(dm, globalPointNumbering, viewer));
1994       PetscCall(ISDestroy(&globalPointNumbering));
1995     } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 output.", PetscViewerFormats[format]);
1996 #else
1997     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
1998 #endif
1999   }
2000   PetscCall(PetscLogEventEnd(DMPLEX_TopologyView, viewer, 0, 0, 0));
2001   PetscFunctionReturn(PETSC_SUCCESS);
2002 }
2003 
2004 /*@
2005   DMPlexCoordinatesView - Saves `DMPLEX` coordinates into a file
2006 
2007   Collective
2008 
2009   Input Parameters:
2010 + dm     - The `DM` whose coordinates are to be saved
2011 - viewer - The `PetscViewer` for saving
2012 
2013   Level: advanced
2014 
2015 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMView()`, `DMPlexTopologyView()`, `DMPlexLabelsView()`, `DMPlexCoordinatesLoad()`, `PetscViewer`
2016 @*/
2017 PetscErrorCode DMPlexCoordinatesView(DM dm, PetscViewer viewer)
2018 {
2019   PetscBool ishdf5;
2020 
2021   PetscFunctionBegin;
2022   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2023   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
2024   PetscCall(PetscObjectTypeCompare((PetscObject)viewer, PETSCVIEWERHDF5, &ishdf5));
2025   PetscCall(PetscLogEventBegin(DMPLEX_CoordinatesView, viewer, 0, 0, 0));
2026   if (ishdf5) {
2027 #if defined(PETSC_HAVE_HDF5)
2028     PetscViewerFormat format;
2029     PetscCall(PetscViewerGetFormat(viewer, &format));
2030     if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
2031       PetscCall(DMPlexCoordinatesView_HDF5_Internal(dm, viewer));
2032     } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 output.", PetscViewerFormats[format]);
2033 #else
2034     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
2035 #endif
2036   }
2037   PetscCall(PetscLogEventEnd(DMPLEX_CoordinatesView, viewer, 0, 0, 0));
2038   PetscFunctionReturn(PETSC_SUCCESS);
2039 }
2040 
2041 /*@
2042   DMPlexLabelsView - Saves `DMPLEX` labels into a file
2043 
2044   Collective
2045 
2046   Input Parameters:
2047 + dm     - The `DM` whose labels are to be saved
2048 - viewer - The `PetscViewer` for saving
2049 
2050   Level: advanced
2051 
2052 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMView()`, `DMPlexTopologyView()`, `DMPlexCoordinatesView()`, `DMPlexLabelsLoad()`, `PetscViewer`
2053 @*/
2054 PetscErrorCode DMPlexLabelsView(DM dm, PetscViewer viewer)
2055 {
2056   PetscBool ishdf5;
2057 
2058   PetscFunctionBegin;
2059   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2060   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
2061   PetscCall(PetscObjectTypeCompare((PetscObject)viewer, PETSCVIEWERHDF5, &ishdf5));
2062   PetscCall(PetscLogEventBegin(DMPLEX_LabelsView, viewer, 0, 0, 0));
2063   if (ishdf5) {
2064 #if defined(PETSC_HAVE_HDF5)
2065     IS                globalPointNumbering;
2066     PetscViewerFormat format;
2067 
2068     PetscCall(PetscViewerGetFormat(viewer, &format));
2069     if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
2070       PetscCall(DMPlexCreatePointNumbering(dm, &globalPointNumbering));
2071       PetscCall(DMPlexLabelsView_HDF5_Internal(dm, globalPointNumbering, viewer));
2072       PetscCall(ISDestroy(&globalPointNumbering));
2073     } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 input.", PetscViewerFormats[format]);
2074 #else
2075     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
2076 #endif
2077   }
2078   PetscCall(PetscLogEventEnd(DMPLEX_LabelsView, viewer, 0, 0, 0));
2079   PetscFunctionReturn(PETSC_SUCCESS);
2080 }
2081 
2082 /*@
2083   DMPlexSectionView - Saves a section associated with a `DMPLEX`
2084 
2085   Collective
2086 
2087   Input Parameters:
2088 + dm        - The `DM` that contains the topology on which the section to be saved is defined
2089 . viewer    - The `PetscViewer` for saving
2090 - sectiondm - The `DM` that contains the section to be saved, can be `NULL`
2091 
2092   Level: advanced
2093 
2094   Notes:
2095   This function is a wrapper around `PetscSectionView()`; in addition to the raw section, it saves information that associates the section points to the topology (`dm`) points. When the topology (`dm`) and the section are later loaded with `DMPlexTopologyLoad()` and `DMPlexSectionLoad()`, respectively, this information is used to match section points with topology points.
2096 
2097   In general `dm` and `sectiondm` are two different objects, the former carrying the topology and the latter carrying the section, and have been given a topology name and a section name, respectively, with `PetscObjectSetName()`. In practice, however, they can be the same object (or in case `sectiondm` is `NULL`) if it carries both topology and section; in that case the name of the object is used as both the topology name and the section name.
2098 
2099 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMView()`, `DMPlexTopologyView()`, `DMPlexCoordinatesView()`, `DMPlexLabelsView()`, `DMPlexGlobalVectorView()`, `DMPlexLocalVectorView()`, `PetscSectionView()`, `DMPlexSectionLoad()`, `PetscViewer`
2100 @*/
2101 PetscErrorCode DMPlexSectionView(DM dm, PetscViewer viewer, DM sectiondm)
2102 {
2103   PetscBool ishdf5;
2104 
2105   PetscFunctionBegin;
2106   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2107   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
2108   if (!sectiondm) sectiondm = dm;
2109   PetscValidHeaderSpecific(sectiondm, DM_CLASSID, 3);
2110   PetscCall(PetscObjectTypeCompare((PetscObject)viewer, PETSCVIEWERHDF5, &ishdf5));
2111   PetscCall(PetscLogEventBegin(DMPLEX_SectionView, viewer, 0, 0, 0));
2112   if (ishdf5) {
2113 #if defined(PETSC_HAVE_HDF5)
2114     PetscCall(DMPlexSectionView_HDF5_Internal(dm, viewer, sectiondm));
2115 #else
2116     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
2117 #endif
2118   }
2119   PetscCall(PetscLogEventEnd(DMPLEX_SectionView, viewer, 0, 0, 0));
2120   PetscFunctionReturn(PETSC_SUCCESS);
2121 }
2122 
2123 /*@
2124   DMPlexGlobalVectorView - Saves a global vector
2125 
2126   Collective
2127 
2128   Input Parameters:
2129 + dm        - The `DM` that represents the topology
2130 . viewer    - The `PetscViewer` to save data with
2131 . sectiondm - The `DM` that contains the global section on which vec is defined, can be `NULL`
2132 - vec       - The global vector to be saved
2133 
2134   Level: advanced
2135 
2136   Notes:
2137   In general `dm` and `sectiondm` are two different objects, the former carrying the topology and the latter carrying the section, and have been given a topology name and a section name, respectively, with `PetscObjectSetName()`. In practice, however, they can be the same object (or in case `sectiondm` is `NULL`) if it carries both topology and section; in that case the name of the object is used as both the topology name and the section name.
2138 
2139   Calling sequence:
2140 .vb
2141        DMCreate(PETSC_COMM_WORLD, &dm);
2142        DMSetType(dm, DMPLEX);
2143        PetscObjectSetName((PetscObject)dm, "topologydm_name");
2144        DMClone(dm, &sectiondm);
2145        PetscObjectSetName((PetscObject)sectiondm, "sectiondm_name");
2146        PetscSectionCreate(PETSC_COMM_WORLD, &section);
2147        DMPlexGetChart(sectiondm, &pStart, &pEnd);
2148        PetscSectionSetChart(section, pStart, pEnd);
2149        PetscSectionSetUp(section);
2150        DMSetLocalSection(sectiondm, section);
2151        PetscSectionDestroy(&section);
2152        DMGetGlobalVector(sectiondm, &vec);
2153        PetscObjectSetName((PetscObject)vec, "vec_name");
2154        DMPlexTopologyView(dm, viewer);
2155        DMPlexSectionView(dm, viewer, sectiondm);
2156        DMPlexGlobalVectorView(dm, viewer, sectiondm, vec);
2157        DMRestoreGlobalVector(sectiondm, &vec);
2158        DMDestroy(&sectiondm);
2159        DMDestroy(&dm);
2160 .ve
2161 
2162 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexTopologyView()`, `DMPlexSectionView()`, `DMPlexLocalVectorView()`, `DMPlexGlobalVectorLoad()`, `DMPlexLocalVectorLoad()`
2163 @*/
2164 PetscErrorCode DMPlexGlobalVectorView(DM dm, PetscViewer viewer, DM sectiondm, Vec vec)
2165 {
2166   PetscBool ishdf5;
2167 
2168   PetscFunctionBegin;
2169   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2170   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
2171   if (!sectiondm) sectiondm = dm;
2172   PetscValidHeaderSpecific(sectiondm, DM_CLASSID, 3);
2173   PetscValidHeaderSpecific(vec, VEC_CLASSID, 4);
2174   /* Check consistency */
2175   {
2176     PetscSection section;
2177     PetscBool    includesConstraints;
2178     PetscInt     m, m1;
2179 
2180     PetscCall(VecGetLocalSize(vec, &m1));
2181     PetscCall(DMGetGlobalSection(sectiondm, &section));
2182     PetscCall(PetscSectionGetIncludesConstraints(section, &includesConstraints));
2183     if (includesConstraints) PetscCall(PetscSectionGetStorageSize(section, &m));
2184     else PetscCall(PetscSectionGetConstrainedStorageSize(section, &m));
2185     PetscCheck(m1 == m, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Global vector size (%" PetscInt_FMT ") != global section storage size (%" PetscInt_FMT ")", m1, m);
2186   }
2187   PetscCall(PetscObjectTypeCompare((PetscObject)viewer, PETSCVIEWERHDF5, &ishdf5));
2188   PetscCall(PetscLogEventBegin(DMPLEX_GlobalVectorView, viewer, 0, 0, 0));
2189   if (ishdf5) {
2190 #if defined(PETSC_HAVE_HDF5)
2191     PetscCall(DMPlexGlobalVectorView_HDF5_Internal(dm, viewer, sectiondm, vec));
2192 #else
2193     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
2194 #endif
2195   }
2196   PetscCall(PetscLogEventEnd(DMPLEX_GlobalVectorView, viewer, 0, 0, 0));
2197   PetscFunctionReturn(PETSC_SUCCESS);
2198 }
2199 
2200 /*@
2201   DMPlexLocalVectorView - Saves a local vector
2202 
2203   Collective
2204 
2205   Input Parameters:
2206 + dm        - The `DM` that represents the topology
2207 . viewer    - The `PetscViewer` to save data with
2208 . sectiondm - The `DM` that contains the local section on which `vec` is defined, can be `NULL`
2209 - vec       - The local vector to be saved
2210 
2211   Level: advanced
2212 
2213   Note:
2214   In general `dm` and `sectiondm` are two different objects, the former carrying the topology and the latter carrying the section, and have been given a topology name and a section name, respectively, with `PetscObjectSetName()`. In practice, however, they can be the same object (or in case `sectiondm` is `NULL`) if it carries both topology and section; in that case the name of the object is used as both the topology name and the section name.
2215 
2216   Calling sequence:
2217 .vb
2218        DMCreate(PETSC_COMM_WORLD, &dm);
2219        DMSetType(dm, DMPLEX);
2220        PetscObjectSetName((PetscObject)dm, "topologydm_name");
2221        DMClone(dm, &sectiondm);
2222        PetscObjectSetName((PetscObject)sectiondm, "sectiondm_name");
2223        PetscSectionCreate(PETSC_COMM_WORLD, &section);
2224        DMPlexGetChart(sectiondm, &pStart, &pEnd);
2225        PetscSectionSetChart(section, pStart, pEnd);
2226        PetscSectionSetUp(section);
2227        DMSetLocalSection(sectiondm, section);
2228        DMGetLocalVector(sectiondm, &vec);
2229        PetscObjectSetName((PetscObject)vec, "vec_name");
2230        DMPlexTopologyView(dm, viewer);
2231        DMPlexSectionView(dm, viewer, sectiondm);
2232        DMPlexLocalVectorView(dm, viewer, sectiondm, vec);
2233        DMRestoreLocalVector(sectiondm, &vec);
2234        DMDestroy(&sectiondm);
2235        DMDestroy(&dm);
2236 .ve
2237 
2238 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexTopologyView()`, `DMPlexSectionView()`, `DMPlexGlobalVectorView()`, `DMPlexGlobalVectorLoad()`, `DMPlexLocalVectorLoad()`
2239 @*/
2240 PetscErrorCode DMPlexLocalVectorView(DM dm, PetscViewer viewer, DM sectiondm, Vec vec)
2241 {
2242   PetscBool ishdf5;
2243 
2244   PetscFunctionBegin;
2245   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2246   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
2247   if (!sectiondm) sectiondm = dm;
2248   PetscValidHeaderSpecific(sectiondm, DM_CLASSID, 3);
2249   PetscValidHeaderSpecific(vec, VEC_CLASSID, 4);
2250   /* Check consistency */
2251   {
2252     PetscSection section;
2253     PetscBool    includesConstraints;
2254     PetscInt     m, m1;
2255 
2256     PetscCall(VecGetLocalSize(vec, &m1));
2257     PetscCall(DMGetLocalSection(sectiondm, &section));
2258     PetscCall(PetscSectionGetIncludesConstraints(section, &includesConstraints));
2259     if (includesConstraints) PetscCall(PetscSectionGetStorageSize(section, &m));
2260     else PetscCall(PetscSectionGetConstrainedStorageSize(section, &m));
2261     PetscCheck(m1 == m, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Local vector size (%" PetscInt_FMT ") != local section storage size (%" PetscInt_FMT ")", m1, m);
2262   }
2263   PetscCall(PetscObjectTypeCompare((PetscObject)viewer, PETSCVIEWERHDF5, &ishdf5));
2264   PetscCall(PetscLogEventBegin(DMPLEX_LocalVectorView, viewer, 0, 0, 0));
2265   if (ishdf5) {
2266 #if defined(PETSC_HAVE_HDF5)
2267     PetscCall(DMPlexLocalVectorView_HDF5_Internal(dm, viewer, sectiondm, vec));
2268 #else
2269     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
2270 #endif
2271   }
2272   PetscCall(PetscLogEventEnd(DMPLEX_LocalVectorView, viewer, 0, 0, 0));
2273   PetscFunctionReturn(PETSC_SUCCESS);
2274 }
2275 
2276 PetscErrorCode DMLoad_Plex(DM dm, PetscViewer viewer)
2277 {
2278   PetscBool ishdf5;
2279 
2280   PetscFunctionBegin;
2281   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2282   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
2283   PetscCall(PetscObjectTypeCompare((PetscObject)viewer, PETSCVIEWERHDF5, &ishdf5));
2284   if (ishdf5) {
2285 #if defined(PETSC_HAVE_HDF5)
2286     PetscViewerFormat format;
2287     PetscCall(PetscViewerGetFormat(viewer, &format));
2288     if (format == PETSC_VIEWER_HDF5_XDMF || format == PETSC_VIEWER_HDF5_VIZ) {
2289       PetscCall(DMPlexLoad_HDF5_Xdmf_Internal(dm, viewer));
2290     } else if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
2291       PetscCall(DMPlexLoad_HDF5_Internal(dm, viewer));
2292     } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 input.", PetscViewerFormats[format]);
2293     PetscFunctionReturn(PETSC_SUCCESS);
2294 #else
2295     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
2296 #endif
2297   } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Viewer type %s not yet supported for DMPlex loading", ((PetscObject)viewer)->type_name);
2298 }
2299 
2300 /*@
2301   DMPlexTopologyLoad - Loads a topology into a `DMPLEX`
2302 
2303   Collective
2304 
2305   Input Parameters:
2306 + dm     - The `DM` into which the topology is loaded
2307 - viewer - The `PetscViewer` for the saved topology
2308 
2309   Output Parameter:
2310 . globalToLocalPointSF - The `PetscSF` that pushes points in [0, N) to the associated points in the loaded `DMPLEX`, where N is the global number of points;
2311   `NULL` if unneeded
2312 
2313   Level: advanced
2314 
2315 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMLoad()`, `DMPlexCoordinatesLoad()`, `DMPlexLabelsLoad()`, `DMView()`, `PetscViewerHDF5Open()`, `PetscViewerPushFormat()`,
2316           `PetscViewer`, `PetscSF`
2317 @*/
2318 PetscErrorCode DMPlexTopologyLoad(DM dm, PetscViewer viewer, PetscSF *globalToLocalPointSF)
2319 {
2320   PetscBool ishdf5;
2321 
2322   PetscFunctionBegin;
2323   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2324   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
2325   if (globalToLocalPointSF) PetscAssertPointer(globalToLocalPointSF, 3);
2326   PetscCall(PetscObjectTypeCompare((PetscObject)viewer, PETSCVIEWERHDF5, &ishdf5));
2327   PetscCall(PetscLogEventBegin(DMPLEX_TopologyLoad, viewer, 0, 0, 0));
2328   if (ishdf5) {
2329 #if defined(PETSC_HAVE_HDF5)
2330     PetscViewerFormat format;
2331     PetscCall(PetscViewerGetFormat(viewer, &format));
2332     if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
2333       PetscCall(DMPlexTopologyLoad_HDF5_Internal(dm, viewer, globalToLocalPointSF));
2334     } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 input.", PetscViewerFormats[format]);
2335 #else
2336     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
2337 #endif
2338   }
2339   PetscCall(PetscLogEventEnd(DMPLEX_TopologyLoad, viewer, 0, 0, 0));
2340   PetscFunctionReturn(PETSC_SUCCESS);
2341 }
2342 
2343 /*@
2344   DMPlexCoordinatesLoad - Loads coordinates into a `DMPLEX`
2345 
2346   Collective
2347 
2348   Input Parameters:
2349 + dm                   - The `DM` into which the coordinates are loaded
2350 . viewer               - The `PetscViewer` for the saved coordinates
2351 - globalToLocalPointSF - The `PetscSF` returned by `DMPlexTopologyLoad()` when loading dm from viewer
2352 
2353   Level: advanced
2354 
2355 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMLoad()`, `DMPlexTopologyLoad()`, `DMPlexLabelsLoad()`, `DMView()`, `PetscViewerHDF5Open()`, `PetscViewerPushFormat()`,
2356           `PetscSF`, `PetscViewer`
2357 @*/
2358 PetscErrorCode DMPlexCoordinatesLoad(DM dm, PetscViewer viewer, PetscSF globalToLocalPointSF)
2359 {
2360   PetscBool ishdf5;
2361 
2362   PetscFunctionBegin;
2363   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2364   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
2365   PetscValidHeaderSpecific(globalToLocalPointSF, PETSCSF_CLASSID, 3);
2366   PetscCall(PetscObjectTypeCompare((PetscObject)viewer, PETSCVIEWERHDF5, &ishdf5));
2367   PetscCall(PetscLogEventBegin(DMPLEX_CoordinatesLoad, viewer, 0, 0, 0));
2368   if (ishdf5) {
2369 #if defined(PETSC_HAVE_HDF5)
2370     PetscViewerFormat format;
2371     PetscCall(PetscViewerGetFormat(viewer, &format));
2372     if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
2373       PetscCall(DMPlexCoordinatesLoad_HDF5_Internal(dm, viewer, globalToLocalPointSF));
2374     } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 input.", PetscViewerFormats[format]);
2375 #else
2376     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
2377 #endif
2378   }
2379   PetscCall(PetscLogEventEnd(DMPLEX_CoordinatesLoad, viewer, 0, 0, 0));
2380   PetscFunctionReturn(PETSC_SUCCESS);
2381 }
2382 
2383 /*@
2384   DMPlexLabelsLoad - Loads labels into a `DMPLEX`
2385 
2386   Collective
2387 
2388   Input Parameters:
2389 + dm                   - The `DM` into which the labels are loaded
2390 . viewer               - The `PetscViewer` for the saved labels
2391 - globalToLocalPointSF - The `PetscSF` returned by `DMPlexTopologyLoad()` when loading `dm` from viewer
2392 
2393   Level: advanced
2394 
2395   Note:
2396   The `PetscSF` argument must not be `NULL` if the `DM` is distributed, otherwise an error occurs.
2397 
2398 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMLoad()`, `DMPlexTopologyLoad()`, `DMPlexCoordinatesLoad()`, `DMView()`, `PetscViewerHDF5Open()`, `PetscViewerPushFormat()`,
2399           `PetscSF`, `PetscViewer`
2400 @*/
2401 PetscErrorCode DMPlexLabelsLoad(DM dm, PetscViewer viewer, PetscSF globalToLocalPointSF)
2402 {
2403   PetscBool ishdf5;
2404 
2405   PetscFunctionBegin;
2406   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2407   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
2408   if (globalToLocalPointSF) PetscValidHeaderSpecific(globalToLocalPointSF, PETSCSF_CLASSID, 3);
2409   PetscCall(PetscObjectTypeCompare((PetscObject)viewer, PETSCVIEWERHDF5, &ishdf5));
2410   PetscCall(PetscLogEventBegin(DMPLEX_LabelsLoad, viewer, 0, 0, 0));
2411   if (ishdf5) {
2412 #if defined(PETSC_HAVE_HDF5)
2413     PetscViewerFormat format;
2414 
2415     PetscCall(PetscViewerGetFormat(viewer, &format));
2416     if (format == PETSC_VIEWER_HDF5_PETSC || format == PETSC_VIEWER_DEFAULT || format == PETSC_VIEWER_NATIVE) {
2417       PetscCall(DMPlexLabelsLoad_HDF5_Internal(dm, viewer, globalToLocalPointSF));
2418     } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "PetscViewerFormat %s not supported for HDF5 input.", PetscViewerFormats[format]);
2419 #else
2420     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
2421 #endif
2422   }
2423   PetscCall(PetscLogEventEnd(DMPLEX_LabelsLoad, viewer, 0, 0, 0));
2424   PetscFunctionReturn(PETSC_SUCCESS);
2425 }
2426 
2427 /*@
2428   DMPlexSectionLoad - Loads section into a `DMPLEX`
2429 
2430   Collective
2431 
2432   Input Parameters:
2433 + dm                   - The `DM` that represents the topology
2434 . viewer               - The `PetscViewer` that represents the on-disk section (sectionA)
2435 . sectiondm            - The `DM` into which the on-disk section (sectionA) is migrated, can be `NULL`
2436 - globalToLocalPointSF - The `PetscSF` returned by `DMPlexTopologyLoad(`) when loading dm from viewer
2437 
2438   Output Parameters:
2439 + globalDofSF - The `PetscSF` that migrates any on-disk `Vec` data associated with sectionA into a global `Vec` associated with the `sectiondm`'s global section (`NULL` if not needed)
2440 - localDofSF  - The `PetscSF` that migrates any on-disk `Vec` data associated with sectionA into a local `Vec` associated with the `sectiondm`'s local section (`NULL` if not needed)
2441 
2442   Level: advanced
2443 
2444   Notes:
2445   This function is a wrapper around `PetscSectionLoad()`; it loads, in addition to the raw section, a list of global point numbers that associates each on-disk section point with a global point number in [0, NX), where NX is the number of topology points in `dm`. Noting that globalToLocalPointSF associates each topology point in dm with a global number in [0, NX), one can readily establish an association of the on-disk section points with the topology points.
2446 
2447   In general `dm` and `sectiondm` are two different objects, the former carrying the topology and the latter carrying the section, and have been given a topology name and a section name, respectively, with `PetscObjectSetName()`. In practice, however, they can be the same object (or in case `sectiondm` is `NULL`) if it carries both topology and section; in that case the name of the object is used as both the topology name and the section name.
2448 
2449   The output parameter, `globalDofSF` (`localDofSF`), can later be used with `DMPlexGlobalVectorLoad()` (`DMPlexLocalVectorLoad()`) to load on-disk vectors into global (local) vectors associated with sectiondm's global (local) section.
2450 
2451   Example using 2 processes:
2452 .vb
2453   NX (number of points on dm): 4
2454   sectionA                   : the on-disk section
2455   vecA                       : a vector associated with sectionA
2456   sectionB                   : sectiondm's local section constructed in this function
2457   vecB (local)               : a vector associated with sectiondm's local section
2458   vecB (global)              : a vector associated with sectiondm's global section
2459 
2460                                      rank 0    rank 1
2461   vecA (global)                  : [.0 .4 .1 | .2 .3]        <- to be loaded in DMPlexGlobalVectorLoad() or DMPlexLocalVectorLoad()
2462   sectionA->atlasOff             :       0 2 | 1             <- loaded in PetscSectionLoad()
2463   sectionA->atlasDof             :       1 3 | 1             <- loaded in PetscSectionLoad()
2464   sectionA's global point numbers:       0 2 | 3             <- loaded in DMPlexSectionLoad()
2465   [0, NX)                        :       0 1 | 2 3           <- conceptual partition used in globalToLocalPointSF
2466   sectionB's global point numbers:     0 1 3 | 3 2           <- associated with [0, NX) by globalToLocalPointSF
2467   sectionB->atlasDof             :     1 0 1 | 1 3
2468   sectionB->atlasOff (no perm)   :     0 1 1 | 0 1
2469   vecB (local)                   :   [.0 .4] | [.4 .1 .2 .3] <- to be constructed by calling DMPlexLocalVectorLoad() with localDofSF
2470   vecB (global)                  :    [.0 .4 | .1 .2 .3]     <- to be constructed by calling DMPlexGlobalVectorLoad() with globalDofSF
2471 .ve
2472   where "|" represents a partition of loaded data, and global point 3 is assumed to be owned by rank 0.
2473 
2474 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMLoad()`, `DMPlexTopologyLoad()`, `DMPlexCoordinatesLoad()`, `DMPlexLabelsLoad()`, `DMPlexGlobalVectorLoad()`, `DMPlexLocalVectorLoad()`, `PetscSectionLoad()`, `DMPlexSectionView()`, `PetscSF`, `PetscViewer`
2475 @*/
2476 PetscErrorCode DMPlexSectionLoad(DM dm, PetscViewer viewer, DM sectiondm, PetscSF globalToLocalPointSF, PetscSF *globalDofSF, PetscSF *localDofSF)
2477 {
2478   PetscBool ishdf5;
2479 
2480   PetscFunctionBegin;
2481   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2482   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
2483   if (!sectiondm) sectiondm = dm;
2484   PetscValidHeaderSpecific(sectiondm, DM_CLASSID, 3);
2485   PetscValidHeaderSpecific(globalToLocalPointSF, PETSCSF_CLASSID, 4);
2486   if (globalDofSF) PetscAssertPointer(globalDofSF, 5);
2487   if (localDofSF) PetscAssertPointer(localDofSF, 6);
2488   PetscCall(PetscObjectTypeCompare((PetscObject)viewer, PETSCVIEWERHDF5, &ishdf5));
2489   PetscCall(PetscLogEventBegin(DMPLEX_SectionLoad, viewer, 0, 0, 0));
2490   if (ishdf5) {
2491 #if defined(PETSC_HAVE_HDF5)
2492     PetscCall(DMPlexSectionLoad_HDF5_Internal(dm, viewer, sectiondm, globalToLocalPointSF, globalDofSF, localDofSF));
2493 #else
2494     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
2495 #endif
2496   }
2497   PetscCall(PetscLogEventEnd(DMPLEX_SectionLoad, viewer, 0, 0, 0));
2498   PetscFunctionReturn(PETSC_SUCCESS);
2499 }
2500 
2501 /*@
2502   DMPlexGlobalVectorLoad - Loads on-disk vector data into a global vector
2503 
2504   Collective
2505 
2506   Input Parameters:
2507 + dm        - The `DM` that represents the topology
2508 . viewer    - The `PetscViewer` that represents the on-disk vector data
2509 . sectiondm - The `DM` that contains the global section on which vec is defined, can be `NULL`
2510 . sf        - The `PetscSF` that migrates the on-disk vector data into vec
2511 - vec       - The global vector to set values of
2512 
2513   Level: advanced
2514 
2515   Notes:
2516   In general dm and sectiondm are two different objects, the former carrying the topology and the latter carrying the section, and have been given a topology name and a section name, respectively, with `PetscObjectSetName()`. In practice, however, they can be the same object (or in case `sectiondm` is `NULL`) if it carries both topology and section; in that case the name of the object is used as both the topology name and the section name.
2517 
2518   Calling sequence:
2519 .vb
2520        DMCreate(PETSC_COMM_WORLD, &dm);
2521        DMSetType(dm, DMPLEX);
2522        PetscObjectSetName((PetscObject)dm, "topologydm_name");
2523        DMPlexTopologyLoad(dm, viewer, &sfX);
2524        DMClone(dm, &sectiondm);
2525        PetscObjectSetName((PetscObject)sectiondm, "sectiondm_name");
2526        DMPlexSectionLoad(dm, viewer, sectiondm, sfX, &gsf, NULL);
2527        DMGetGlobalVector(sectiondm, &vec);
2528        PetscObjectSetName((PetscObject)vec, "vec_name");
2529        DMPlexGlobalVectorLoad(dm, viewer, sectiondm, gsf, vec);
2530        DMRestoreGlobalVector(sectiondm, &vec);
2531        PetscSFDestroy(&gsf);
2532        PetscSFDestroy(&sfX);
2533        DMDestroy(&sectiondm);
2534        DMDestroy(&dm);
2535 .ve
2536 
2537 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexTopologyLoad()`, `DMPlexSectionLoad()`, `DMPlexLocalVectorLoad()`, `DMPlexGlobalVectorView()`, `DMPlexLocalVectorView()`,
2538           `PetscSF`, `PetscViewer`
2539 @*/
2540 PetscErrorCode DMPlexGlobalVectorLoad(DM dm, PetscViewer viewer, DM sectiondm, PetscSF sf, Vec vec)
2541 {
2542   PetscBool ishdf5;
2543 
2544   PetscFunctionBegin;
2545   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2546   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
2547   if (!sectiondm) sectiondm = dm;
2548   PetscValidHeaderSpecific(sectiondm, DM_CLASSID, 3);
2549   PetscValidHeaderSpecific(sf, PETSCSF_CLASSID, 4);
2550   PetscValidHeaderSpecific(vec, VEC_CLASSID, 5);
2551   /* Check consistency */
2552   {
2553     PetscSection section;
2554     PetscBool    includesConstraints;
2555     PetscInt     m, m1;
2556 
2557     PetscCall(VecGetLocalSize(vec, &m1));
2558     PetscCall(DMGetGlobalSection(sectiondm, &section));
2559     PetscCall(PetscSectionGetIncludesConstraints(section, &includesConstraints));
2560     if (includesConstraints) PetscCall(PetscSectionGetStorageSize(section, &m));
2561     else PetscCall(PetscSectionGetConstrainedStorageSize(section, &m));
2562     PetscCheck(m1 == m, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Global vector size (%" PetscInt_FMT ") != global section storage size (%" PetscInt_FMT ")", m1, m);
2563   }
2564   PetscCall(PetscObjectTypeCompare((PetscObject)viewer, PETSCVIEWERHDF5, &ishdf5));
2565   PetscCall(PetscLogEventBegin(DMPLEX_GlobalVectorLoad, viewer, 0, 0, 0));
2566   if (ishdf5) {
2567 #if defined(PETSC_HAVE_HDF5)
2568     PetscCall(DMPlexVecLoad_HDF5_Internal(dm, viewer, sectiondm, sf, vec));
2569 #else
2570     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
2571 #endif
2572   }
2573   PetscCall(PetscLogEventEnd(DMPLEX_GlobalVectorLoad, viewer, 0, 0, 0));
2574   PetscFunctionReturn(PETSC_SUCCESS);
2575 }
2576 
2577 /*@
2578   DMPlexLocalVectorLoad - Loads on-disk vector data into a local vector
2579 
2580   Collective
2581 
2582   Input Parameters:
2583 + dm        - The `DM` that represents the topology
2584 . viewer    - The `PetscViewer` that represents the on-disk vector data
2585 . sectiondm - The `DM` that contains the local section on which vec is defined, can be `NULL`
2586 . sf        - The `PetscSF` that migrates the on-disk vector data into vec
2587 - vec       - The local vector to set values of
2588 
2589   Level: advanced
2590 
2591   Notes:
2592   In general `dm` and `sectiondm` are two different objects, the former carrying the topology and the latter carrying the section, and have been given a topology name and a section name, respectively, with `PetscObjectSetName()`. In practice, however, they can be the same object (or in case `sectiondm` is `NULL`) if it carries both topology and section; in that case the name of the object is used as both the topology name and the section name.
2593 
2594   Calling sequence:
2595 .vb
2596        DMCreate(PETSC_COMM_WORLD, &dm);
2597        DMSetType(dm, DMPLEX);
2598        PetscObjectSetName((PetscObject)dm, "topologydm_name");
2599        DMPlexTopologyLoad(dm, viewer, &sfX);
2600        DMClone(dm, &sectiondm);
2601        PetscObjectSetName((PetscObject)sectiondm, "sectiondm_name");
2602        DMPlexSectionLoad(dm, viewer, sectiondm, sfX, NULL, &lsf);
2603        DMGetLocalVector(sectiondm, &vec);
2604        PetscObjectSetName((PetscObject)vec, "vec_name");
2605        DMPlexLocalVectorLoad(dm, viewer, sectiondm, lsf, vec);
2606        DMRestoreLocalVector(sectiondm, &vec);
2607        PetscSFDestroy(&lsf);
2608        PetscSFDestroy(&sfX);
2609        DMDestroy(&sectiondm);
2610        DMDestroy(&dm);
2611 .ve
2612 
2613 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexTopologyLoad()`, `DMPlexSectionLoad()`, `DMPlexGlobalVectorLoad()`, `DMPlexGlobalVectorView()`, `DMPlexLocalVectorView()`,
2614           `PetscSF`, `PetscViewer`
2615 @*/
2616 PetscErrorCode DMPlexLocalVectorLoad(DM dm, PetscViewer viewer, DM sectiondm, PetscSF sf, Vec vec)
2617 {
2618   PetscBool ishdf5;
2619 
2620   PetscFunctionBegin;
2621   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2622   PetscValidHeaderSpecific(viewer, PETSC_VIEWER_CLASSID, 2);
2623   if (!sectiondm) sectiondm = dm;
2624   PetscValidHeaderSpecific(sectiondm, DM_CLASSID, 3);
2625   PetscValidHeaderSpecific(sf, PETSCSF_CLASSID, 4);
2626   PetscValidHeaderSpecific(vec, VEC_CLASSID, 5);
2627   /* Check consistency */
2628   {
2629     PetscSection section;
2630     PetscBool    includesConstraints;
2631     PetscInt     m, m1;
2632 
2633     PetscCall(VecGetLocalSize(vec, &m1));
2634     PetscCall(DMGetLocalSection(sectiondm, &section));
2635     PetscCall(PetscSectionGetIncludesConstraints(section, &includesConstraints));
2636     if (includesConstraints) PetscCall(PetscSectionGetStorageSize(section, &m));
2637     else PetscCall(PetscSectionGetConstrainedStorageSize(section, &m));
2638     PetscCheck(m1 == m, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Local vector size (%" PetscInt_FMT ") != local section storage size (%" PetscInt_FMT ")", m1, m);
2639   }
2640   PetscCall(PetscObjectTypeCompare((PetscObject)viewer, PETSCVIEWERHDF5, &ishdf5));
2641   PetscCall(PetscLogEventBegin(DMPLEX_LocalVectorLoad, viewer, 0, 0, 0));
2642   if (ishdf5) {
2643 #if defined(PETSC_HAVE_HDF5)
2644     PetscCall(DMPlexVecLoad_HDF5_Internal(dm, viewer, sectiondm, sf, vec));
2645 #else
2646     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
2647 #endif
2648   }
2649   PetscCall(PetscLogEventEnd(DMPLEX_LocalVectorLoad, viewer, 0, 0, 0));
2650   PetscFunctionReturn(PETSC_SUCCESS);
2651 }
2652 
2653 PetscErrorCode DMDestroy_Plex(DM dm)
2654 {
2655   DM_Plex *mesh = (DM_Plex *)dm->data;
2656 
2657   PetscFunctionBegin;
2658   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMSetUpGLVisViewer_C", NULL));
2659   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexInsertBoundaryValues_C", NULL));
2660   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMCreateNeumannOverlap_C", NULL));
2661   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMInterpolateSolution_C", NULL));
2662   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexInsertTimeDerivativeBoundaryValues_C", NULL));
2663   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexGetOverlap_C", NULL));
2664   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexDistributeGetDefault_C", NULL));
2665   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexDistributeSetDefault_C", NULL));
2666   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "MatComputeNeumannOverlap_C", NULL));
2667   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexReorderGetDefault_C", NULL));
2668   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexReorderSetDefault_C", NULL));
2669   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMReorderSectionGetDefault_C", NULL));
2670   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMReorderSectionSetDefault_C", NULL));
2671   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMReorderSectionGetType_C", NULL));
2672   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMReorderSectionSetType_C", NULL));
2673   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexGetOverlap_C", NULL));
2674   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexSetOverlap_C", NULL));
2675   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexGetUseCeed_C", NULL));
2676   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMPlexSetUseCeed_C", NULL));
2677   PetscCall(PetscObjectComposeFunction((PetscObject)dm, "DMGetIsoperiodicPointSF_C", NULL));
2678   if (--mesh->refct > 0) PetscFunctionReturn(PETSC_SUCCESS);
2679   PetscCall(PetscSectionDestroy(&mesh->coneSection));
2680   PetscCall(PetscFree(mesh->cones));
2681   PetscCall(PetscFree(mesh->coneOrientations));
2682   PetscCall(PetscSectionDestroy(&mesh->supportSection));
2683   PetscCall(PetscSectionDestroy(&mesh->subdomainSection));
2684   PetscCall(PetscFree(mesh->supports));
2685   PetscCall(PetscFree(mesh->cellTypes));
2686   PetscCall(DMPlexTransformDestroy(&mesh->tr));
2687   PetscCall(PetscFree(mesh->tetgenOpts));
2688   PetscCall(PetscFree(mesh->triangleOpts));
2689   PetscCall(PetscFree(mesh->transformType));
2690   PetscCall(PetscFree(mesh->distributionName));
2691   PetscCall(PetscPartitionerDestroy(&mesh->partitioner));
2692   PetscCall(DMLabelDestroy(&mesh->subpointMap));
2693   PetscCall(ISDestroy(&mesh->subpointIS));
2694   PetscCall(ISDestroy(&mesh->globalVertexNumbers));
2695   PetscCall(ISDestroy(&mesh->globalCellNumbers));
2696   if (mesh->periodic.face_sfs) {
2697     for (PetscInt i = 0; i < mesh->periodic.num_face_sfs; i++) PetscCall(PetscSFDestroy(&mesh->periodic.face_sfs[i]));
2698     PetscCall(PetscFree(mesh->periodic.face_sfs));
2699   }
2700   PetscCall(PetscSFDestroy(&mesh->periodic.composed_sf));
2701   if (mesh->periodic.periodic_points) {
2702     for (PetscInt i = 0; i < mesh->periodic.num_face_sfs; i++) PetscCall(ISDestroy(&mesh->periodic.periodic_points[i]));
2703     PetscCall(PetscFree(mesh->periodic.periodic_points));
2704   }
2705   if (mesh->periodic.transform) PetscCall(PetscFree(mesh->periodic.transform));
2706   PetscCall(PetscSectionDestroy(&mesh->anchorSection));
2707   PetscCall(ISDestroy(&mesh->anchorIS));
2708   PetscCall(PetscSectionDestroy(&mesh->parentSection));
2709   PetscCall(PetscFree(mesh->parents));
2710   PetscCall(PetscFree(mesh->childIDs));
2711   PetscCall(PetscSectionDestroy(&mesh->childSection));
2712   PetscCall(PetscFree(mesh->children));
2713   PetscCall(DMDestroy(&mesh->referenceTree));
2714   PetscCall(PetscGridHashDestroy(&mesh->lbox));
2715   PetscCall(PetscFree(mesh->neighbors));
2716   if (mesh->metricCtx) PetscCall(PetscFree(mesh->metricCtx));
2717   if (mesh->nonempty_comm != MPI_COMM_NULL && mesh->nonempty_comm != MPI_COMM_SELF) PetscCallMPI(MPI_Comm_free(&mesh->nonempty_comm));
2718   /* This was originally freed in DMDestroy(), but that prevents reference counting of backend objects */
2719   PetscCall(PetscFree(mesh));
2720   PetscFunctionReturn(PETSC_SUCCESS);
2721 }
2722 
2723 PetscErrorCode DMCreateMatrix_Plex(DM dm, Mat *J)
2724 {
2725   PetscSection           sectionGlobal, sectionLocal;
2726   PetscInt               bs = -1, mbs;
2727   PetscInt               localSize, localStart = 0;
2728   PetscBool              isShell, isBlock, isSeqBlock, isMPIBlock, isSymBlock, isSymSeqBlock, isSymMPIBlock, isMatIS;
2729   MatType                mtype;
2730   ISLocalToGlobalMapping ltog;
2731 
2732   PetscFunctionBegin;
2733   PetscCall(MatInitializePackage());
2734   mtype = dm->mattype;
2735   PetscCall(DMGetLocalSection(dm, &sectionLocal));
2736   PetscCall(DMGetGlobalSection(dm, &sectionGlobal));
2737   /* PetscCall(PetscSectionGetStorageSize(sectionGlobal, &localSize)); */
2738   PetscCall(PetscSectionGetConstrainedStorageSize(sectionGlobal, &localSize));
2739   PetscCallMPI(MPI_Exscan(&localSize, &localStart, 1, MPIU_INT, MPI_SUM, PetscObjectComm((PetscObject)dm)));
2740   PetscCall(MatCreate(PetscObjectComm((PetscObject)dm), J));
2741   PetscCall(MatSetSizes(*J, localSize, localSize, PETSC_DETERMINE, PETSC_DETERMINE));
2742   PetscCall(MatSetType(*J, mtype));
2743   PetscCall(MatSetFromOptions(*J));
2744   PetscCall(MatGetBlockSize(*J, &mbs));
2745   if (mbs > 1) bs = mbs;
2746   PetscCall(PetscStrcmp(mtype, MATSHELL, &isShell));
2747   PetscCall(PetscStrcmp(mtype, MATBAIJ, &isBlock));
2748   PetscCall(PetscStrcmp(mtype, MATSEQBAIJ, &isSeqBlock));
2749   PetscCall(PetscStrcmp(mtype, MATMPIBAIJ, &isMPIBlock));
2750   PetscCall(PetscStrcmp(mtype, MATSBAIJ, &isSymBlock));
2751   PetscCall(PetscStrcmp(mtype, MATSEQSBAIJ, &isSymSeqBlock));
2752   PetscCall(PetscStrcmp(mtype, MATMPISBAIJ, &isSymMPIBlock));
2753   PetscCall(PetscStrcmp(mtype, MATIS, &isMatIS));
2754   if (!isShell) {
2755     // There are three states with pblocks, since block starts can have no dofs:
2756     // UNKNOWN) New Block:   An open block has been signalled by pblocks[p] == 1
2757     // TRUE)    Block Start: The first entry in a block has been added
2758     // FALSE)   Block Add:   An additional block entry has been added, since pblocks[p] == 0
2759     PetscBT         blst;
2760     PetscBool3      bstate     = PETSC_BOOL3_UNKNOWN;
2761     PetscBool       fillMatrix = (PetscBool)(!dm->prealloc_only && !isMatIS);
2762     const PetscInt *perm       = NULL;
2763     PetscInt       *dnz, *onz, *dnzu, *onzu, bsLocal[2], bsMinMax[2], *pblocks;
2764     PetscInt        pStart, pEnd, dof, cdof, num_fields;
2765 
2766     PetscCall(DMGetLocalToGlobalMapping(dm, &ltog));
2767     PetscCall(PetscSectionGetBlockStarts(sectionLocal, &blst));
2768     if (sectionLocal->perm) PetscCall(ISGetIndices(sectionLocal->perm, &perm));
2769 
2770     PetscCall(PetscCalloc1(localSize, &pblocks));
2771     PetscCall(PetscSectionGetChart(sectionGlobal, &pStart, &pEnd));
2772     PetscCall(PetscSectionGetNumFields(sectionGlobal, &num_fields));
2773     // We need to process in the permuted order to get block sizes right
2774     for (PetscInt point = pStart; point < pEnd; ++point) {
2775       const PetscInt p = perm ? perm[point] : point;
2776 
2777       switch (dm->blocking_type) {
2778       case DM_BLOCKING_TOPOLOGICAL_POINT: { // One block per topological point
2779         PetscInt bdof, offset;
2780 
2781         PetscCall(PetscSectionGetDof(sectionGlobal, p, &dof));
2782         PetscCall(PetscSectionGetOffset(sectionGlobal, p, &offset));
2783         PetscCall(PetscSectionGetConstraintDof(sectionGlobal, p, &cdof));
2784         if (blst && PetscBTLookup(blst, p)) bstate = PETSC_BOOL3_UNKNOWN;
2785         if (dof > 0) {
2786           // State change
2787           if (bstate == PETSC_BOOL3_UNKNOWN) bstate = PETSC_BOOL3_TRUE;
2788           else if (bstate == PETSC_BOOL3_TRUE && blst && !PetscBTLookup(blst, p)) bstate = PETSC_BOOL3_FALSE;
2789 
2790           for (PetscInt i = 0; i < dof - cdof; ++i) pblocks[offset - localStart + i] = dof - cdof;
2791           // Signal block concatenation
2792           if (bstate == PETSC_BOOL3_FALSE && dof - cdof) pblocks[offset - localStart] = -(dof - cdof);
2793         }
2794         dof  = dof < 0 ? -(dof + 1) : dof;
2795         bdof = cdof && (dof - cdof) ? 1 : dof;
2796         if (dof) {
2797           if (bs < 0) {
2798             bs = bdof;
2799           } else if (bs != bdof) {
2800             bs = 1;
2801           }
2802         }
2803       } break;
2804       case DM_BLOCKING_FIELD_NODE: {
2805         for (PetscInt field = 0; field < num_fields; field++) {
2806           PetscInt num_comp, bdof, offset;
2807           PetscCall(PetscSectionGetFieldComponents(sectionGlobal, field, &num_comp));
2808           PetscCall(PetscSectionGetFieldDof(sectionGlobal, p, field, &dof));
2809           if (dof < 0) continue;
2810           PetscCall(PetscSectionGetFieldOffset(sectionGlobal, p, field, &offset));
2811           PetscCall(PetscSectionGetFieldConstraintDof(sectionGlobal, p, field, &cdof));
2812           PetscAssert(dof % num_comp == 0, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "Point %" PetscInt_FMT " field %" PetscInt_FMT " has %" PetscInt_FMT " dof, not divisible by %" PetscInt_FMT " component ", p, field, dof, num_comp);
2813           PetscInt num_nodes = dof / num_comp;
2814           for (PetscInt i = 0; i < dof - cdof; i++) pblocks[offset - localStart + i] = (dof - cdof) / num_nodes;
2815           // Handle possibly constant block size (unlikely)
2816           bdof = cdof && (dof - cdof) ? 1 : dof;
2817           if (dof) {
2818             if (bs < 0) {
2819               bs = bdof;
2820             } else if (bs != bdof) {
2821               bs = 1;
2822             }
2823           }
2824         }
2825       } break;
2826       }
2827     }
2828     if (sectionLocal->perm) PetscCall(ISRestoreIndices(sectionLocal->perm, &perm));
2829     /* Must have same blocksize on all procs (some might have no points) */
2830     bsLocal[0] = bs < 0 ? PETSC_INT_MAX : bs;
2831     bsLocal[1] = bs;
2832     PetscCall(PetscGlobalMinMaxInt(PetscObjectComm((PetscObject)dm), bsLocal, bsMinMax));
2833     if (bsMinMax[0] != bsMinMax[1]) bs = 1;
2834     else bs = bsMinMax[0];
2835     bs = PetscMax(1, bs);
2836     PetscCall(MatSetLocalToGlobalMapping(*J, ltog, ltog));
2837     if (dm->prealloc_skip) { // User will likely use MatSetPreallocationCOO(), but still set structural parameters
2838       PetscCall(MatSetBlockSize(*J, bs));
2839       PetscCall(MatSetUp(*J));
2840     } else {
2841       PetscCall(PetscCalloc4(localSize / bs, &dnz, localSize / bs, &onz, localSize / bs, &dnzu, localSize / bs, &onzu));
2842       PetscCall(DMPlexPreallocateOperator(dm, bs, dnz, onz, dnzu, onzu, *J, fillMatrix));
2843       PetscCall(PetscFree4(dnz, onz, dnzu, onzu));
2844     }
2845     if (pblocks) { // Consolidate blocks
2846       PetscInt nblocks = 0;
2847       pblocks[0]       = PetscAbs(pblocks[0]);
2848       for (PetscInt i = 0; i < localSize; i += PetscMax(1, pblocks[i])) {
2849         if (pblocks[i] == 0) continue;
2850         // Negative block size indicates the blocks should be concatenated
2851         if (pblocks[i] < 0) {
2852           pblocks[i] = -pblocks[i];
2853           pblocks[nblocks - 1] += pblocks[i];
2854         } else {
2855           pblocks[nblocks++] = pblocks[i]; // nblocks always <= i
2856         }
2857         for (PetscInt j = 1; j < pblocks[i]; j++)
2858           PetscCheck(pblocks[i + j] == pblocks[i], PETSC_COMM_SELF, PETSC_ERR_PLIB, "Block of size %" PetscInt_FMT " at %" PetscInt_FMT " mismatches entry %" PetscInt_FMT " at %" PetscInt_FMT, pblocks[i], i, pblocks[i + j], i + j);
2859       }
2860       PetscCall(MatSetVariableBlockSizes(*J, nblocks, pblocks));
2861     }
2862     PetscCall(PetscFree(pblocks));
2863   }
2864   PetscCall(MatSetDM(*J, dm));
2865   PetscFunctionReturn(PETSC_SUCCESS);
2866 }
2867 
2868 /*@
2869   DMPlexGetSubdomainSection - Returns the section associated with the subdomain
2870 
2871   Not Collective
2872 
2873   Input Parameter:
2874 . dm - The `DMPLEX`
2875 
2876   Output Parameter:
2877 . subsection - The subdomain section
2878 
2879   Level: developer
2880 
2881 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `PetscSection`
2882 @*/
2883 PetscErrorCode DMPlexGetSubdomainSection(DM dm, PetscSection *subsection)
2884 {
2885   DM_Plex *mesh = (DM_Plex *)dm->data;
2886 
2887   PetscFunctionBegin;
2888   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2889   if (!mesh->subdomainSection) {
2890     PetscSection section;
2891     PetscSF      sf;
2892 
2893     PetscCall(PetscSFCreate(PETSC_COMM_SELF, &sf));
2894     PetscCall(DMGetLocalSection(dm, &section));
2895     PetscCall(PetscSectionCreateGlobalSection(section, sf, PETSC_TRUE, PETSC_FALSE, PETSC_TRUE, &mesh->subdomainSection));
2896     PetscCall(PetscSFDestroy(&sf));
2897   }
2898   *subsection = mesh->subdomainSection;
2899   PetscFunctionReturn(PETSC_SUCCESS);
2900 }
2901 
2902 /*@
2903   DMPlexGetChart - Return the interval for all mesh points [`pStart`, `pEnd`)
2904 
2905   Not Collective
2906 
2907   Input Parameter:
2908 . dm - The `DMPLEX`
2909 
2910   Output Parameters:
2911 + pStart - The first mesh point
2912 - pEnd   - The upper bound for mesh points
2913 
2914   Level: beginner
2915 
2916 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreate()`, `DMPlexSetChart()`
2917 @*/
2918 PetscErrorCode DMPlexGetChart(DM dm, PetscInt *pStart, PetscInt *pEnd)
2919 {
2920   DM_Plex *mesh = (DM_Plex *)dm->data;
2921 
2922   PetscFunctionBegin;
2923   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2924   if (mesh->tr) PetscCall(DMPlexTransformGetChart(mesh->tr, pStart, pEnd));
2925   else PetscCall(PetscSectionGetChart(mesh->coneSection, pStart, pEnd));
2926   PetscFunctionReturn(PETSC_SUCCESS);
2927 }
2928 
2929 /*@
2930   DMPlexSetChart - Set the interval for all mesh points [`pStart`, `pEnd`)
2931 
2932   Not Collective
2933 
2934   Input Parameters:
2935 + dm     - The `DMPLEX`
2936 . pStart - The first mesh point
2937 - pEnd   - The upper bound for mesh points
2938 
2939   Level: beginner
2940 
2941 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreate()`, `DMPlexGetChart()`
2942 @*/
2943 PetscErrorCode DMPlexSetChart(DM dm, PetscInt pStart, PetscInt pEnd)
2944 {
2945   DM_Plex *mesh = (DM_Plex *)dm->data;
2946 
2947   PetscFunctionBegin;
2948   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2949   PetscCall(PetscSectionSetChart(mesh->coneSection, pStart, pEnd));
2950   PetscCall(PetscSectionSetChart(mesh->supportSection, pStart, pEnd));
2951   PetscCall(PetscFree(mesh->cellTypes));
2952   PetscFunctionReturn(PETSC_SUCCESS);
2953 }
2954 
2955 /*@
2956   DMPlexGetConeSize - Return the number of in-edges for this point in the DAG
2957 
2958   Not Collective
2959 
2960   Input Parameters:
2961 + dm - The `DMPLEX`
2962 - p  - The point, which must lie in the chart set with `DMPlexSetChart()`
2963 
2964   Output Parameter:
2965 . size - The cone size for point `p`
2966 
2967   Level: beginner
2968 
2969 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreate()`, `DMPlexSetConeSize()`, `DMPlexSetChart()`
2970 @*/
2971 PetscErrorCode DMPlexGetConeSize(DM dm, PetscInt p, PetscInt *size)
2972 {
2973   DM_Plex *mesh = (DM_Plex *)dm->data;
2974 
2975   PetscFunctionBegin;
2976   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2977   PetscAssertPointer(size, 3);
2978   if (mesh->tr) PetscCall(DMPlexTransformGetConeSize(mesh->tr, p, size));
2979   else PetscCall(PetscSectionGetDof(mesh->coneSection, p, size));
2980   PetscFunctionReturn(PETSC_SUCCESS);
2981 }
2982 
2983 /*@
2984   DMPlexSetConeSize - Set the number of in-edges for this point in the DAG
2985 
2986   Not Collective
2987 
2988   Input Parameters:
2989 + dm   - The `DMPLEX`
2990 . p    - The point, which must lie in the chart set with `DMPlexSetChart()`
2991 - size - The cone size for point `p`
2992 
2993   Level: beginner
2994 
2995   Note:
2996   This should be called after `DMPlexSetChart()`.
2997 
2998 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexSetCone()`, `DMPlexCreate()`, `DMPlexGetConeSize()`, `DMPlexSetChart()`
2999 @*/
3000 PetscErrorCode DMPlexSetConeSize(DM dm, PetscInt p, PetscInt size)
3001 {
3002   DM_Plex *mesh = (DM_Plex *)dm->data;
3003 
3004   PetscFunctionBegin;
3005   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3006   PetscCheck(!mesh->tr, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Cannot call DMPlexSetConeSize() on a mesh with a transform defined.");
3007   PetscCall(PetscSectionSetDof(mesh->coneSection, p, size));
3008   PetscFunctionReturn(PETSC_SUCCESS);
3009 }
3010 
3011 /*@C
3012   DMPlexGetCone - Return the points on the in-edges for this point in the DAG
3013 
3014   Not Collective
3015 
3016   Input Parameters:
3017 + dm - The `DMPLEX`
3018 - p  - The point, which must lie in the chart set with `DMPlexSetChart()`
3019 
3020   Output Parameter:
3021 . cone - An array of points which are on the in-edges for point `p`, the length of `cone` is the result of `DMPlexGetConeSize()`
3022 
3023   Level: beginner
3024 
3025   Fortran Notes:
3026   `cone` must be declared with
3027 .vb
3028   PetscInt, pointer :: cone(:)
3029 .ve
3030 
3031   You must also call `DMPlexRestoreCone()` after you finish using the array.
3032   `DMPlexRestoreCone()` is not needed/available in C.
3033 
3034 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetConeSize()`, `DMPlexSetCone()`, `DMPlexGetConeTuple()`, `DMPlexSetChart()`, `DMPlexRestoreCone()`
3035 @*/
3036 PetscErrorCode DMPlexGetCone(DM dm, PetscInt p, const PetscInt *cone[])
3037 {
3038   DM_Plex *mesh = (DM_Plex *)dm->data;
3039   PetscInt off;
3040 
3041   PetscFunctionBegin;
3042   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3043   PetscAssertPointer(cone, 3);
3044   PetscCall(PetscSectionGetOffset(mesh->coneSection, p, &off));
3045   *cone = PetscSafePointerPlusOffset(mesh->cones, off);
3046   PetscFunctionReturn(PETSC_SUCCESS);
3047 }
3048 
3049 /*@
3050   DMPlexGetConeTuple - Return the points on the in-edges of several points in the DAG
3051 
3052   Not Collective
3053 
3054   Input Parameters:
3055 + dm - The `DMPLEX`
3056 - p  - The `IS` of points, which must lie in the chart set with `DMPlexSetChart()`
3057 
3058   Output Parameters:
3059 + pConesSection - `PetscSection` describing the layout of `pCones`
3060 - pCones        - An `IS` containing the points which are on the in-edges for the point set `p`
3061 
3062   Level: intermediate
3063 
3064 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreate()`, `DMPlexGetCone()`, `DMPlexGetConeRecursive()`, `DMPlexSetChart()`, `PetscSection`, `IS`
3065 @*/
3066 PetscErrorCode DMPlexGetConeTuple(DM dm, IS p, PetscSection *pConesSection, IS *pCones)
3067 {
3068   PetscSection cs, newcs;
3069   PetscInt    *cones;
3070   PetscInt    *newarr = NULL;
3071   PetscInt     n;
3072 
3073   PetscFunctionBegin;
3074   PetscCall(DMPlexGetCones(dm, &cones));
3075   PetscCall(DMPlexGetConeSection(dm, &cs));
3076   PetscCall(PetscSectionExtractDofsFromArray(cs, MPIU_INT, cones, p, &newcs, pCones ? ((void **)&newarr) : NULL));
3077   if (pConesSection) *pConesSection = newcs;
3078   if (pCones) {
3079     PetscCall(PetscSectionGetStorageSize(newcs, &n));
3080     PetscCall(ISCreateGeneral(PetscObjectComm((PetscObject)p), n, newarr, PETSC_OWN_POINTER, pCones));
3081   }
3082   PetscFunctionReturn(PETSC_SUCCESS);
3083 }
3084 
3085 /*@
3086   DMPlexGetConeRecursiveVertices - Expand each given point into its cone points and do that recursively until we end up just with vertices.
3087 
3088   Not Collective
3089 
3090   Input Parameters:
3091 + dm     - The `DMPLEX`
3092 - points - The `IS` of points, which must lie in the chart set with `DMPlexSetChart()`
3093 
3094   Output Parameter:
3095 . expandedPoints - An `IS` containing the of vertices recursively expanded from input points
3096 
3097   Level: advanced
3098 
3099   Notes:
3100   Like `DMPlexGetConeRecursive()` but returns only the 0-depth `IS` (i.e. vertices only) and no sections.
3101 
3102   There is no corresponding Restore function, just call `ISDestroy()` on the returned `IS` to deallocate.
3103 
3104 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreate()`, `DMPlexGetCone()`, `DMPlexGetConeTuple()`, `DMPlexGetConeRecursive()`, `DMPlexRestoreConeRecursive()`,
3105           `DMPlexGetDepth()`, `IS`
3106 @*/
3107 PetscErrorCode DMPlexGetConeRecursiveVertices(DM dm, IS points, IS *expandedPoints)
3108 {
3109   IS      *expandedPointsAll;
3110   PetscInt depth;
3111 
3112   PetscFunctionBegin;
3113   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3114   PetscValidHeaderSpecific(points, IS_CLASSID, 2);
3115   PetscAssertPointer(expandedPoints, 3);
3116   PetscCall(DMPlexGetConeRecursive(dm, points, &depth, &expandedPointsAll, NULL));
3117   *expandedPoints = expandedPointsAll[0];
3118   PetscCall(PetscObjectReference((PetscObject)expandedPointsAll[0]));
3119   PetscCall(DMPlexRestoreConeRecursive(dm, points, &depth, &expandedPointsAll, NULL));
3120   PetscFunctionReturn(PETSC_SUCCESS);
3121 }
3122 
3123 /*@
3124   DMPlexGetConeRecursive - Expand each given point into its cone points and do that recursively until we end up just with vertices
3125   (DAG points of depth 0, i.e., without cones).
3126 
3127   Not Collective
3128 
3129   Input Parameters:
3130 + dm     - The `DMPLEX`
3131 - points - The `IS` of points, which must lie in the chart set with `DMPlexSetChart()`
3132 
3133   Output Parameters:
3134 + depth          - (optional) Size of the output arrays, equal to `DMPLEX` depth, returned by `DMPlexGetDepth()`
3135 . expandedPoints - (optional) An array of index sets with recursively expanded cones
3136 - sections       - (optional) An array of sections which describe mappings from points to their cone points
3137 
3138   Level: advanced
3139 
3140   Notes:
3141   Like `DMPlexGetConeTuple()` but recursive.
3142 
3143   Array `expandedPoints` has size equal to `depth`. Each `expandedPoints`[d] contains DAG points with maximum depth d, recursively cone-wise expanded from the input points.
3144   For example, for d=0 it contains only vertices, for d=1 it can contain vertices and edges, etc.
3145 
3146   Array section has size equal to `depth`.  Each `PetscSection` `sections`[d] realizes mapping from `expandedPoints`[d+1] (section points) to `expandedPoints`[d] (section dofs) as follows\:
3147   (1) DAG points in `expandedPoints`[d+1] with `depth` d+1 to their cone points in `expandedPoints`[d];
3148   (2) DAG points in `expandedPoints`[d+1] with `depth` in [0,d] to the same points in `expandedPoints`[d].
3149 
3150 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreate()`, `DMPlexGetCone()`, `DMPlexGetConeTuple()`, `DMPlexRestoreConeRecursive()`, `DMPlexGetConeRecursiveVertices()`,
3151           `DMPlexGetDepth()`, `PetscSection`, `IS`
3152 @*/
3153 PetscErrorCode DMPlexGetConeRecursive(DM dm, IS points, PetscInt *depth, IS *expandedPoints[], PetscSection *sections[])
3154 {
3155   const PetscInt *arr0 = NULL, *cone = NULL;
3156   PetscInt       *arr = NULL, *newarr = NULL;
3157   PetscInt        d, depth_, i, n, newn, cn, co, start, end;
3158   IS             *expandedPoints_;
3159   PetscSection   *sections_;
3160 
3161   PetscFunctionBegin;
3162   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3163   PetscValidHeaderSpecific(points, IS_CLASSID, 2);
3164   if (depth) PetscAssertPointer(depth, 3);
3165   if (expandedPoints) PetscAssertPointer(expandedPoints, 4);
3166   if (sections) PetscAssertPointer(sections, 5);
3167   PetscCall(ISGetLocalSize(points, &n));
3168   PetscCall(ISGetIndices(points, &arr0));
3169   PetscCall(DMPlexGetDepth(dm, &depth_));
3170   PetscCall(PetscCalloc1(depth_, &expandedPoints_));
3171   PetscCall(PetscCalloc1(depth_, &sections_));
3172   arr = (PetscInt *)arr0; /* this is ok because first generation of arr is not modified */
3173   for (d = depth_ - 1; d >= 0; d--) {
3174     PetscCall(PetscSectionCreate(PETSC_COMM_SELF, &sections_[d]));
3175     PetscCall(PetscSectionSetChart(sections_[d], 0, n));
3176     for (i = 0; i < n; i++) {
3177       PetscCall(DMPlexGetDepthStratum(dm, d + 1, &start, &end));
3178       if (arr[i] >= start && arr[i] < end) {
3179         PetscCall(DMPlexGetConeSize(dm, arr[i], &cn));
3180         PetscCall(PetscSectionSetDof(sections_[d], i, cn));
3181       } else {
3182         PetscCall(PetscSectionSetDof(sections_[d], i, 1));
3183       }
3184     }
3185     PetscCall(PetscSectionSetUp(sections_[d]));
3186     PetscCall(PetscSectionGetStorageSize(sections_[d], &newn));
3187     PetscCall(PetscMalloc1(newn, &newarr));
3188     for (i = 0; i < n; i++) {
3189       PetscCall(PetscSectionGetDof(sections_[d], i, &cn));
3190       PetscCall(PetscSectionGetOffset(sections_[d], i, &co));
3191       if (cn > 1) {
3192         PetscCall(DMPlexGetCone(dm, arr[i], &cone));
3193         PetscCall(PetscMemcpy(&newarr[co], cone, cn * sizeof(PetscInt)));
3194       } else {
3195         newarr[co] = arr[i];
3196       }
3197     }
3198     PetscCall(ISCreateGeneral(PETSC_COMM_SELF, newn, newarr, PETSC_OWN_POINTER, &expandedPoints_[d]));
3199     arr = newarr;
3200     n   = newn;
3201   }
3202   PetscCall(ISRestoreIndices(points, &arr0));
3203   *depth = depth_;
3204   if (expandedPoints) *expandedPoints = expandedPoints_;
3205   else {
3206     for (d = 0; d < depth_; d++) PetscCall(ISDestroy(&expandedPoints_[d]));
3207     PetscCall(PetscFree(expandedPoints_));
3208   }
3209   if (sections) *sections = sections_;
3210   else {
3211     for (d = 0; d < depth_; d++) PetscCall(PetscSectionDestroy(&sections_[d]));
3212     PetscCall(PetscFree(sections_));
3213   }
3214   PetscFunctionReturn(PETSC_SUCCESS);
3215 }
3216 
3217 /*@
3218   DMPlexRestoreConeRecursive - Deallocates arrays created by `DMPlexGetConeRecursive()`
3219 
3220   Not Collective
3221 
3222   Input Parameters:
3223 + dm     - The `DMPLEX`
3224 - points - The `IS` of points, which must lie in the chart set with `DMPlexSetChart()`
3225 
3226   Output Parameters:
3227 + depth          - (optional) Size of the output arrays, equal to `DMPLEX` depth, returned by `DMPlexGetDepth()`
3228 . expandedPoints - (optional) An array of recursively expanded cones
3229 - sections       - (optional) An array of sections which describe mappings from points to their cone points
3230 
3231   Level: advanced
3232 
3233   Note:
3234   See `DMPlexGetConeRecursive()`
3235 
3236 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreate()`, `DMPlexGetCone()`, `DMPlexGetConeTuple()`, `DMPlexGetConeRecursive()`, `DMPlexGetConeRecursiveVertices()`,
3237           `DMPlexGetDepth()`, `IS`, `PetscSection`
3238 @*/
3239 PetscErrorCode DMPlexRestoreConeRecursive(DM dm, IS points, PetscInt *depth, IS *expandedPoints[], PetscSection *sections[])
3240 {
3241   PetscInt d, depth_;
3242 
3243   PetscFunctionBegin;
3244   PetscCall(DMPlexGetDepth(dm, &depth_));
3245   PetscCheck(!depth || *depth == depth_, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "depth changed since last call to DMPlexGetConeRecursive");
3246   if (depth) *depth = 0;
3247   if (expandedPoints) {
3248     for (d = 0; d < depth_; d++) PetscCall(ISDestroy(&(*expandedPoints)[d]));
3249     PetscCall(PetscFree(*expandedPoints));
3250   }
3251   if (sections) {
3252     for (d = 0; d < depth_; d++) PetscCall(PetscSectionDestroy(&(*sections)[d]));
3253     PetscCall(PetscFree(*sections));
3254   }
3255   PetscFunctionReturn(PETSC_SUCCESS);
3256 }
3257 
3258 /*@
3259   DMPlexSetCone - Set the points on the in-edges for this point in the DAG; that is these are the points that cover the specific point
3260 
3261   Not Collective
3262 
3263   Input Parameters:
3264 + dm   - The `DMPLEX`
3265 . p    - The point, which must lie in the chart set with `DMPlexSetChart()`
3266 - cone - An array of points which are on the in-edges for point `p`, its length must have been previously provided with `DMPlexSetConeSize()`
3267 
3268   Level: beginner
3269 
3270   Note:
3271   This should be called after all calls to `DMPlexSetConeSize()` and `DMSetUp()`.
3272 
3273 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreate()`, `DMPlexGetCone()`, `DMPlexSetChart()`, `DMPlexSetConeSize()`, `DMSetUp()`, `DMPlexSetSupport()`, `DMPlexSetSupportSize()`
3274 @*/
3275 PetscErrorCode DMPlexSetCone(DM dm, PetscInt p, const PetscInt cone[])
3276 {
3277   DM_Plex *mesh = (DM_Plex *)dm->data;
3278   PetscInt dof, off, c;
3279 
3280   PetscFunctionBegin;
3281   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3282   PetscCall(PetscSectionGetDof(mesh->coneSection, p, &dof));
3283   if (dof) PetscAssertPointer(cone, 3);
3284   PetscCall(PetscSectionGetOffset(mesh->coneSection, p, &off));
3285   if (PetscDefined(USE_DEBUG)) {
3286     PetscInt pStart, pEnd;
3287     PetscCall(PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd));
3288     PetscCheck(!(p < pStart) && !(p >= pEnd), PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %" PetscInt_FMT " is not in the valid range [%" PetscInt_FMT ", %" PetscInt_FMT ")", p, pStart, pEnd);
3289     for (c = 0; c < dof; ++c) {
3290       PetscCheck(!(cone[c] < pStart) && !(cone[c] >= pEnd), PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone point %" PetscInt_FMT " is not in the valid range [%" PetscInt_FMT ", %" PetscInt_FMT ")", cone[c], pStart, pEnd);
3291       mesh->cones[off + c] = cone[c];
3292     }
3293   } else {
3294     for (c = 0; c < dof; ++c) mesh->cones[off + c] = cone[c];
3295   }
3296   PetscFunctionReturn(PETSC_SUCCESS);
3297 }
3298 
3299 /*@C
3300   DMPlexGetConeOrientation - Return the orientations on the in-edges for this point in the DAG
3301 
3302   Not Collective
3303 
3304   Input Parameters:
3305 + dm - The `DMPLEX`
3306 - p  - The point, which must lie in the chart set with `DMPlexSetChart()`
3307 
3308   Output Parameter:
3309 . coneOrientation - An array of orientations which are on the in-edges for point `p`. An orientation is an
3310                     integer giving the prescription for cone traversal. Its length is given by the result of `DMPlexSetConeSize()`
3311 
3312   Level: beginner
3313 
3314   Note:
3315   The number indexes the symmetry transformations for the cell type (see manual). Orientation 0 is always
3316   the identity transformation. Negative orientation indicates reflection so that -(o+1) is the reflection
3317   of o, however it is not necessarily the inverse. To get the inverse, use `DMPolytopeTypeComposeOrientationInv()`
3318   with the identity.
3319 
3320   Fortran Notes:
3321   You must call `DMPlexRestoreConeOrientation()` after you finish using the returned array.
3322   `DMPlexRestoreConeOrientation()` is not needed/available in C.
3323 
3324 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexSetConeSize()`, `DMPolytopeTypeComposeOrientation()`, `DMPolytopeTypeComposeOrientationInv()`,
3325           `DMPlexCreate()`, `DMPlexGetCone()`, `DMPlexSetCone()`, `DMPlexSetChart()`
3326 @*/
3327 PetscErrorCode DMPlexGetConeOrientation(DM dm, PetscInt p, const PetscInt *coneOrientation[])
3328 {
3329   DM_Plex *mesh = (DM_Plex *)dm->data;
3330   PetscInt off;
3331 
3332   PetscFunctionBegin;
3333   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3334   if (PetscDefined(USE_DEBUG)) {
3335     PetscInt dof;
3336     PetscCall(PetscSectionGetDof(mesh->coneSection, p, &dof));
3337     if (dof) PetscAssertPointer(coneOrientation, 3);
3338   }
3339   PetscCall(PetscSectionGetOffset(mesh->coneSection, p, &off));
3340 
3341   *coneOrientation = &mesh->coneOrientations[off];
3342   PetscFunctionReturn(PETSC_SUCCESS);
3343 }
3344 
3345 /*@
3346   DMPlexSetConeOrientation - Set the orientations on the in-edges for this point in the DAG
3347 
3348   Not Collective
3349 
3350   Input Parameters:
3351 + dm              - The `DMPLEX`
3352 . p               - The point, which must lie in the chart set with `DMPlexSetChart()`
3353 - coneOrientation - An array of orientations. Its length is given by the result of `DMPlexSetConeSize()`
3354 
3355   Level: beginner
3356 
3357   Notes:
3358   This should be called after all calls to `DMPlexSetConeSize()` and `DMSetUp()`.
3359 
3360   The meaning of coneOrientation is detailed in `DMPlexGetConeOrientation()`.
3361 
3362 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreate()`, `DMPlexGetConeOrientation()`, `DMPlexSetCone()`, `DMPlexSetChart()`, `DMPlexSetConeSize()`, `DMSetUp()`
3363 @*/
3364 PetscErrorCode DMPlexSetConeOrientation(DM dm, PetscInt p, const PetscInt coneOrientation[])
3365 {
3366   DM_Plex *mesh = (DM_Plex *)dm->data;
3367   PetscInt pStart, pEnd;
3368   PetscInt dof, off, c;
3369 
3370   PetscFunctionBegin;
3371   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3372   PetscCall(PetscSectionGetDof(mesh->coneSection, p, &dof));
3373   if (dof) PetscAssertPointer(coneOrientation, 3);
3374   PetscCall(PetscSectionGetOffset(mesh->coneSection, p, &off));
3375   if (PetscDefined(USE_DEBUG)) {
3376     PetscCall(PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd));
3377     PetscCheck(!(p < pStart) && !(p >= pEnd), PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %" PetscInt_FMT " is not in the valid range [%" PetscInt_FMT ", %" PetscInt_FMT ")", p, pStart, pEnd);
3378     for (c = 0; c < dof; ++c) {
3379       PetscInt cdof, o = coneOrientation[c];
3380 
3381       PetscCall(PetscSectionGetDof(mesh->coneSection, mesh->cones[off + c], &cdof));
3382       PetscCheck(!o || (o >= -(cdof + 1) && o < cdof), PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone orientation %" PetscInt_FMT " is not in the valid range [%" PetscInt_FMT ". %" PetscInt_FMT ")", o, -(cdof + 1), cdof);
3383       mesh->coneOrientations[off + c] = o;
3384     }
3385   } else {
3386     for (c = 0; c < dof; ++c) mesh->coneOrientations[off + c] = coneOrientation[c];
3387   }
3388   PetscFunctionReturn(PETSC_SUCCESS);
3389 }
3390 
3391 /*@
3392   DMPlexInsertCone - Insert a point into the in-edges for the point p in the DAG
3393 
3394   Not Collective
3395 
3396   Input Parameters:
3397 + dm        - The `DMPLEX`
3398 . p         - The point, which must lie in the chart set with `DMPlexSetChart()`
3399 . conePos   - The local index in the cone where the point should be put
3400 - conePoint - The mesh point to insert
3401 
3402   Level: beginner
3403 
3404 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreate()`, `DMPlexGetCone()`, `DMPlexSetChart()`, `DMPlexSetConeSize()`, `DMSetUp()`
3405 @*/
3406 PetscErrorCode DMPlexInsertCone(DM dm, PetscInt p, PetscInt conePos, PetscInt conePoint)
3407 {
3408   DM_Plex *mesh = (DM_Plex *)dm->data;
3409   PetscInt pStart, pEnd;
3410   PetscInt dof, off;
3411 
3412   PetscFunctionBegin;
3413   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3414   if (PetscDefined(USE_DEBUG)) {
3415     PetscCall(PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd));
3416     PetscCheck(!(p < pStart) && !(p >= pEnd), PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %" PetscInt_FMT " is not in the valid range [%" PetscInt_FMT ", %" PetscInt_FMT ")", p, pStart, pEnd);
3417     PetscCheck(!(conePoint < pStart) && !(conePoint >= pEnd), PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone point %" PetscInt_FMT " is not in the valid range [%" PetscInt_FMT ", %" PetscInt_FMT ")", conePoint, pStart, pEnd);
3418     PetscCall(PetscSectionGetDof(mesh->coneSection, p, &dof));
3419     PetscCheck(!(conePos < 0) && !(conePos >= dof), PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone position %" PetscInt_FMT " of point %" PetscInt_FMT " is not in the valid range [0, %" PetscInt_FMT ")", conePos, p, dof);
3420   }
3421   PetscCall(PetscSectionGetOffset(mesh->coneSection, p, &off));
3422   mesh->cones[off + conePos] = conePoint;
3423   PetscFunctionReturn(PETSC_SUCCESS);
3424 }
3425 
3426 /*@
3427   DMPlexInsertConeOrientation - Insert a point orientation for the in-edge for the point p in the DAG
3428 
3429   Not Collective
3430 
3431   Input Parameters:
3432 + dm              - The `DMPLEX`
3433 . p               - The point, which must lie in the chart set with `DMPlexSetChart()`
3434 . conePos         - The local index in the cone where the point should be put
3435 - coneOrientation - The point orientation to insert
3436 
3437   Level: beginner
3438 
3439   Note:
3440   The meaning of coneOrientation values is detailed in `DMPlexGetConeOrientation()`.
3441 
3442 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreate()`, `DMPlexGetCone()`, `DMPlexSetChart()`, `DMPlexSetConeSize()`, `DMSetUp()`
3443 @*/
3444 PetscErrorCode DMPlexInsertConeOrientation(DM dm, PetscInt p, PetscInt conePos, PetscInt coneOrientation)
3445 {
3446   DM_Plex *mesh = (DM_Plex *)dm->data;
3447   PetscInt pStart, pEnd;
3448   PetscInt dof, off;
3449 
3450   PetscFunctionBegin;
3451   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3452   if (PetscDefined(USE_DEBUG)) {
3453     PetscCall(PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd));
3454     PetscCheck(!(p < pStart) && !(p >= pEnd), PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %" PetscInt_FMT " is not in the valid range [%" PetscInt_FMT ", %" PetscInt_FMT ")", p, pStart, pEnd);
3455     PetscCall(PetscSectionGetDof(mesh->coneSection, p, &dof));
3456     PetscCheck(!(conePos < 0) && !(conePos >= dof), PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cone position %" PetscInt_FMT " of point %" PetscInt_FMT " is not in the valid range [0, %" PetscInt_FMT ")", conePos, p, dof);
3457   }
3458   PetscCall(PetscSectionGetOffset(mesh->coneSection, p, &off));
3459   mesh->coneOrientations[off + conePos] = coneOrientation;
3460   PetscFunctionReturn(PETSC_SUCCESS);
3461 }
3462 
3463 /*@C
3464   DMPlexGetOrientedCone - Return the points and orientations on the in-edges for this point in the DAG
3465 
3466   Not collective
3467 
3468   Input Parameters:
3469 + dm - The DMPlex
3470 - p  - The point, which must lie in the chart set with DMPlexSetChart()
3471 
3472   Output Parameters:
3473 + cone - An array of points which are on the in-edges for point `p`
3474 - ornt - An array of orientations which are on the in-edges for point `p`. An orientation is an
3475          integer giving the prescription for cone traversal.
3476 
3477   Level: beginner
3478 
3479   Notes:
3480   The number indexes the symmetry transformations for the cell type (see manual). Orientation 0 is always
3481   the identity transformation. Negative orientation indicates reflection so that -(o+1) is the reflection
3482   of o, however it is not necessarily the inverse. To get the inverse, use `DMPolytopeTypeComposeOrientationInv()`
3483   with the identity.
3484 
3485   You must also call `DMPlexRestoreOrientedCone()` after you finish using the returned array.
3486 
3487   Fortran Notes:
3488   `cone` and `ornt` must be declared with
3489 .vb
3490   PetscInt, pointer :: cone(:)
3491   PetscInt, pointer :: ornt(:)
3492 .ve
3493 
3494 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexRestoreOrientedCone()`, `DMPlexGetConeSize()`, `DMPlexGetCone()`, `DMPlexGetChart()`
3495 @*/
3496 PetscErrorCode DMPlexGetOrientedCone(DM dm, PetscInt p, const PetscInt *cone[], const PetscInt *ornt[])
3497 {
3498   DM_Plex *mesh = (DM_Plex *)dm->data;
3499 
3500   PetscFunctionBegin;
3501   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3502   if (mesh->tr) {
3503     PetscCall(DMPlexTransformGetCone(mesh->tr, p, cone, ornt));
3504   } else {
3505     PetscInt off;
3506     if (PetscDefined(USE_DEBUG)) {
3507       PetscInt dof;
3508       PetscCall(PetscSectionGetDof(mesh->coneSection, p, &dof));
3509       if (dof) {
3510         if (cone) PetscAssertPointer(cone, 3);
3511         if (ornt) PetscAssertPointer(ornt, 4);
3512       }
3513     }
3514     PetscCall(PetscSectionGetOffset(mesh->coneSection, p, &off));
3515     if (cone) *cone = PetscSafePointerPlusOffset(mesh->cones, off);
3516     if (ornt) *ornt = PetscSafePointerPlusOffset(mesh->coneOrientations, off);
3517   }
3518   PetscFunctionReturn(PETSC_SUCCESS);
3519 }
3520 
3521 /*@C
3522   DMPlexRestoreOrientedCone - Restore the points and orientations on the in-edges for this point in the DAG obtained with `DMPlexGetOrientedCone()`
3523 
3524   Not Collective
3525 
3526   Input Parameters:
3527 + dm   - The DMPlex
3528 . p    - The point, which must lie in the chart set with `DMPlexSetChart()`
3529 . cone - An array of points which are on the in-edges for point p
3530 - ornt - An array of orientations which are on the in-edges for point `p`. An orientation is an
3531          integer giving the prescription for cone traversal.
3532 
3533   Level: beginner
3534 
3535 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetOrientedCone()`, `DMPlexGetConeSize()`, `DMPlexGetCone()`, `DMPlexGetChart()`
3536 @*/
3537 PetscErrorCode DMPlexRestoreOrientedCone(DM dm, PetscInt p, const PetscInt *cone[], const PetscInt *ornt[])
3538 {
3539   DM_Plex *mesh = (DM_Plex *)dm->data;
3540 
3541   PetscFunctionBegin;
3542   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3543   if (mesh->tr) PetscCall(DMPlexTransformRestoreCone(mesh->tr, p, cone, ornt));
3544   PetscFunctionReturn(PETSC_SUCCESS);
3545 }
3546 
3547 /*@
3548   DMPlexGetSupportSize - Return the number of out-edges for this point in the DAG
3549 
3550   Not Collective
3551 
3552   Input Parameters:
3553 + dm - The `DMPLEX`
3554 - p  - The point, which must lie in the chart set with `DMPlexSetChart()`
3555 
3556   Output Parameter:
3557 . size - The support size for point `p`
3558 
3559   Level: beginner
3560 
3561 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreate()`, `DMPlexSetConeSize()`, `DMPlexSetChart()`, `DMPlexGetConeSize()`
3562 @*/
3563 PetscErrorCode DMPlexGetSupportSize(DM dm, PetscInt p, PetscInt *size)
3564 {
3565   DM_Plex *mesh = (DM_Plex *)dm->data;
3566 
3567   PetscFunctionBegin;
3568   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3569   PetscAssertPointer(size, 3);
3570   PetscCall(PetscSectionGetDof(mesh->supportSection, p, size));
3571   PetscFunctionReturn(PETSC_SUCCESS);
3572 }
3573 
3574 /*@
3575   DMPlexSetSupportSize - Set the number of out-edges for this point in the DAG
3576 
3577   Not Collective
3578 
3579   Input Parameters:
3580 + dm   - The `DMPLEX`
3581 . p    - The point, which must lie in the chart set with `DMPlexSetChart()`
3582 - size - The support size for point `p`
3583 
3584   Level: beginner
3585 
3586   Note:
3587   This should be called after `DMPlexSetChart()`.
3588 
3589 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreate()`, `DMPlexGetSupportSize()`, `DMPlexSetChart()`
3590 @*/
3591 PetscErrorCode DMPlexSetSupportSize(DM dm, PetscInt p, PetscInt size)
3592 {
3593   DM_Plex *mesh = (DM_Plex *)dm->data;
3594 
3595   PetscFunctionBegin;
3596   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3597   PetscCall(PetscSectionSetDof(mesh->supportSection, p, size));
3598   PetscFunctionReturn(PETSC_SUCCESS);
3599 }
3600 
3601 /*@C
3602   DMPlexGetSupport - Return the points on the out-edges for this point in the DAG
3603 
3604   Not Collective
3605 
3606   Input Parameters:
3607 + dm - The `DMPLEX`
3608 - p  - The point, which must lie in the chart set with `DMPlexSetChart()`
3609 
3610   Output Parameter:
3611 . support - An array of points which are on the out-edges for point `p`, its length is that obtained from `DMPlexGetSupportSize()`
3612 
3613   Level: beginner
3614 
3615   Fortran Notes:
3616   `support` must be declared with
3617 .vb
3618   PetscInt, pointer :: support(:)
3619 .ve
3620 
3621   You must also call `DMPlexRestoreSupport()` after you finish using the returned array.
3622   `DMPlexRestoreSupport()` is not needed/available in C.
3623 
3624 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetSupportSize()`, `DMPlexSetSupport()`, `DMPlexGetCone()`, `DMPlexSetChart()`
3625 @*/
3626 PetscErrorCode DMPlexGetSupport(DM dm, PetscInt p, const PetscInt *support[])
3627 {
3628   DM_Plex *mesh = (DM_Plex *)dm->data;
3629   PetscInt off;
3630 
3631   PetscFunctionBegin;
3632   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3633   PetscAssertPointer(support, 3);
3634   PetscCall(PetscSectionGetOffset(mesh->supportSection, p, &off));
3635   *support = PetscSafePointerPlusOffset(mesh->supports, off);
3636   PetscFunctionReturn(PETSC_SUCCESS);
3637 }
3638 
3639 /*@
3640   DMPlexSetSupport - Set the points on the out-edges for this point in the DAG, that is the list of points that this point covers
3641 
3642   Not Collective
3643 
3644   Input Parameters:
3645 + dm      - The `DMPLEX`
3646 . p       - The point, which must lie in the chart set with `DMPlexSetChart()`
3647 - support - An array of points which are on the out-edges for point `p`, its length is that obtained from `DMPlexGetSupportSize()`
3648 
3649   Level: beginner
3650 
3651   Note:
3652   This should be called after all calls to `DMPlexSetSupportSize()` and `DMSetUp()`.
3653 
3654 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexSetCone()`, `DMPlexSetConeSize()`, `DMPlexCreate()`, `DMPlexGetSupport()`, `DMPlexSetChart()`, `DMPlexSetSupportSize()`, `DMSetUp()`
3655 @*/
3656 PetscErrorCode DMPlexSetSupport(DM dm, PetscInt p, const PetscInt support[])
3657 {
3658   DM_Plex *mesh = (DM_Plex *)dm->data;
3659   PetscInt pStart, pEnd;
3660   PetscInt dof, off, c;
3661 
3662   PetscFunctionBegin;
3663   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3664   PetscCall(PetscSectionGetChart(mesh->supportSection, &pStart, &pEnd));
3665   PetscCall(PetscSectionGetDof(mesh->supportSection, p, &dof));
3666   if (dof) PetscAssertPointer(support, 3);
3667   PetscCall(PetscSectionGetOffset(mesh->supportSection, p, &off));
3668   PetscCheck(!(p < pStart) && !(p >= pEnd), PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %" PetscInt_FMT " is not in the valid range [%" PetscInt_FMT ", %" PetscInt_FMT ")", p, pStart, pEnd);
3669   for (c = 0; c < dof; ++c) {
3670     PetscCheck(!(support[c] < pStart) && !(support[c] >= pEnd), PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support point %" PetscInt_FMT " is not in the valid range [%" PetscInt_FMT ", %" PetscInt_FMT ")", support[c], pStart, pEnd);
3671     mesh->supports[off + c] = support[c];
3672   }
3673   PetscFunctionReturn(PETSC_SUCCESS);
3674 }
3675 
3676 /*@
3677   DMPlexInsertSupport - Insert a point into the out-edges for the point p in the DAG
3678 
3679   Not Collective
3680 
3681   Input Parameters:
3682 + dm           - The `DMPLEX`
3683 . p            - The point, which must lie in the chart set with `DMPlexSetChart()`
3684 . supportPos   - The local index in the cone where the point should be put
3685 - supportPoint - The mesh point to insert
3686 
3687   Level: beginner
3688 
3689 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreate()`, `DMPlexGetCone()`, `DMPlexSetChart()`, `DMPlexSetConeSize()`, `DMSetUp()`
3690 @*/
3691 PetscErrorCode DMPlexInsertSupport(DM dm, PetscInt p, PetscInt supportPos, PetscInt supportPoint)
3692 {
3693   DM_Plex *mesh = (DM_Plex *)dm->data;
3694   PetscInt pStart, pEnd;
3695   PetscInt dof, off;
3696 
3697   PetscFunctionBegin;
3698   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3699   PetscCall(PetscSectionGetChart(mesh->supportSection, &pStart, &pEnd));
3700   PetscCall(PetscSectionGetDof(mesh->supportSection, p, &dof));
3701   PetscCall(PetscSectionGetOffset(mesh->supportSection, p, &off));
3702   PetscCheck(!(p < pStart) && !(p >= pEnd), PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Mesh point %" PetscInt_FMT " is not in the valid range [%" PetscInt_FMT ", %" PetscInt_FMT ")", p, pStart, pEnd);
3703   PetscCheck(!(supportPoint < pStart) && !(supportPoint >= pEnd), PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support point %" PetscInt_FMT " is not in the valid range [%" PetscInt_FMT ", %" PetscInt_FMT ")", supportPoint, pStart, pEnd);
3704   PetscCheck(supportPos < dof, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Support position %" PetscInt_FMT " of point %" PetscInt_FMT " is not in the valid range [0, %" PetscInt_FMT ")", supportPos, p, dof);
3705   mesh->supports[off + supportPos] = supportPoint;
3706   PetscFunctionReturn(PETSC_SUCCESS);
3707 }
3708 
3709 /* Converts an orientation o in the current numbering to the previous scheme used in Plex */
3710 PetscInt DMPolytopeConvertNewOrientation_Internal(DMPolytopeType ct, PetscInt o)
3711 {
3712   switch (ct) {
3713   case DM_POLYTOPE_SEGMENT:
3714     if (o == -1) return -2;
3715     break;
3716   case DM_POLYTOPE_TRIANGLE:
3717     if (o == -3) return -1;
3718     if (o == -2) return -3;
3719     if (o == -1) return -2;
3720     break;
3721   case DM_POLYTOPE_QUADRILATERAL:
3722     if (o == -4) return -2;
3723     if (o == -3) return -1;
3724     if (o == -2) return -4;
3725     if (o == -1) return -3;
3726     break;
3727   default:
3728     return o;
3729   }
3730   return o;
3731 }
3732 
3733 /* Converts an orientation o in the previous scheme used in Plex to the current numbering */
3734 PetscInt DMPolytopeConvertOldOrientation_Internal(DMPolytopeType ct, PetscInt o)
3735 {
3736   switch (ct) {
3737   case DM_POLYTOPE_SEGMENT:
3738     if ((o == -2) || (o == 1)) return -1;
3739     if (o == -1) return 0;
3740     break;
3741   case DM_POLYTOPE_TRIANGLE:
3742     if (o == -3) return -2;
3743     if (o == -2) return -1;
3744     if (o == -1) return -3;
3745     break;
3746   case DM_POLYTOPE_QUADRILATERAL:
3747     if (o == -4) return -2;
3748     if (o == -3) return -1;
3749     if (o == -2) return -4;
3750     if (o == -1) return -3;
3751     break;
3752   default:
3753     return o;
3754   }
3755   return o;
3756 }
3757 
3758 /* Takes in a mesh whose orientations are in the previous scheme and converts them all to the current numbering */
3759 PetscErrorCode DMPlexConvertOldOrientations_Internal(DM dm)
3760 {
3761   PetscInt pStart, pEnd, p;
3762 
3763   PetscFunctionBegin;
3764   PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
3765   for (p = pStart; p < pEnd; ++p) {
3766     const PetscInt *cone, *ornt;
3767     PetscInt        coneSize, c;
3768 
3769     PetscCall(DMPlexGetConeSize(dm, p, &coneSize));
3770     PetscCall(DMPlexGetCone(dm, p, &cone));
3771     PetscCall(DMPlexGetConeOrientation(dm, p, &ornt));
3772     for (c = 0; c < coneSize; ++c) {
3773       DMPolytopeType ct;
3774       const PetscInt o = ornt[c];
3775 
3776       PetscCall(DMPlexGetCellType(dm, cone[c], &ct));
3777       switch (ct) {
3778       case DM_POLYTOPE_SEGMENT:
3779         if ((o == -2) || (o == 1)) PetscCall(DMPlexInsertConeOrientation(dm, p, c, -1));
3780         if (o == -1) PetscCall(DMPlexInsertConeOrientation(dm, p, c, 0));
3781         break;
3782       case DM_POLYTOPE_TRIANGLE:
3783         if (o == -3) PetscCall(DMPlexInsertConeOrientation(dm, p, c, -2));
3784         if (o == -2) PetscCall(DMPlexInsertConeOrientation(dm, p, c, -1));
3785         if (o == -1) PetscCall(DMPlexInsertConeOrientation(dm, p, c, -3));
3786         break;
3787       case DM_POLYTOPE_QUADRILATERAL:
3788         if (o == -4) PetscCall(DMPlexInsertConeOrientation(dm, p, c, -2));
3789         if (o == -3) PetscCall(DMPlexInsertConeOrientation(dm, p, c, -1));
3790         if (o == -2) PetscCall(DMPlexInsertConeOrientation(dm, p, c, -4));
3791         if (o == -1) PetscCall(DMPlexInsertConeOrientation(dm, p, c, -3));
3792         break;
3793       default:
3794         break;
3795       }
3796     }
3797   }
3798   PetscFunctionReturn(PETSC_SUCCESS);
3799 }
3800 
3801 static inline PetscErrorCode DMPlexGetTransitiveClosure_Hot_Private(DM dm, PetscInt p, PetscBool useCone, PetscInt *size, const PetscInt *arr[], const PetscInt *ornt[])
3802 {
3803   DM_Plex *mesh = (DM_Plex *)dm->data;
3804 
3805   PetscFunctionBeginHot;
3806   if (PetscDefined(USE_DEBUG) || mesh->tr) {
3807     if (useCone) {
3808       PetscCall(DMPlexGetConeSize(dm, p, size));
3809       PetscCall(DMPlexGetOrientedCone(dm, p, arr, ornt));
3810     } else {
3811       PetscCall(DMPlexGetSupportSize(dm, p, size));
3812       PetscCall(DMPlexGetSupport(dm, p, arr));
3813     }
3814   } else {
3815     if (useCone) {
3816       const PetscSection s   = mesh->coneSection;
3817       const PetscInt     ps  = p - s->pStart;
3818       const PetscInt     off = s->atlasOff[ps];
3819 
3820       *size = s->atlasDof[ps];
3821       *arr  = mesh->cones + off;
3822       *ornt = mesh->coneOrientations + off;
3823     } else {
3824       const PetscSection s   = mesh->supportSection;
3825       const PetscInt     ps  = p - s->pStart;
3826       const PetscInt     off = s->atlasOff[ps];
3827 
3828       *size = s->atlasDof[ps];
3829       *arr  = mesh->supports + off;
3830     }
3831   }
3832   PetscFunctionReturn(PETSC_SUCCESS);
3833 }
3834 
3835 static inline PetscErrorCode DMPlexRestoreTransitiveClosure_Hot_Private(DM dm, PetscInt p, PetscBool useCone, PetscInt *size, const PetscInt *arr[], const PetscInt *ornt[])
3836 {
3837   DM_Plex *mesh = (DM_Plex *)dm->data;
3838 
3839   PetscFunctionBeginHot;
3840   if (PetscDefined(USE_DEBUG) || mesh->tr) {
3841     if (useCone) PetscCall(DMPlexRestoreOrientedCone(dm, p, arr, ornt));
3842   }
3843   PetscFunctionReturn(PETSC_SUCCESS);
3844 }
3845 
3846 static PetscErrorCode DMPlexGetTransitiveClosure_Depth1_Private(DM dm, PetscInt p, PetscInt ornt, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
3847 {
3848   DMPolytopeType  ct = DM_POLYTOPE_UNKNOWN;
3849   PetscInt       *closure;
3850   const PetscInt *tmp = NULL, *tmpO = NULL;
3851   PetscInt        off = 0, tmpSize, t;
3852 
3853   PetscFunctionBeginHot;
3854   if (ornt) {
3855     PetscCall(DMPlexGetCellType(dm, p, &ct));
3856     if (ct == DM_POLYTOPE_FV_GHOST || ct == DM_POLYTOPE_INTERIOR_GHOST || ct == DM_POLYTOPE_UNKNOWN || ct == DM_POLYTOPE_UNKNOWN_CELL || ct == DM_POLYTOPE_UNKNOWN_FACE) ct = DM_POLYTOPE_UNKNOWN;
3857   }
3858   if (*points) {
3859     closure = *points;
3860   } else {
3861     PetscInt maxConeSize, maxSupportSize;
3862     PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize));
3863     PetscCall(DMGetWorkArray(dm, 2 * (PetscMax(maxConeSize, maxSupportSize) + 1), MPIU_INT, &closure));
3864   }
3865   PetscCall(DMPlexGetTransitiveClosure_Hot_Private(dm, p, useCone, &tmpSize, &tmp, &tmpO));
3866   if (ct == DM_POLYTOPE_UNKNOWN) {
3867     closure[off++] = p;
3868     closure[off++] = 0;
3869     for (t = 0; t < tmpSize; ++t) {
3870       closure[off++] = tmp[t];
3871       closure[off++] = tmpO ? tmpO[t] : 0;
3872     }
3873   } else {
3874     const PetscInt *arr = DMPolytopeTypeGetArrangement(ct, ornt);
3875 
3876     /* We assume that cells with a valid type have faces with a valid type */
3877     closure[off++] = p;
3878     closure[off++] = ornt;
3879     for (t = 0; t < tmpSize; ++t) {
3880       DMPolytopeType ft;
3881 
3882       PetscCall(DMPlexGetCellType(dm, tmp[t], &ft));
3883       closure[off++] = tmp[arr[t]];
3884       closure[off++] = tmpO ? DMPolytopeTypeComposeOrientation(ft, ornt, tmpO[t]) : 0;
3885     }
3886   }
3887   PetscCall(DMPlexRestoreTransitiveClosure_Hot_Private(dm, p, useCone, &tmpSize, &tmp, &tmpO));
3888   if (numPoints) *numPoints = tmpSize + 1;
3889   if (points) *points = closure;
3890   PetscFunctionReturn(PETSC_SUCCESS);
3891 }
3892 
3893 /* We need a special tensor version because we want to allow duplicate points in the endcaps for hybrid cells */
3894 static PetscErrorCode DMPlexTransitiveClosure_Tensor_Internal(DM dm, PetscInt point, DMPolytopeType ct, PetscInt o, PetscBool useCone, PetscInt *numPoints, PetscInt **points)
3895 {
3896   const PetscInt *arr = DMPolytopeTypeGetArrangement(ct, o);
3897   const PetscInt *cone, *ornt;
3898   PetscInt       *pts, *closure = NULL;
3899   DMPolytopeType  ft;
3900   PetscInt        maxConeSize, maxSupportSize, coneSeries, supportSeries, maxSize;
3901   PetscInt        dim, coneSize, c, d, clSize, cl;
3902 
3903   PetscFunctionBeginHot;
3904   PetscCall(DMGetDimension(dm, &dim));
3905   PetscCall(DMPlexGetTransitiveClosure_Hot_Private(dm, point, PETSC_TRUE, &coneSize, &cone, &ornt));
3906   PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize));
3907   coneSeries    = (maxConeSize > 1) ? ((PetscPowInt(maxConeSize, dim + 1) - 1) / (maxConeSize - 1)) : dim + 1;
3908   supportSeries = (maxSupportSize > 1) ? ((PetscPowInt(maxSupportSize, dim + 1) - 1) / (maxSupportSize - 1)) : dim + 1;
3909   maxSize       = PetscMax(coneSeries, supportSeries);
3910   if (*points) {
3911     pts = *points;
3912   } else PetscCall(DMGetWorkArray(dm, 2 * maxSize, MPIU_INT, &pts));
3913   c        = 0;
3914   pts[c++] = point;
3915   pts[c++] = o;
3916   PetscCall(DMPlexGetCellType(dm, cone[arr[0 * 2 + 0]], &ft));
3917   PetscCall(DMPlexGetTransitiveClosure_Internal(dm, cone[arr[0 * 2 + 0]], DMPolytopeTypeComposeOrientation(ft, arr[0 * 2 + 1], ornt[0]), useCone, &clSize, &closure));
3918   for (cl = 0; cl < clSize * 2; cl += 2) {
3919     pts[c++] = closure[cl];
3920     pts[c++] = closure[cl + 1];
3921   }
3922   PetscCall(DMPlexGetTransitiveClosure_Internal(dm, cone[arr[1 * 2 + 0]], DMPolytopeTypeComposeOrientation(ft, arr[1 * 2 + 1], ornt[1]), useCone, &clSize, &closure));
3923   for (cl = 0; cl < clSize * 2; cl += 2) {
3924     pts[c++] = closure[cl];
3925     pts[c++] = closure[cl + 1];
3926   }
3927   PetscCall(DMPlexRestoreTransitiveClosure(dm, cone[0], useCone, &clSize, &closure));
3928   for (d = 2; d < coneSize; ++d) {
3929     PetscCall(DMPlexGetCellType(dm, cone[arr[d * 2 + 0]], &ft));
3930     pts[c++] = cone[arr[d * 2 + 0]];
3931     pts[c++] = DMPolytopeTypeComposeOrientation(ft, arr[d * 2 + 1], ornt[d]);
3932   }
3933   PetscCall(DMPlexRestoreTransitiveClosure_Hot_Private(dm, point, PETSC_TRUE, &coneSize, &cone, &ornt));
3934   if (dim >= 3) {
3935     for (d = 2; d < coneSize; ++d) {
3936       const PetscInt  fpoint = cone[arr[d * 2 + 0]];
3937       const PetscInt *fcone, *fornt;
3938       PetscInt        fconeSize, fc, i;
3939 
3940       PetscCall(DMPlexGetCellType(dm, fpoint, &ft));
3941       const PetscInt *farr = DMPolytopeTypeGetArrangement(ft, DMPolytopeTypeComposeOrientation(ft, arr[d * 2 + 1], ornt[d]));
3942       PetscCall(DMPlexGetTransitiveClosure_Hot_Private(dm, fpoint, PETSC_TRUE, &fconeSize, &fcone, &fornt));
3943       for (fc = 0; fc < fconeSize; ++fc) {
3944         const PetscInt cp = fcone[farr[fc * 2 + 0]];
3945         const PetscInt co = farr[fc * 2 + 1];
3946 
3947         for (i = 0; i < c; i += 2)
3948           if (pts[i] == cp) break;
3949         if (i == c) {
3950           PetscCall(DMPlexGetCellType(dm, cp, &ft));
3951           pts[c++] = cp;
3952           pts[c++] = DMPolytopeTypeComposeOrientation(ft, co, fornt[farr[fc * 2 + 0]]);
3953         }
3954       }
3955       PetscCall(DMPlexRestoreTransitiveClosure_Hot_Private(dm, fpoint, PETSC_TRUE, &fconeSize, &fcone, &fornt));
3956     }
3957   }
3958   *numPoints = c / 2;
3959   *points    = pts;
3960   PetscFunctionReturn(PETSC_SUCCESS);
3961 }
3962 
3963 PetscErrorCode DMPlexGetTransitiveClosure_Internal(DM dm, PetscInt p, PetscInt ornt, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
3964 {
3965   DMPolytopeType ct;
3966   PetscInt      *closure, *fifo;
3967   PetscInt       closureSize = 0, fifoStart = 0, fifoSize = 0;
3968   PetscInt       maxConeSize, maxSupportSize, coneSeries, supportSeries;
3969   PetscInt       depth, maxSize;
3970 
3971   PetscFunctionBeginHot;
3972   PetscCall(DMPlexGetDepth(dm, &depth));
3973   if (depth == 1) {
3974     PetscCall(DMPlexGetTransitiveClosure_Depth1_Private(dm, p, ornt, useCone, numPoints, points));
3975     PetscFunctionReturn(PETSC_SUCCESS);
3976   }
3977   PetscCall(DMPlexGetCellType(dm, p, &ct));
3978   if (ct == DM_POLYTOPE_FV_GHOST || ct == DM_POLYTOPE_INTERIOR_GHOST || ct == DM_POLYTOPE_UNKNOWN || ct == DM_POLYTOPE_UNKNOWN_CELL || ct == DM_POLYTOPE_UNKNOWN_FACE) ct = DM_POLYTOPE_UNKNOWN;
3979   if (DMPolytopeTypeIsHybrid(ct) && ct != DM_POLYTOPE_POINT_PRISM_TENSOR) {
3980     PetscCall(DMPlexTransitiveClosure_Tensor_Internal(dm, p, ct, ornt, useCone, numPoints, points));
3981     PetscFunctionReturn(PETSC_SUCCESS);
3982   }
3983   PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize));
3984   coneSeries    = (maxConeSize > 1) ? ((PetscPowInt(maxConeSize, depth + 1) - 1) / (maxConeSize - 1)) : depth + 1;
3985   supportSeries = (maxSupportSize > 1) ? ((PetscPowInt(maxSupportSize, depth + 1) - 1) / (maxSupportSize - 1)) : depth + 1;
3986   maxSize       = PetscMax(coneSeries, supportSeries);
3987   PetscCall(DMGetWorkArray(dm, 3 * maxSize, MPIU_INT, &fifo));
3988   if (*points) {
3989     closure = *points;
3990   } else PetscCall(DMGetWorkArray(dm, 2 * maxSize, MPIU_INT, &closure));
3991   closure[closureSize++] = p;
3992   closure[closureSize++] = ornt;
3993   fifo[fifoSize++]       = p;
3994   fifo[fifoSize++]       = ornt;
3995   fifo[fifoSize++]       = ct;
3996   /* Should kick out early when depth is reached, rather than checking all vertices for empty cones */
3997   while (fifoSize - fifoStart) {
3998     const PetscInt       q    = fifo[fifoStart++];
3999     const PetscInt       o    = fifo[fifoStart++];
4000     const DMPolytopeType qt   = (DMPolytopeType)fifo[fifoStart++];
4001     const PetscInt      *qarr = DMPolytopeTypeGetArrangement(qt, o);
4002     const PetscInt      *tmp, *tmpO = NULL;
4003     PetscInt             tmpSize, t;
4004 
4005     if (PetscDefined(USE_DEBUG)) {
4006       PetscInt nO = DMPolytopeTypeGetNumArrangements(qt) / 2;
4007       PetscCheck(!o || !(o >= nO || o < -nO), PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid orientation %" PetscInt_FMT " not in [%" PetscInt_FMT ",%" PetscInt_FMT ") for %s %" PetscInt_FMT, o, -nO, nO, DMPolytopeTypes[qt], q);
4008     }
4009     PetscCall(DMPlexGetTransitiveClosure_Hot_Private(dm, q, useCone, &tmpSize, &tmp, &tmpO));
4010     for (t = 0; t < tmpSize; ++t) {
4011       const PetscInt ip = useCone && qarr ? qarr[t * 2] : t;
4012       const PetscInt io = useCone && qarr ? qarr[t * 2 + 1] : 0;
4013       const PetscInt cp = tmp[ip];
4014       PetscCall(DMPlexGetCellType(dm, cp, &ct));
4015       const PetscInt co = tmpO ? DMPolytopeTypeComposeOrientation(ct, io, tmpO[ip]) : 0;
4016       PetscInt       c;
4017 
4018       /* Check for duplicate */
4019       for (c = 0; c < closureSize; c += 2) {
4020         if (closure[c] == cp) break;
4021       }
4022       if (c == closureSize) {
4023         closure[closureSize++] = cp;
4024         closure[closureSize++] = co;
4025         fifo[fifoSize++]       = cp;
4026         fifo[fifoSize++]       = co;
4027         fifo[fifoSize++]       = ct;
4028       }
4029     }
4030     PetscCall(DMPlexRestoreTransitiveClosure_Hot_Private(dm, q, useCone, &tmpSize, &tmp, &tmpO));
4031   }
4032   PetscCall(DMRestoreWorkArray(dm, 3 * maxSize, MPIU_INT, &fifo));
4033   if (numPoints) *numPoints = closureSize / 2;
4034   if (points) *points = closure;
4035   PetscFunctionReturn(PETSC_SUCCESS);
4036 }
4037 
4038 /*@C
4039   DMPlexGetTransitiveClosure - Return the points on the transitive closure of the in-edges or out-edges for this point in the DAG
4040 
4041   Not Collective
4042 
4043   Input Parameters:
4044 + dm      - The `DMPLEX`
4045 . p       - The mesh point
4046 - useCone - `PETSC_TRUE` for the closure, otherwise return the star
4047 
4048   Input/Output Parameter:
4049 . points - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...];
4050            if *points is `NULL` on input, internal storage will be returned, use `DMPlexRestoreTransitiveClosure()`,
4051            otherwise the provided array is used to hold the values
4052 
4053   Output Parameter:
4054 . numPoints - The number of points in the closure, so `points` is of size 2*`numPoints`
4055 
4056   Level: beginner
4057 
4058   Note:
4059   If using internal storage (points is `NULL` on input), each call overwrites the last output.
4060 
4061   Fortran Notes:
4062   `points` must be declared with
4063 .vb
4064   PetscInt, pointer :: points(:)
4065 .ve
4066   and is always allocated by the function.
4067 
4068   The `numPoints` argument is not present in the Fortran binding.
4069 
4070 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexRestoreTransitiveClosure()`, `DMPlexCreate()`, `DMPlexSetCone()`, `DMPlexSetChart()`, `DMPlexGetCone()`
4071 @*/
4072 PetscErrorCode DMPlexGetTransitiveClosure(DM dm, PetscInt p, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
4073 {
4074   PetscFunctionBeginHot;
4075   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4076   if (numPoints) PetscAssertPointer(numPoints, 4);
4077   if (points) PetscAssertPointer(points, 5);
4078   if (PetscDefined(USE_DEBUG)) {
4079     PetscInt pStart, pEnd;
4080     PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
4081     PetscCheck(p >= pStart && p < pEnd, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Point %" PetscInt_FMT " is not in [%" PetscInt_FMT ", %" PetscInt_FMT ")", p, pStart, pEnd);
4082   }
4083   PetscCall(DMPlexGetTransitiveClosure_Internal(dm, p, 0, useCone, numPoints, points));
4084   PetscFunctionReturn(PETSC_SUCCESS);
4085 }
4086 
4087 /*@C
4088   DMPlexRestoreTransitiveClosure - Restore the array of points on the transitive closure of the in-edges or out-edges for this point in the DAG
4089 
4090   Not Collective
4091 
4092   Input Parameters:
4093 + dm        - The `DMPLEX`
4094 . p         - The mesh point
4095 . useCone   - `PETSC_TRUE` for the closure, otherwise return the star
4096 . numPoints - The number of points in the closure, so points[] is of size 2*`numPoints`
4097 - points    - The points and point orientations, interleaved as pairs [p0, o0, p1, o1, ...]
4098 
4099   Level: beginner
4100 
4101   Note:
4102   If not using internal storage (points is not `NULL` on input), this call is unnecessary
4103 
4104 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetTransitiveClosure()`, `DMPlexCreate()`, `DMPlexSetCone()`, `DMPlexSetChart()`, `DMPlexGetCone()`
4105 @*/
4106 PetscErrorCode DMPlexRestoreTransitiveClosure(DM dm, PetscInt p, PetscBool useCone, PetscInt *numPoints, PetscInt *points[])
4107 {
4108   PetscFunctionBeginHot;
4109   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4110   if (numPoints) *numPoints = 0;
4111   PetscCall(DMRestoreWorkArray(dm, 0, MPIU_INT, points));
4112   PetscFunctionReturn(PETSC_SUCCESS);
4113 }
4114 
4115 /*@
4116   DMPlexGetMaxSizes - Return the maximum number of in-edges (cone) and out-edges (support) for any point in the DAG
4117 
4118   Not Collective
4119 
4120   Input Parameter:
4121 . dm - The `DMPLEX`
4122 
4123   Output Parameters:
4124 + maxConeSize    - The maximum number of in-edges
4125 - maxSupportSize - The maximum number of out-edges
4126 
4127   Level: beginner
4128 
4129 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreate()`, `DMPlexSetConeSize()`, `DMPlexSetChart()`
4130 @*/
4131 PetscErrorCode DMPlexGetMaxSizes(DM dm, PetscInt *maxConeSize, PetscInt *maxSupportSize)
4132 {
4133   DM_Plex *mesh = (DM_Plex *)dm->data;
4134 
4135   PetscFunctionBegin;
4136   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4137   if (maxConeSize) PetscCall(PetscSectionGetMaxDof(mesh->coneSection, maxConeSize));
4138   if (maxSupportSize) PetscCall(PetscSectionGetMaxDof(mesh->supportSection, maxSupportSize));
4139   PetscFunctionReturn(PETSC_SUCCESS);
4140 }
4141 
4142 PetscErrorCode DMSetUp_Plex(DM dm)
4143 {
4144   DM_Plex *mesh = (DM_Plex *)dm->data;
4145   PetscInt size, maxSupportSize;
4146 
4147   PetscFunctionBegin;
4148   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4149   PetscCall(PetscSectionSetUp(mesh->coneSection));
4150   PetscCall(PetscSectionGetStorageSize(mesh->coneSection, &size));
4151   PetscCall(PetscMalloc1(size, &mesh->cones));
4152   PetscCall(PetscCalloc1(size, &mesh->coneOrientations));
4153   PetscCall(PetscSectionGetMaxDof(mesh->supportSection, &maxSupportSize));
4154   if (maxSupportSize) {
4155     PetscCall(PetscSectionSetUp(mesh->supportSection));
4156     PetscCall(PetscSectionGetStorageSize(mesh->supportSection, &size));
4157     PetscCall(PetscMalloc1(size, &mesh->supports));
4158   }
4159   PetscFunctionReturn(PETSC_SUCCESS);
4160 }
4161 
4162 PetscErrorCode DMCreateSubDM_Plex(DM dm, PetscInt numFields, const PetscInt fields[], IS *is, DM *subdm)
4163 {
4164   PetscFunctionBegin;
4165   if (subdm) PetscCall(DMClone(dm, subdm));
4166   PetscCall(DMCreateSectionSubDM(dm, numFields, fields, NULL, NULL, is, subdm));
4167   if (subdm) (*subdm)->useNatural = dm->useNatural;
4168   if (dm->useNatural && dm->sfMigration) {
4169     PetscSF sfNatural;
4170 
4171     (*subdm)->sfMigration = dm->sfMigration;
4172     PetscCall(PetscObjectReference((PetscObject)dm->sfMigration));
4173     PetscCall(DMPlexCreateGlobalToNaturalSF(*subdm, NULL, (*subdm)->sfMigration, &sfNatural));
4174     (*subdm)->sfNatural = sfNatural;
4175   }
4176   PetscFunctionReturn(PETSC_SUCCESS);
4177 }
4178 
4179 PetscErrorCode DMCreateSuperDM_Plex(DM dms[], PetscInt len, IS **is, DM *superdm)
4180 {
4181   PetscInt i = 0;
4182 
4183   PetscFunctionBegin;
4184   PetscCall(DMClone(dms[0], superdm));
4185   PetscCall(DMCreateSectionSuperDM(dms, len, is, superdm));
4186   (*superdm)->useNatural = PETSC_FALSE;
4187   for (i = 0; i < len; i++) {
4188     if (dms[i]->useNatural && dms[i]->sfMigration) {
4189       PetscSF sfNatural;
4190 
4191       (*superdm)->sfMigration = dms[i]->sfMigration;
4192       PetscCall(PetscObjectReference((PetscObject)dms[i]->sfMigration));
4193       (*superdm)->useNatural = PETSC_TRUE;
4194       PetscCall(DMPlexCreateGlobalToNaturalSF(*superdm, NULL, (*superdm)->sfMigration, &sfNatural));
4195       (*superdm)->sfNatural = sfNatural;
4196       break;
4197     }
4198   }
4199   PetscFunctionReturn(PETSC_SUCCESS);
4200 }
4201 
4202 /*@
4203   DMPlexSymmetrize - Create support (out-edge) information from cone (in-edge) information
4204 
4205   Not Collective
4206 
4207   Input Parameter:
4208 . dm - The `DMPLEX`
4209 
4210   Level: beginner
4211 
4212   Note:
4213   This should be called after all calls to `DMPlexSetCone()`
4214 
4215 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreate()`, `DMPlexSetChart()`, `DMPlexSetConeSize()`, `DMPlexSetCone()`
4216 @*/
4217 PetscErrorCode DMPlexSymmetrize(DM dm)
4218 {
4219   DM_Plex  *mesh = (DM_Plex *)dm->data;
4220   PetscInt *offsets;
4221   PetscInt  supportSize;
4222   PetscInt  pStart, pEnd, p;
4223 
4224   PetscFunctionBegin;
4225   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4226   PetscCheck(!mesh->supports, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONGSTATE, "Supports were already setup in this DMPlex");
4227   PetscCall(PetscLogEventBegin(DMPLEX_Symmetrize, dm, 0, 0, 0));
4228   /* Calculate support sizes */
4229   PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
4230   for (p = pStart; p < pEnd; ++p) {
4231     PetscInt dof, off, c;
4232 
4233     PetscCall(PetscSectionGetDof(mesh->coneSection, p, &dof));
4234     PetscCall(PetscSectionGetOffset(mesh->coneSection, p, &off));
4235     for (c = off; c < off + dof; ++c) PetscCall(PetscSectionAddDof(mesh->supportSection, mesh->cones[c], 1));
4236   }
4237   PetscCall(PetscSectionSetUp(mesh->supportSection));
4238   /* Calculate supports */
4239   PetscCall(PetscSectionGetStorageSize(mesh->supportSection, &supportSize));
4240   PetscCall(PetscMalloc1(supportSize, &mesh->supports));
4241   PetscCall(PetscCalloc1(pEnd - pStart, &offsets));
4242   for (p = pStart; p < pEnd; ++p) {
4243     PetscInt dof, off, c;
4244 
4245     PetscCall(PetscSectionGetDof(mesh->coneSection, p, &dof));
4246     PetscCall(PetscSectionGetOffset(mesh->coneSection, p, &off));
4247     for (c = off; c < off + dof; ++c) {
4248       const PetscInt q = mesh->cones[c];
4249       PetscInt       offS;
4250 
4251       PetscCall(PetscSectionGetOffset(mesh->supportSection, q, &offS));
4252 
4253       mesh->supports[offS + offsets[q]] = p;
4254       ++offsets[q];
4255     }
4256   }
4257   PetscCall(PetscFree(offsets));
4258   PetscCall(PetscLogEventEnd(DMPLEX_Symmetrize, dm, 0, 0, 0));
4259   PetscFunctionReturn(PETSC_SUCCESS);
4260 }
4261 
4262 static PetscErrorCode DMPlexCreateDepthStratum(DM dm, DMLabel label, PetscInt depth, PetscInt pStart, PetscInt pEnd)
4263 {
4264   IS stratumIS;
4265 
4266   PetscFunctionBegin;
4267   if (pStart >= pEnd) PetscFunctionReturn(PETSC_SUCCESS);
4268   if (PetscDefined(USE_DEBUG)) {
4269     PetscInt  qStart, qEnd, numLevels, level;
4270     PetscBool overlap = PETSC_FALSE;
4271     PetscCall(DMLabelGetNumValues(label, &numLevels));
4272     for (level = 0; level < numLevels; level++) {
4273       PetscCall(DMLabelGetStratumBounds(label, level, &qStart, &qEnd));
4274       if ((pStart >= qStart && pStart < qEnd) || (pEnd > qStart && pEnd <= qEnd)) {
4275         overlap = PETSC_TRUE;
4276         break;
4277       }
4278     }
4279     PetscCheck(!overlap, PETSC_COMM_SELF, PETSC_ERR_PLIB, "New depth %" PetscInt_FMT " range [%" PetscInt_FMT ",%" PetscInt_FMT ") overlaps with depth %" PetscInt_FMT " range [%" PetscInt_FMT ",%" PetscInt_FMT ")", depth, pStart, pEnd, level, qStart, qEnd);
4280   }
4281   PetscCall(ISCreateStride(PETSC_COMM_SELF, pEnd - pStart, pStart, 1, &stratumIS));
4282   PetscCall(DMLabelSetStratumIS(label, depth, stratumIS));
4283   PetscCall(ISDestroy(&stratumIS));
4284   PetscFunctionReturn(PETSC_SUCCESS);
4285 }
4286 
4287 static PetscErrorCode DMPlexStratify_CellType_Private(DM dm, DMLabel label)
4288 {
4289   PetscInt *pMin, *pMax;
4290   PetscInt  pStart, pEnd;
4291   PetscInt  dmin = PETSC_INT_MAX, dmax = PETSC_INT_MIN;
4292 
4293   PetscFunctionBegin;
4294   {
4295     DMLabel label2;
4296 
4297     PetscCall(DMPlexGetCellTypeLabel(dm, &label2));
4298     PetscCall(PetscObjectViewFromOptions((PetscObject)label2, NULL, "-ct_view"));
4299   }
4300   PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
4301   for (PetscInt p = pStart; p < pEnd; ++p) {
4302     DMPolytopeType ct;
4303 
4304     PetscCall(DMPlexGetCellType(dm, p, &ct));
4305     dmin = PetscMin(DMPolytopeTypeGetDim(ct), dmin);
4306     dmax = PetscMax(DMPolytopeTypeGetDim(ct), dmax);
4307   }
4308   PetscCall(PetscMalloc2(dmax + 1, &pMin, dmax + 1, &pMax));
4309   for (PetscInt d = dmin; d <= dmax; ++d) {
4310     pMin[d] = PETSC_INT_MAX;
4311     pMax[d] = PETSC_INT_MIN;
4312   }
4313   for (PetscInt p = pStart; p < pEnd; ++p) {
4314     DMPolytopeType ct;
4315     PetscInt       d;
4316 
4317     PetscCall(DMPlexGetCellType(dm, p, &ct));
4318     d       = DMPolytopeTypeGetDim(ct);
4319     pMin[d] = PetscMin(p, pMin[d]);
4320     pMax[d] = PetscMax(p, pMax[d]);
4321   }
4322   for (PetscInt d = dmin; d <= dmax; ++d) {
4323     if (pMin[d] > pMax[d]) continue;
4324     PetscCall(DMPlexCreateDepthStratum(dm, label, d, pMin[d], pMax[d] + 1));
4325   }
4326   PetscCall(PetscFree2(pMin, pMax));
4327   PetscFunctionReturn(PETSC_SUCCESS);
4328 }
4329 
4330 static PetscErrorCode DMPlexStratify_Topological_Private(DM dm, DMLabel label)
4331 {
4332   PetscInt pStart, pEnd;
4333   PetscInt numRoots = 0, numLeaves = 0;
4334 
4335   PetscFunctionBegin;
4336   PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
4337   {
4338     /* Initialize roots and count leaves */
4339     PetscInt sMin = PETSC_INT_MAX;
4340     PetscInt sMax = PETSC_INT_MIN;
4341     PetscInt coneSize, supportSize;
4342 
4343     for (PetscInt p = pStart; p < pEnd; ++p) {
4344       PetscCall(DMPlexGetConeSize(dm, p, &coneSize));
4345       PetscCall(DMPlexGetSupportSize(dm, p, &supportSize));
4346       if (!coneSize && supportSize) {
4347         sMin = PetscMin(p, sMin);
4348         sMax = PetscMax(p, sMax);
4349         ++numRoots;
4350       } else if (!supportSize && coneSize) {
4351         ++numLeaves;
4352       } else if (!supportSize && !coneSize) {
4353         /* Isolated points */
4354         sMin = PetscMin(p, sMin);
4355         sMax = PetscMax(p, sMax);
4356       }
4357     }
4358     PetscCall(DMPlexCreateDepthStratum(dm, label, 0, sMin, sMax + 1));
4359   }
4360 
4361   if (numRoots + numLeaves == (pEnd - pStart)) {
4362     PetscInt sMin = PETSC_INT_MAX;
4363     PetscInt sMax = PETSC_INT_MIN;
4364     PetscInt coneSize, supportSize;
4365 
4366     for (PetscInt p = pStart; p < pEnd; ++p) {
4367       PetscCall(DMPlexGetConeSize(dm, p, &coneSize));
4368       PetscCall(DMPlexGetSupportSize(dm, p, &supportSize));
4369       if (!supportSize && coneSize) {
4370         sMin = PetscMin(p, sMin);
4371         sMax = PetscMax(p, sMax);
4372       }
4373     }
4374     PetscCall(DMPlexCreateDepthStratum(dm, label, 1, sMin, sMax + 1));
4375   } else {
4376     PetscInt level = 0;
4377     PetscInt qStart, qEnd;
4378 
4379     PetscCall(DMLabelGetStratumBounds(label, level, &qStart, &qEnd));
4380     while (qEnd > qStart) {
4381       PetscInt sMin = PETSC_INT_MAX;
4382       PetscInt sMax = PETSC_INT_MIN;
4383 
4384       for (PetscInt q = qStart; q < qEnd; ++q) {
4385         const PetscInt *support;
4386         PetscInt        supportSize;
4387 
4388         PetscCall(DMPlexGetSupportSize(dm, q, &supportSize));
4389         PetscCall(DMPlexGetSupport(dm, q, &support));
4390         for (PetscInt s = 0; s < supportSize; ++s) {
4391           sMin = PetscMin(support[s], sMin);
4392           sMax = PetscMax(support[s], sMax);
4393         }
4394       }
4395       PetscCall(DMLabelGetNumValues(label, &level));
4396       PetscCall(DMPlexCreateDepthStratum(dm, label, level, sMin, sMax + 1));
4397       PetscCall(DMLabelGetStratumBounds(label, level, &qStart, &qEnd));
4398     }
4399   }
4400   PetscFunctionReturn(PETSC_SUCCESS);
4401 }
4402 
4403 /*@
4404   DMPlexStratify - Computes the strata for all points in the `DMPLEX`
4405 
4406   Collective
4407 
4408   Input Parameter:
4409 . dm - The `DMPLEX`
4410 
4411   Level: beginner
4412 
4413   Notes:
4414   The strata group all points of the same grade, and this function calculates the strata. This
4415   grade can be seen as the height (or depth) of the point in the DAG.
4416 
4417   The DAG for most topologies is a graded poset (https://en.wikipedia.org/wiki/Graded_poset), and
4418   can be illustrated by a Hasse Diagram (https://en.wikipedia.org/wiki/Hasse_diagram).
4419   Concretely, `DMPlexStratify()` creates a new label named "depth" containing the depth in the DAG of each point. For cell-vertex
4420   meshes, vertices are depth 0 and cells are depth 1. For fully interpolated meshes, depth 0 for vertices, 1 for edges, and so on
4421   until cells have depth equal to the dimension of the mesh. The depth label can be accessed through `DMPlexGetDepthLabel()` or `DMPlexGetDepthStratum()`, or
4422   manually via `DMGetLabel()`.  The height is defined implicitly by height = maxDimension - depth, and can be accessed
4423   via `DMPlexGetHeightStratum()`.  For example, cells have height 0 and faces have height 1.
4424 
4425   The depth of a point is calculated by executing a breadth-first search (BFS) on the DAG. This could produce surprising results
4426   if run on a partially interpolated mesh, meaning one that had some edges and faces, but not others. For example, suppose that
4427   we had a mesh consisting of one triangle (c0) and three vertices (v0, v1, v2), and only one edge is on the boundary so we choose
4428   to interpolate only that one (e0), so that
4429 .vb
4430   cone(c0) = {e0, v2}
4431   cone(e0) = {v0, v1}
4432 .ve
4433   If `DMPlexStratify()` is run on this mesh, it will give depths
4434 .vb
4435    depth 0 = {v0, v1, v2}
4436    depth 1 = {e0, c0}
4437 .ve
4438   where the triangle has been given depth 1, instead of 2, because it is reachable from vertex v2.
4439 
4440   `DMPlexStratify()` should be called after all calls to `DMPlexSymmetrize()`
4441 
4442 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreate()`, `DMPlexSymmetrize()`, `DMPlexComputeCellTypes()`
4443 @*/
4444 PetscErrorCode DMPlexStratify(DM dm)
4445 {
4446   DM_Plex  *mesh = (DM_Plex *)dm->data;
4447   DMLabel   label;
4448   PetscBool flg = PETSC_FALSE;
4449 
4450   PetscFunctionBegin;
4451   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4452   PetscCall(PetscLogEventBegin(DMPLEX_Stratify, dm, 0, 0, 0));
4453 
4454   // Create depth label
4455   PetscCall(DMRemoveLabel(dm, "depth", NULL));
4456   PetscCall(DMCreateLabel(dm, "depth"));
4457   PetscCall(DMPlexGetDepthLabel(dm, &label));
4458 
4459   PetscCall(PetscOptionsGetBool(NULL, dm->hdr.prefix, "-dm_plex_stratify_celltype", &flg, NULL));
4460   if (flg) PetscCall(DMPlexStratify_CellType_Private(dm, label));
4461   else PetscCall(DMPlexStratify_Topological_Private(dm, label));
4462 
4463   { /* just in case there is an empty process */
4464     PetscInt numValues, maxValues = 0, v;
4465 
4466     PetscCall(DMLabelGetNumValues(label, &numValues));
4467     PetscCallMPI(MPIU_Allreduce(&numValues, &maxValues, 1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject)dm)));
4468     for (v = numValues; v < maxValues; v++) PetscCall(DMLabelAddStratum(label, v));
4469   }
4470   PetscCall(PetscObjectStateGet((PetscObject)label, &mesh->depthState));
4471   PetscCall(PetscLogEventEnd(DMPLEX_Stratify, dm, 0, 0, 0));
4472   PetscFunctionReturn(PETSC_SUCCESS);
4473 }
4474 
4475 PetscErrorCode DMPlexComputeCellType_Internal(DM dm, PetscInt p, PetscInt pdepth, DMPolytopeType *pt)
4476 {
4477   DMPolytopeType ct = DM_POLYTOPE_UNKNOWN;
4478   PetscInt       dim, depth, pheight, coneSize;
4479 
4480   PetscFunctionBeginHot;
4481   PetscCall(DMGetDimension(dm, &dim));
4482   PetscCall(DMPlexGetDepth(dm, &depth));
4483   PetscCall(DMPlexGetConeSize(dm, p, &coneSize));
4484   pheight = depth - pdepth;
4485   if (depth <= 1) {
4486     switch (pdepth) {
4487     case 0:
4488       ct = DM_POLYTOPE_POINT;
4489       break;
4490     case 1:
4491       switch (coneSize) {
4492       case 2:
4493         ct = DM_POLYTOPE_SEGMENT;
4494         break;
4495       case 3:
4496         ct = DM_POLYTOPE_TRIANGLE;
4497         break;
4498       case 4:
4499         switch (dim) {
4500         case 2:
4501           ct = DM_POLYTOPE_QUADRILATERAL;
4502           break;
4503         case 3:
4504           ct = DM_POLYTOPE_TETRAHEDRON;
4505           break;
4506         default:
4507           break;
4508         }
4509         break;
4510       case 5:
4511         ct = DM_POLYTOPE_PYRAMID;
4512         break;
4513       case 6:
4514         ct = DM_POLYTOPE_TRI_PRISM_TENSOR;
4515         break;
4516       case 8:
4517         ct = DM_POLYTOPE_HEXAHEDRON;
4518         break;
4519       default:
4520         break;
4521       }
4522     }
4523   } else {
4524     if (pdepth == 0) {
4525       ct = DM_POLYTOPE_POINT;
4526     } else if (pheight == 0) {
4527       switch (dim) {
4528       case 1:
4529         switch (coneSize) {
4530         case 2:
4531           ct = DM_POLYTOPE_SEGMENT;
4532           break;
4533         default:
4534           break;
4535         }
4536         break;
4537       case 2:
4538         switch (coneSize) {
4539         case 3:
4540           ct = DM_POLYTOPE_TRIANGLE;
4541           break;
4542         case 4:
4543           ct = DM_POLYTOPE_QUADRILATERAL;
4544           break;
4545         default:
4546           break;
4547         }
4548         break;
4549       case 3:
4550         switch (coneSize) {
4551         case 4:
4552           ct = DM_POLYTOPE_TETRAHEDRON;
4553           break;
4554         case 5: {
4555           const PetscInt *cone;
4556           PetscInt        faceConeSize;
4557 
4558           PetscCall(DMPlexGetCone(dm, p, &cone));
4559           PetscCall(DMPlexGetConeSize(dm, cone[0], &faceConeSize));
4560           switch (faceConeSize) {
4561           case 3:
4562             ct = DM_POLYTOPE_TRI_PRISM_TENSOR;
4563             break;
4564           case 4:
4565             ct = DM_POLYTOPE_PYRAMID;
4566             break;
4567           }
4568         } break;
4569         case 6:
4570           ct = DM_POLYTOPE_HEXAHEDRON;
4571           break;
4572         default:
4573           break;
4574         }
4575         break;
4576       default:
4577         break;
4578       }
4579     } else if (pheight > 0) {
4580       switch (coneSize) {
4581       case 2:
4582         ct = DM_POLYTOPE_SEGMENT;
4583         break;
4584       case 3:
4585         ct = DM_POLYTOPE_TRIANGLE;
4586         break;
4587       case 4:
4588         ct = DM_POLYTOPE_QUADRILATERAL;
4589         break;
4590       default:
4591         break;
4592       }
4593     }
4594   }
4595   *pt = ct;
4596   PetscFunctionReturn(PETSC_SUCCESS);
4597 }
4598 
4599 /*@
4600   DMPlexComputeCellTypes - Infer the polytope type of every cell using its dimension and cone size.
4601 
4602   Collective
4603 
4604   Input Parameter:
4605 . dm - The `DMPLEX`
4606 
4607   Level: developer
4608 
4609   Note:
4610   This function is normally called automatically when a cell type is requested. It creates an
4611   internal `DMLabel` named "celltype" which can be directly accessed using `DMGetLabel()`. A user may disable
4612   automatic creation by creating the label manually, using `DMCreateLabel`(dm, "celltype").
4613 
4614   `DMPlexComputeCellTypes()` should be called after all calls to `DMPlexSymmetrize()` and `DMPlexStratify()`
4615 
4616 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreate()`, `DMPlexSymmetrize()`, `DMPlexStratify()`, `DMGetLabel()`, `DMCreateLabel()`
4617 @*/
4618 PetscErrorCode DMPlexComputeCellTypes(DM dm)
4619 {
4620   DM_Plex *mesh;
4621   DMLabel  ctLabel;
4622   PetscInt pStart, pEnd, p;
4623 
4624   PetscFunctionBegin;
4625   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4626   mesh = (DM_Plex *)dm->data;
4627   PetscCall(DMCreateLabel(dm, "celltype"));
4628   PetscCall(DMPlexGetCellTypeLabel(dm, &ctLabel));
4629   PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
4630   PetscCall(PetscFree(mesh->cellTypes));
4631   PetscCall(PetscMalloc1(pEnd - pStart, &mesh->cellTypes));
4632   for (p = pStart; p < pEnd; ++p) {
4633     DMPolytopeType ct = DM_POLYTOPE_UNKNOWN;
4634     PetscInt       pdepth;
4635 
4636     PetscCall(DMPlexGetPointDepth(dm, p, &pdepth));
4637     PetscCall(DMPlexComputeCellType_Internal(dm, p, pdepth, &ct));
4638     PetscCheck(ct != DM_POLYTOPE_UNKNOWN && ct != DM_POLYTOPE_UNKNOWN_CELL && ct != DM_POLYTOPE_UNKNOWN_FACE, PETSC_COMM_SELF, PETSC_ERR_SUP, "Point %" PetscInt_FMT " has invalid celltype (%s)", p, DMPolytopeTypes[ct]);
4639     PetscCall(DMLabelSetValue(ctLabel, p, ct));
4640     mesh->cellTypes[p - pStart].value_as_uint8 = (uint8_t)ct;
4641   }
4642   PetscCall(PetscObjectStateGet((PetscObject)ctLabel, &mesh->celltypeState));
4643   PetscCall(PetscObjectViewFromOptions((PetscObject)ctLabel, NULL, "-dm_plex_celltypes_view"));
4644   PetscFunctionReturn(PETSC_SUCCESS);
4645 }
4646 
4647 /*@C
4648   DMPlexGetJoin - Get an array for the join of the set of points
4649 
4650   Not Collective
4651 
4652   Input Parameters:
4653 + dm        - The `DMPLEX` object
4654 . numPoints - The number of input points for the join
4655 - points    - The input points
4656 
4657   Output Parameters:
4658 + numCoveredPoints - The number of points in the join
4659 - coveredPoints    - The points in the join
4660 
4661   Level: intermediate
4662 
4663   Note:
4664   Currently, this is restricted to a single level join
4665 
4666   Fortran Notes:
4667   `converedPoints` must be declared with
4668 .vb
4669   PetscInt, pointer :: coveredPints(:)
4670 .ve
4671 
4672   The `numCoveredPoints` argument is not present in the Fortran binding.
4673 
4674 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexRestoreJoin()`, `DMPlexGetMeet()`
4675 @*/
4676 PetscErrorCode DMPlexGetJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt *coveredPoints[])
4677 {
4678   DM_Plex  *mesh = (DM_Plex *)dm->data;
4679   PetscInt *join[2];
4680   PetscInt  joinSize, i = 0;
4681   PetscInt  dof, off, p, c, m;
4682   PetscInt  maxSupportSize;
4683 
4684   PetscFunctionBegin;
4685   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4686   PetscAssertPointer(points, 3);
4687   PetscAssertPointer(numCoveredPoints, 4);
4688   PetscAssertPointer(coveredPoints, 5);
4689   PetscCall(PetscSectionGetMaxDof(mesh->supportSection, &maxSupportSize));
4690   PetscCall(DMGetWorkArray(dm, maxSupportSize, MPIU_INT, &join[0]));
4691   PetscCall(DMGetWorkArray(dm, maxSupportSize, MPIU_INT, &join[1]));
4692   /* Copy in support of first point */
4693   PetscCall(PetscSectionGetDof(mesh->supportSection, points[0], &dof));
4694   PetscCall(PetscSectionGetOffset(mesh->supportSection, points[0], &off));
4695   for (joinSize = 0; joinSize < dof; ++joinSize) join[i][joinSize] = mesh->supports[off + joinSize];
4696   /* Check each successive support */
4697   for (p = 1; p < numPoints; ++p) {
4698     PetscInt newJoinSize = 0;
4699 
4700     PetscCall(PetscSectionGetDof(mesh->supportSection, points[p], &dof));
4701     PetscCall(PetscSectionGetOffset(mesh->supportSection, points[p], &off));
4702     for (c = 0; c < dof; ++c) {
4703       const PetscInt point = mesh->supports[off + c];
4704 
4705       for (m = 0; m < joinSize; ++m) {
4706         if (point == join[i][m]) {
4707           join[1 - i][newJoinSize++] = point;
4708           break;
4709         }
4710       }
4711     }
4712     joinSize = newJoinSize;
4713     i        = 1 - i;
4714   }
4715   *numCoveredPoints = joinSize;
4716   *coveredPoints    = join[i];
4717   PetscCall(DMRestoreWorkArray(dm, maxSupportSize, MPIU_INT, &join[1 - i]));
4718   PetscFunctionReturn(PETSC_SUCCESS);
4719 }
4720 
4721 /*@C
4722   DMPlexRestoreJoin - Restore an array for the join of the set of points obtained with `DMPlexGetJoin()`
4723 
4724   Not Collective
4725 
4726   Input Parameters:
4727 + dm        - The `DMPLEX` object
4728 . numPoints - The number of input points for the join
4729 - points    - The input points
4730 
4731   Output Parameters:
4732 + numCoveredPoints - The number of points in the join
4733 - coveredPoints    - The points in the join
4734 
4735   Level: intermediate
4736 
4737   Fortran Notes:
4738   The `numCoveredPoints` argument is not present in the Fortran binding since it is internal to the array.
4739 
4740 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetJoin()`, `DMPlexGetFullJoin()`, `DMPlexGetMeet()`
4741 @*/
4742 PetscErrorCode DMPlexRestoreJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt *coveredPoints[])
4743 {
4744   PetscFunctionBegin;
4745   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4746   if (points) PetscAssertPointer(points, 3);
4747   if (numCoveredPoints) PetscAssertPointer(numCoveredPoints, 4);
4748   PetscAssertPointer(coveredPoints, 5);
4749   PetscCall(DMRestoreWorkArray(dm, 0, MPIU_INT, (void *)coveredPoints));
4750   if (numCoveredPoints) *numCoveredPoints = 0;
4751   PetscFunctionReturn(PETSC_SUCCESS);
4752 }
4753 
4754 /*@C
4755   DMPlexGetFullJoin - Get an array for the join of the set of points
4756 
4757   Not Collective
4758 
4759   Input Parameters:
4760 + dm        - The `DMPLEX` object
4761 . numPoints - The number of input points for the join
4762 - points    - The input points, its length is `numPoints`
4763 
4764   Output Parameters:
4765 + numCoveredPoints - The number of points in the join
4766 - coveredPoints    - The points in the join, its length is `numCoveredPoints`
4767 
4768   Level: intermediate
4769 
4770   Fortran Notes:
4771   `points` and `converedPoints` must be declared with
4772 .vb
4773   PetscInt, pointer :: points(:)
4774   PetscInt, pointer :: coveredPints(:)
4775 .ve
4776 
4777   The `numCoveredPoints` argument is not present in the Fortran binding.
4778 
4779 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetJoin()`, `DMPlexRestoreJoin()`, `DMPlexGetMeet()`
4780 @*/
4781 PetscErrorCode DMPlexGetFullJoin(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt *coveredPoints[])
4782 {
4783   PetscInt *offsets, **closures;
4784   PetscInt *join[2];
4785   PetscInt  depth = 0, maxSize, joinSize = 0, i = 0;
4786   PetscInt  p, d, c, m, ms;
4787 
4788   PetscFunctionBegin;
4789   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4790   PetscAssertPointer(points, 3);
4791   PetscAssertPointer(numCoveredPoints, 4);
4792   PetscAssertPointer(coveredPoints, 5);
4793 
4794   PetscCall(DMPlexGetDepth(dm, &depth));
4795   PetscCall(PetscCalloc1(numPoints, &closures));
4796   PetscCall(DMGetWorkArray(dm, numPoints * (depth + 2), MPIU_INT, &offsets));
4797   PetscCall(DMPlexGetMaxSizes(dm, NULL, &ms));
4798   maxSize = (ms > 1) ? ((PetscPowInt(ms, depth + 1) - 1) / (ms - 1)) : depth + 1;
4799   PetscCall(DMGetWorkArray(dm, maxSize, MPIU_INT, &join[0]));
4800   PetscCall(DMGetWorkArray(dm, maxSize, MPIU_INT, &join[1]));
4801 
4802   for (p = 0; p < numPoints; ++p) {
4803     PetscInt closureSize;
4804 
4805     PetscCall(DMPlexGetTransitiveClosure(dm, points[p], PETSC_FALSE, &closureSize, &closures[p]));
4806 
4807     offsets[p * (depth + 2) + 0] = 0;
4808     for (d = 0; d < depth + 1; ++d) {
4809       PetscInt pStart, pEnd, i;
4810 
4811       PetscCall(DMPlexGetDepthStratum(dm, d, &pStart, &pEnd));
4812       for (i = offsets[p * (depth + 2) + d]; i < closureSize; ++i) {
4813         if ((pStart > closures[p][i * 2]) || (pEnd <= closures[p][i * 2])) {
4814           offsets[p * (depth + 2) + d + 1] = i;
4815           break;
4816         }
4817       }
4818       if (i == closureSize) offsets[p * (depth + 2) + d + 1] = i;
4819     }
4820     PetscCheck(offsets[p * (depth + 2) + depth + 1] == closureSize, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Total size of closure %" PetscInt_FMT " should be %" PetscInt_FMT, offsets[p * (depth + 2) + depth + 1], closureSize);
4821   }
4822   for (d = 0; d < depth + 1; ++d) {
4823     PetscInt dof;
4824 
4825     /* Copy in support of first point */
4826     dof = offsets[d + 1] - offsets[d];
4827     for (joinSize = 0; joinSize < dof; ++joinSize) join[i][joinSize] = closures[0][(offsets[d] + joinSize) * 2];
4828     /* Check each successive cone */
4829     for (p = 1; p < numPoints && joinSize; ++p) {
4830       PetscInt newJoinSize = 0;
4831 
4832       dof = offsets[p * (depth + 2) + d + 1] - offsets[p * (depth + 2) + d];
4833       for (c = 0; c < dof; ++c) {
4834         const PetscInt point = closures[p][(offsets[p * (depth + 2) + d] + c) * 2];
4835 
4836         for (m = 0; m < joinSize; ++m) {
4837           if (point == join[i][m]) {
4838             join[1 - i][newJoinSize++] = point;
4839             break;
4840           }
4841         }
4842       }
4843       joinSize = newJoinSize;
4844       i        = 1 - i;
4845     }
4846     if (joinSize) break;
4847   }
4848   *numCoveredPoints = joinSize;
4849   *coveredPoints    = join[i];
4850   for (p = 0; p < numPoints; ++p) PetscCall(DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_FALSE, NULL, &closures[p]));
4851   PetscCall(PetscFree(closures));
4852   PetscCall(DMRestoreWorkArray(dm, numPoints * (depth + 2), MPIU_INT, &offsets));
4853   PetscCall(DMRestoreWorkArray(dm, ms, MPIU_INT, &join[1 - i]));
4854   PetscFunctionReturn(PETSC_SUCCESS);
4855 }
4856 
4857 /*@C
4858   DMPlexGetMeet - Get an array for the meet of the set of points
4859 
4860   Not Collective
4861 
4862   Input Parameters:
4863 + dm        - The `DMPLEX` object
4864 . numPoints - The number of input points for the meet
4865 - points    - The input points, of length `numPoints`
4866 
4867   Output Parameters:
4868 + numCoveringPoints - The number of points in the meet
4869 - coveringPoints    - The points in the meet, of length `numCoveringPoints`
4870 
4871   Level: intermediate
4872 
4873   Note:
4874   Currently, this is restricted to a single level meet
4875 
4876   Fortran Notes:
4877   `coveringPoints` must be declared with
4878 .vb
4879   PetscInt, pointer :: coveringPoints(:)
4880 .ve
4881 
4882   The `numCoveredPoints` argument is not present in the Fortran binding since it is internal to the array.
4883 
4884 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexRestoreMeet()`, `DMPlexGetJoin()`
4885 @*/
4886 PetscErrorCode DMPlexGetMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveringPoints, const PetscInt *coveringPoints[])
4887 {
4888   DM_Plex  *mesh = (DM_Plex *)dm->data;
4889   PetscInt *meet[2];
4890   PetscInt  meetSize, i = 0;
4891   PetscInt  dof, off, p, c, m;
4892   PetscInt  maxConeSize;
4893 
4894   PetscFunctionBegin;
4895   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4896   PetscAssertPointer(points, 3);
4897   PetscAssertPointer(numCoveringPoints, 4);
4898   PetscAssertPointer(coveringPoints, 5);
4899   PetscCall(PetscSectionGetMaxDof(mesh->coneSection, &maxConeSize));
4900   PetscCall(DMGetWorkArray(dm, maxConeSize, MPIU_INT, &meet[0]));
4901   PetscCall(DMGetWorkArray(dm, maxConeSize, MPIU_INT, &meet[1]));
4902   /* Copy in cone of first point */
4903   PetscCall(PetscSectionGetDof(mesh->coneSection, points[0], &dof));
4904   PetscCall(PetscSectionGetOffset(mesh->coneSection, points[0], &off));
4905   for (meetSize = 0; meetSize < dof; ++meetSize) meet[i][meetSize] = mesh->cones[off + meetSize];
4906   /* Check each successive cone */
4907   for (p = 1; p < numPoints; ++p) {
4908     PetscInt newMeetSize = 0;
4909 
4910     PetscCall(PetscSectionGetDof(mesh->coneSection, points[p], &dof));
4911     PetscCall(PetscSectionGetOffset(mesh->coneSection, points[p], &off));
4912     for (c = 0; c < dof; ++c) {
4913       const PetscInt point = mesh->cones[off + c];
4914 
4915       for (m = 0; m < meetSize; ++m) {
4916         if (point == meet[i][m]) {
4917           meet[1 - i][newMeetSize++] = point;
4918           break;
4919         }
4920       }
4921     }
4922     meetSize = newMeetSize;
4923     i        = 1 - i;
4924   }
4925   *numCoveringPoints = meetSize;
4926   *coveringPoints    = meet[i];
4927   PetscCall(DMRestoreWorkArray(dm, maxConeSize, MPIU_INT, &meet[1 - i]));
4928   PetscFunctionReturn(PETSC_SUCCESS);
4929 }
4930 
4931 /*@C
4932   DMPlexRestoreMeet - Restore an array for the meet of the set of points obtained with `DMPlexGetMeet()`
4933 
4934   Not Collective
4935 
4936   Input Parameters:
4937 + dm        - The `DMPLEX` object
4938 . numPoints - The number of input points for the meet
4939 - points    - The input points
4940 
4941   Output Parameters:
4942 + numCoveredPoints - The number of points in the meet
4943 - coveredPoints    - The points in the meet
4944 
4945   Level: intermediate
4946 
4947   Fortran Notes:
4948   The `numCoveredPoints` argument is not present in the Fortran binding since it is internal to the array.
4949 
4950 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetMeet()`, `DMPlexGetFullMeet()`, `DMPlexGetJoin()`
4951 @*/
4952 PetscErrorCode DMPlexRestoreMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt *coveredPoints[])
4953 {
4954   PetscFunctionBegin;
4955   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4956   if (points) PetscAssertPointer(points, 3);
4957   if (numCoveredPoints) PetscAssertPointer(numCoveredPoints, 4);
4958   PetscAssertPointer(coveredPoints, 5);
4959   PetscCall(DMRestoreWorkArray(dm, 0, MPIU_INT, (void *)coveredPoints));
4960   if (numCoveredPoints) *numCoveredPoints = 0;
4961   PetscFunctionReturn(PETSC_SUCCESS);
4962 }
4963 
4964 /*@C
4965   DMPlexGetFullMeet - Get an array for the meet of the set of points
4966 
4967   Not Collective
4968 
4969   Input Parameters:
4970 + dm        - The `DMPLEX` object
4971 . numPoints - The number of input points for the meet
4972 - points    - The input points, of length  `numPoints`
4973 
4974   Output Parameters:
4975 + numCoveredPoints - The number of points in the meet
4976 - coveredPoints    - The points in the meet, of length  `numCoveredPoints`
4977 
4978   Level: intermediate
4979 
4980   Fortran Notes:
4981   `points` and `coveredPoints` must be declared with
4982 .vb
4983   PetscInt, pointer :: points(:)
4984   PetscInt, pointer :: coveredPoints(:)
4985 .ve
4986 
4987   The `numCoveredPoints` argument is not present in the Fortran binding since it is internal to the array.
4988 
4989 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetMeet()`, `DMPlexRestoreMeet()`, `DMPlexGetJoin()`
4990 @*/
4991 PetscErrorCode DMPlexGetFullMeet(DM dm, PetscInt numPoints, const PetscInt points[], PetscInt *numCoveredPoints, const PetscInt *coveredPoints[])
4992 {
4993   PetscInt *offsets, **closures;
4994   PetscInt *meet[2];
4995   PetscInt  height = 0, maxSize, meetSize = 0, i = 0;
4996   PetscInt  p, h, c, m, mc;
4997 
4998   PetscFunctionBegin;
4999   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5000   PetscAssertPointer(points, 3);
5001   PetscAssertPointer(numCoveredPoints, 4);
5002   PetscAssertPointer(coveredPoints, 5);
5003 
5004   PetscCall(DMPlexGetDepth(dm, &height));
5005   PetscCall(PetscMalloc1(numPoints, &closures));
5006   PetscCall(DMGetWorkArray(dm, numPoints * (height + 2), MPIU_INT, &offsets));
5007   PetscCall(DMPlexGetMaxSizes(dm, &mc, NULL));
5008   maxSize = (mc > 1) ? ((PetscPowInt(mc, height + 1) - 1) / (mc - 1)) : height + 1;
5009   PetscCall(DMGetWorkArray(dm, maxSize, MPIU_INT, &meet[0]));
5010   PetscCall(DMGetWorkArray(dm, maxSize, MPIU_INT, &meet[1]));
5011 
5012   for (p = 0; p < numPoints; ++p) {
5013     PetscInt closureSize;
5014 
5015     PetscCall(DMPlexGetTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closures[p]));
5016 
5017     offsets[p * (height + 2) + 0] = 0;
5018     for (h = 0; h < height + 1; ++h) {
5019       PetscInt pStart, pEnd, i;
5020 
5021       PetscCall(DMPlexGetHeightStratum(dm, h, &pStart, &pEnd));
5022       for (i = offsets[p * (height + 2) + h]; i < closureSize; ++i) {
5023         if ((pStart > closures[p][i * 2]) || (pEnd <= closures[p][i * 2])) {
5024           offsets[p * (height + 2) + h + 1] = i;
5025           break;
5026         }
5027       }
5028       if (i == closureSize) offsets[p * (height + 2) + h + 1] = i;
5029     }
5030     PetscCheck(offsets[p * (height + 2) + height + 1] == closureSize, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Total size of closure %" PetscInt_FMT " should be %" PetscInt_FMT, offsets[p * (height + 2) + height + 1], closureSize);
5031   }
5032   for (h = 0; h < height + 1; ++h) {
5033     PetscInt dof;
5034 
5035     /* Copy in cone of first point */
5036     dof = offsets[h + 1] - offsets[h];
5037     for (meetSize = 0; meetSize < dof; ++meetSize) meet[i][meetSize] = closures[0][(offsets[h] + meetSize) * 2];
5038     /* Check each successive cone */
5039     for (p = 1; p < numPoints && meetSize; ++p) {
5040       PetscInt newMeetSize = 0;
5041 
5042       dof = offsets[p * (height + 2) + h + 1] - offsets[p * (height + 2) + h];
5043       for (c = 0; c < dof; ++c) {
5044         const PetscInt point = closures[p][(offsets[p * (height + 2) + h] + c) * 2];
5045 
5046         for (m = 0; m < meetSize; ++m) {
5047           if (point == meet[i][m]) {
5048             meet[1 - i][newMeetSize++] = point;
5049             break;
5050           }
5051         }
5052       }
5053       meetSize = newMeetSize;
5054       i        = 1 - i;
5055     }
5056     if (meetSize) break;
5057   }
5058   *numCoveredPoints = meetSize;
5059   *coveredPoints    = meet[i];
5060   for (p = 0; p < numPoints; ++p) PetscCall(DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_TRUE, NULL, &closures[p]));
5061   PetscCall(PetscFree(closures));
5062   PetscCall(DMRestoreWorkArray(dm, numPoints * (height + 2), MPIU_INT, &offsets));
5063   PetscCall(DMRestoreWorkArray(dm, mc, MPIU_INT, &meet[1 - i]));
5064   PetscFunctionReturn(PETSC_SUCCESS);
5065 }
5066 
5067 /*@
5068   DMPlexEqual - Determine if two `DM` have the same topology
5069 
5070   Not Collective
5071 
5072   Input Parameters:
5073 + dmA - A `DMPLEX` object
5074 - dmB - A `DMPLEX` object
5075 
5076   Output Parameter:
5077 . equal - `PETSC_TRUE` if the topologies are identical
5078 
5079   Level: intermediate
5080 
5081   Note:
5082   We are not solving graph isomorphism, so we do not permute.
5083 
5084 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetCone()`
5085 @*/
5086 PetscErrorCode DMPlexEqual(DM dmA, DM dmB, PetscBool *equal)
5087 {
5088   PetscInt depth, depthB, pStart, pEnd, pStartB, pEndB, p;
5089 
5090   PetscFunctionBegin;
5091   PetscValidHeaderSpecific(dmA, DM_CLASSID, 1);
5092   PetscValidHeaderSpecific(dmB, DM_CLASSID, 2);
5093   PetscAssertPointer(equal, 3);
5094 
5095   *equal = PETSC_FALSE;
5096   PetscCall(DMPlexGetDepth(dmA, &depth));
5097   PetscCall(DMPlexGetDepth(dmB, &depthB));
5098   if (depth != depthB) PetscFunctionReturn(PETSC_SUCCESS);
5099   PetscCall(DMPlexGetChart(dmA, &pStart, &pEnd));
5100   PetscCall(DMPlexGetChart(dmB, &pStartB, &pEndB));
5101   if ((pStart != pStartB) || (pEnd != pEndB)) PetscFunctionReturn(PETSC_SUCCESS);
5102   for (p = pStart; p < pEnd; ++p) {
5103     const PetscInt *cone, *coneB, *ornt, *orntB, *support, *supportB;
5104     PetscInt        coneSize, coneSizeB, c, supportSize, supportSizeB, s;
5105 
5106     PetscCall(DMPlexGetConeSize(dmA, p, &coneSize));
5107     PetscCall(DMPlexGetCone(dmA, p, &cone));
5108     PetscCall(DMPlexGetConeOrientation(dmA, p, &ornt));
5109     PetscCall(DMPlexGetConeSize(dmB, p, &coneSizeB));
5110     PetscCall(DMPlexGetCone(dmB, p, &coneB));
5111     PetscCall(DMPlexGetConeOrientation(dmB, p, &orntB));
5112     if (coneSize != coneSizeB) PetscFunctionReturn(PETSC_SUCCESS);
5113     for (c = 0; c < coneSize; ++c) {
5114       if (cone[c] != coneB[c]) PetscFunctionReturn(PETSC_SUCCESS);
5115       if (ornt[c] != orntB[c]) PetscFunctionReturn(PETSC_SUCCESS);
5116     }
5117     PetscCall(DMPlexGetSupportSize(dmA, p, &supportSize));
5118     PetscCall(DMPlexGetSupport(dmA, p, &support));
5119     PetscCall(DMPlexGetSupportSize(dmB, p, &supportSizeB));
5120     PetscCall(DMPlexGetSupport(dmB, p, &supportB));
5121     if (supportSize != supportSizeB) PetscFunctionReturn(PETSC_SUCCESS);
5122     for (s = 0; s < supportSize; ++s) {
5123       if (support[s] != supportB[s]) PetscFunctionReturn(PETSC_SUCCESS);
5124     }
5125   }
5126   *equal = PETSC_TRUE;
5127   PetscFunctionReturn(PETSC_SUCCESS);
5128 }
5129 
5130 /*@
5131   DMPlexGetNumFaceVertices - Returns the number of vertices on a face
5132 
5133   Not Collective
5134 
5135   Input Parameters:
5136 + dm         - The `DMPLEX`
5137 . cellDim    - The cell dimension
5138 - numCorners - The number of vertices on a cell
5139 
5140   Output Parameter:
5141 . numFaceVertices - The number of vertices on a face
5142 
5143   Level: developer
5144 
5145   Note:
5146   Of course this can only work for a restricted set of symmetric shapes
5147 
5148 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetCone()`
5149 @*/
5150 PetscErrorCode DMPlexGetNumFaceVertices(DM dm, PetscInt cellDim, PetscInt numCorners, PetscInt *numFaceVertices)
5151 {
5152   MPI_Comm comm;
5153 
5154   PetscFunctionBegin;
5155   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
5156   PetscAssertPointer(numFaceVertices, 4);
5157   switch (cellDim) {
5158   case 0:
5159     *numFaceVertices = 0;
5160     break;
5161   case 1:
5162     *numFaceVertices = 1;
5163     break;
5164   case 2:
5165     switch (numCorners) {
5166     case 3:                 /* triangle */
5167       *numFaceVertices = 2; /* Edge has 2 vertices */
5168       break;
5169     case 4:                 /* quadrilateral */
5170       *numFaceVertices = 2; /* Edge has 2 vertices */
5171       break;
5172     case 6:                 /* quadratic triangle, tri and quad cohesive Lagrange cells */
5173       *numFaceVertices = 3; /* Edge has 3 vertices */
5174       break;
5175     case 9:                 /* quadratic quadrilateral, quadratic quad cohesive Lagrange cells */
5176       *numFaceVertices = 3; /* Edge has 3 vertices */
5177       break;
5178     default:
5179       SETERRQ(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid number of face corners %" PetscInt_FMT " for dimension %" PetscInt_FMT, numCorners, cellDim);
5180     }
5181     break;
5182   case 3:
5183     switch (numCorners) {
5184     case 4:                 /* tetradehdron */
5185       *numFaceVertices = 3; /* Face has 3 vertices */
5186       break;
5187     case 6:                 /* tet cohesive cells */
5188       *numFaceVertices = 4; /* Face has 4 vertices */
5189       break;
5190     case 8:                 /* hexahedron */
5191       *numFaceVertices = 4; /* Face has 4 vertices */
5192       break;
5193     case 9:                 /* tet cohesive Lagrange cells */
5194       *numFaceVertices = 6; /* Face has 6 vertices */
5195       break;
5196     case 10:                /* quadratic tetrahedron */
5197       *numFaceVertices = 6; /* Face has 6 vertices */
5198       break;
5199     case 12:                /* hex cohesive Lagrange cells */
5200       *numFaceVertices = 6; /* Face has 6 vertices */
5201       break;
5202     case 18:                /* quadratic tet cohesive Lagrange cells */
5203       *numFaceVertices = 6; /* Face has 6 vertices */
5204       break;
5205     case 27:                /* quadratic hexahedron, quadratic hex cohesive Lagrange cells */
5206       *numFaceVertices = 9; /* Face has 9 vertices */
5207       break;
5208     default:
5209       SETERRQ(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid number of face corners %" PetscInt_FMT " for dimension %" PetscInt_FMT, numCorners, cellDim);
5210     }
5211     break;
5212   default:
5213     SETERRQ(comm, PETSC_ERR_ARG_OUTOFRANGE, "Invalid cell dimension %" PetscInt_FMT, cellDim);
5214   }
5215   PetscFunctionReturn(PETSC_SUCCESS);
5216 }
5217 
5218 /*@
5219   DMPlexGetDepthLabel - Get the `DMLabel` recording the depth of each point
5220 
5221   Not Collective
5222 
5223   Input Parameter:
5224 . dm - The `DMPLEX` object
5225 
5226   Output Parameter:
5227 . depthLabel - The `DMLabel` recording point depth
5228 
5229   Level: developer
5230 
5231 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetDepth()`, `DMPlexGetHeightStratum()`, `DMPlexGetDepthStratum()`, `DMPlexGetPointDepth()`,
5232 @*/
5233 PetscErrorCode DMPlexGetDepthLabel(DM dm, DMLabel *depthLabel)
5234 {
5235   PetscFunctionBegin;
5236   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5237   PetscAssertPointer(depthLabel, 2);
5238   *depthLabel = dm->depthLabel;
5239   PetscFunctionReturn(PETSC_SUCCESS);
5240 }
5241 
5242 /*@
5243   DMPlexGetDepth - Get the depth of the DAG representing this mesh
5244 
5245   Not Collective
5246 
5247   Input Parameter:
5248 . dm - The `DMPLEX` object
5249 
5250   Output Parameter:
5251 . depth - The number of strata (breadth first levels) in the DAG
5252 
5253   Level: developer
5254 
5255   Notes:
5256   This returns maximum of point depths over all points, i.e. maximum value of the label returned by `DMPlexGetDepthLabel()`.
5257 
5258   The point depth is described more in detail in `DMPlexGetDepthStratum()`.
5259 
5260   An empty mesh gives -1.
5261 
5262 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetDepthLabel()`, `DMPlexGetDepthStratum()`, `DMPlexGetPointDepth()`, `DMPlexSymmetrize()`
5263 @*/
5264 PetscErrorCode DMPlexGetDepth(DM dm, PetscInt *depth)
5265 {
5266   DM_Plex *mesh = (DM_Plex *)dm->data;
5267   DMLabel  label;
5268   PetscInt d = -1;
5269 
5270   PetscFunctionBegin;
5271   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5272   PetscAssertPointer(depth, 2);
5273   if (mesh->tr) {
5274     PetscCall(DMPlexTransformGetDepth(mesh->tr, depth));
5275   } else {
5276     PetscCall(DMPlexGetDepthLabel(dm, &label));
5277     // Allow missing depths
5278     if (label) PetscCall(DMLabelGetValueBounds(label, NULL, &d));
5279     *depth = d;
5280   }
5281   PetscFunctionReturn(PETSC_SUCCESS);
5282 }
5283 
5284 /*@
5285   DMPlexGetDepthStratum - Get the bounds [`start`, `end`) for all points at a certain depth.
5286 
5287   Not Collective
5288 
5289   Input Parameters:
5290 + dm    - The `DMPLEX` object
5291 - depth - The requested depth
5292 
5293   Output Parameters:
5294 + start - The first point at this `depth`
5295 - end   - One beyond the last point at this `depth`
5296 
5297   Level: developer
5298 
5299   Notes:
5300   Depth indexing is related to topological dimension.  Depth stratum 0 contains the lowest topological dimension points,
5301   often "vertices".  If the mesh is "interpolated" (see `DMPlexInterpolate()`), then depth stratum 1 contains the next
5302   higher dimension, e.g., "edges".
5303 
5304 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetHeightStratum()`, `DMPlexGetCellTypeStratum()`, `DMPlexGetDepth()`, `DMPlexGetDepthLabel()`, `DMPlexGetPointDepth()`, `DMPlexSymmetrize()`, `DMPlexInterpolate()`
5305 @*/
5306 PetscErrorCode DMPlexGetDepthStratum(DM dm, PetscInt depth, PetscInt *start, PetscInt *end)
5307 {
5308   DM_Plex *mesh = (DM_Plex *)dm->data;
5309   DMLabel  label;
5310   PetscInt pStart, pEnd;
5311 
5312   PetscFunctionBegin;
5313   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5314   if (start) {
5315     PetscAssertPointer(start, 3);
5316     *start = 0;
5317   }
5318   if (end) {
5319     PetscAssertPointer(end, 4);
5320     *end = 0;
5321   }
5322   PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
5323   if (pStart == pEnd) PetscFunctionReturn(PETSC_SUCCESS);
5324   if (depth < 0) {
5325     if (start) *start = pStart;
5326     if (end) *end = pEnd;
5327     PetscFunctionReturn(PETSC_SUCCESS);
5328   }
5329   if (mesh->tr) {
5330     PetscCall(DMPlexTransformGetDepthStratum(mesh->tr, depth, start, end));
5331   } else {
5332     PetscCall(DMPlexGetDepthLabel(dm, &label));
5333     PetscCheck(label, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "No label named depth was found");
5334     PetscCall(DMLabelGetStratumBounds(label, depth, start, end));
5335   }
5336   PetscFunctionReturn(PETSC_SUCCESS);
5337 }
5338 
5339 /*@
5340   DMPlexGetHeightStratum - Get the bounds [`start`, `end`) for all points at a certain height.
5341 
5342   Not Collective
5343 
5344   Input Parameters:
5345 + dm     - The `DMPLEX` object
5346 - height - The requested height
5347 
5348   Output Parameters:
5349 + start - The first point at this `height`
5350 - end   - One beyond the last point at this `height`
5351 
5352   Level: developer
5353 
5354   Notes:
5355   Height indexing is related to topological codimension.  Height stratum 0 contains the highest topological dimension
5356   points, often called "cells" or "elements".  If the mesh is "interpolated" (see `DMPlexInterpolate()`), then height
5357   stratum 1 contains the boundary of these "cells", often called "faces" or "facets".
5358 
5359 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetDepthStratum()`, `DMPlexGetCellTypeStratum()`, `DMPlexGetDepth()`, `DMPlexGetPointHeight()`
5360 @*/
5361 PetscErrorCode DMPlexGetHeightStratum(DM dm, PetscInt height, PetscInt *start, PetscInt *end)
5362 {
5363   DMLabel  label;
5364   PetscInt depth, pStart, pEnd;
5365 
5366   PetscFunctionBegin;
5367   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5368   if (start) {
5369     PetscAssertPointer(start, 3);
5370     *start = 0;
5371   }
5372   if (end) {
5373     PetscAssertPointer(end, 4);
5374     *end = 0;
5375   }
5376   PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
5377   if (pStart == pEnd) PetscFunctionReturn(PETSC_SUCCESS);
5378   if (height < 0) {
5379     if (start) *start = pStart;
5380     if (end) *end = pEnd;
5381     PetscFunctionReturn(PETSC_SUCCESS);
5382   }
5383   PetscCall(DMPlexGetDepthLabel(dm, &label));
5384   if (label) PetscCall(DMLabelGetNumValues(label, &depth));
5385   else PetscCall(DMGetDimension(dm, &depth));
5386   PetscCheck(depth >= 0, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Depth not yet computed");
5387   PetscCall(DMPlexGetDepthStratum(dm, depth - 1 - height, start, end));
5388   PetscFunctionReturn(PETSC_SUCCESS);
5389 }
5390 
5391 /*@
5392   DMPlexGetPointDepth - Get the `depth` of a given point
5393 
5394   Not Collective
5395 
5396   Input Parameters:
5397 + dm    - The `DMPLEX` object
5398 - point - The point
5399 
5400   Output Parameter:
5401 . depth - The depth of the `point`
5402 
5403   Level: intermediate
5404 
5405 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetCellType()`, `DMPlexGetDepthLabel()`, `DMPlexGetDepth()`, `DMPlexGetPointHeight()`
5406 @*/
5407 PetscErrorCode DMPlexGetPointDepth(DM dm, PetscInt point, PetscInt *depth)
5408 {
5409   PetscFunctionBegin;
5410   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5411   PetscAssertPointer(depth, 3);
5412   PetscCall(DMLabelGetValue(dm->depthLabel, point, depth));
5413   PetscFunctionReturn(PETSC_SUCCESS);
5414 }
5415 
5416 /*@
5417   DMPlexGetPointHeight - Get the `height` of a given point
5418 
5419   Not Collective
5420 
5421   Input Parameters:
5422 + dm    - The `DMPLEX` object
5423 - point - The point
5424 
5425   Output Parameter:
5426 . height - The height of the `point`
5427 
5428   Level: intermediate
5429 
5430 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetCellType()`, `DMPlexGetDepthLabel()`, `DMPlexGetDepth()`, `DMPlexGetPointDepth()`
5431 @*/
5432 PetscErrorCode DMPlexGetPointHeight(DM dm, PetscInt point, PetscInt *height)
5433 {
5434   PetscInt n, pDepth;
5435 
5436   PetscFunctionBegin;
5437   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5438   PetscAssertPointer(height, 3);
5439   PetscCall(DMLabelGetNumValues(dm->depthLabel, &n));
5440   PetscCall(DMLabelGetValue(dm->depthLabel, point, &pDepth));
5441   *height = n - 1 - pDepth; /* DAG depth is n-1 */
5442   PetscFunctionReturn(PETSC_SUCCESS);
5443 }
5444 
5445 /*@
5446   DMPlexGetCellTypeLabel - Get the `DMLabel` recording the polytope type of each cell
5447 
5448   Not Collective
5449 
5450   Input Parameter:
5451 . dm - The `DMPLEX` object
5452 
5453   Output Parameter:
5454 . celltypeLabel - The `DMLabel` recording cell polytope type
5455 
5456   Level: developer
5457 
5458   Note:
5459   This function will trigger automatica computation of cell types. This can be disabled by calling
5460   `DMCreateLabel`(dm, "celltype") beforehand.
5461 
5462 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetCellType()`, `DMPlexGetDepthLabel()`, `DMCreateLabel()`
5463 @*/
5464 PetscErrorCode DMPlexGetCellTypeLabel(DM dm, DMLabel *celltypeLabel)
5465 {
5466   PetscFunctionBegin;
5467   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5468   PetscAssertPointer(celltypeLabel, 2);
5469   if (!dm->celltypeLabel) PetscCall(DMPlexComputeCellTypes(dm));
5470   *celltypeLabel = dm->celltypeLabel;
5471   PetscFunctionReturn(PETSC_SUCCESS);
5472 }
5473 
5474 /*@
5475   DMPlexGetCellType - Get the polytope type of a given cell
5476 
5477   Not Collective
5478 
5479   Input Parameters:
5480 + dm   - The `DMPLEX` object
5481 - cell - The cell
5482 
5483   Output Parameter:
5484 . celltype - The polytope type of the cell
5485 
5486   Level: intermediate
5487 
5488 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPolytopeType`, `DMPlexGetCellTypeLabel()`, `DMPlexGetDepthLabel()`, `DMPlexGetDepth()`
5489 @*/
5490 PetscErrorCode DMPlexGetCellType(DM dm, PetscInt cell, DMPolytopeType *celltype)
5491 {
5492   DM_Plex *mesh = (DM_Plex *)dm->data;
5493   DMLabel  label;
5494   PetscInt ct;
5495 
5496   PetscFunctionBegin;
5497   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5498   PetscAssertPointer(celltype, 3);
5499   if (mesh->tr) {
5500     PetscCall(DMPlexTransformGetCellType(mesh->tr, cell, celltype));
5501   } else {
5502     PetscInt pStart, pEnd;
5503 
5504     PetscCall(PetscSectionGetChart(mesh->coneSection, &pStart, NULL));
5505     if (!mesh->cellTypes) { /* XXX remove? optimize? */
5506       PetscCall(PetscSectionGetChart(mesh->coneSection, NULL, &pEnd));
5507       PetscCall(PetscMalloc1(pEnd - pStart, &mesh->cellTypes));
5508       PetscCall(DMPlexGetCellTypeLabel(dm, &label));
5509       for (PetscInt p = pStart; p < pEnd; p++) {
5510         PetscCall(DMLabelGetValue(label, p, &ct));
5511         mesh->cellTypes[p - pStart].value_as_uint8 = (uint8_t)ct;
5512       }
5513     }
5514     *celltype = (DMPolytopeType)mesh->cellTypes[cell - pStart].value_as_uint8;
5515     if (PetscDefined(USE_DEBUG)) {
5516       PetscCall(DMPlexGetCellTypeLabel(dm, &label));
5517       PetscCall(DMLabelGetValue(label, cell, &ct));
5518       PetscCheck(ct >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Cell %" PetscInt_FMT " has not been assigned a cell type", cell);
5519       PetscCheck(ct == (PetscInt)*celltype, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid cellType for %" PetscInt_FMT ": %d != %" PetscInt_FMT, cell, (int)*celltype, ct);
5520     }
5521   }
5522   PetscFunctionReturn(PETSC_SUCCESS);
5523 }
5524 
5525 /*@
5526   DMPlexSetCellType - Set the polytope type of a given cell
5527 
5528   Not Collective
5529 
5530   Input Parameters:
5531 + dm       - The `DMPLEX` object
5532 . cell     - The cell
5533 - celltype - The polytope type of the cell
5534 
5535   Level: advanced
5536 
5537   Note:
5538   By default, cell types will be automatically computed using `DMPlexComputeCellTypes()` before this function
5539   is executed. This function will override the computed type. However, if automatic classification will not succeed
5540   and a user wants to manually specify all types, the classification must be disabled by calling
5541   DMCreateLabel(dm, "celltype") before getting or setting any cell types.
5542 
5543 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetCellTypeLabel()`, `DMPlexGetDepthLabel()`, `DMPlexGetDepth()`, `DMPlexComputeCellTypes()`, `DMCreateLabel()`
5544 @*/
5545 PetscErrorCode DMPlexSetCellType(DM dm, PetscInt cell, DMPolytopeType celltype)
5546 {
5547   DM_Plex *mesh = (DM_Plex *)dm->data;
5548   DMLabel  label;
5549   PetscInt pStart, pEnd;
5550 
5551   PetscFunctionBegin;
5552   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5553   PetscCall(PetscSectionGetChart(mesh->coneSection, &pStart, &pEnd));
5554   PetscCall(DMPlexGetCellTypeLabel(dm, &label));
5555   PetscCall(DMLabelSetValue(label, cell, celltype));
5556   if (!mesh->cellTypes) PetscCall(PetscMalloc1(pEnd - pStart, &mesh->cellTypes));
5557   mesh->cellTypes[cell - pStart].value_as_uint8 = (uint8_t)celltype;
5558   PetscFunctionReturn(PETSC_SUCCESS);
5559 }
5560 
5561 PetscErrorCode DMCreateCoordinateDM_Plex(DM dm, DM *cdm)
5562 {
5563   PetscSection section;
5564   PetscInt     maxHeight;
5565   const char  *prefix;
5566 
5567   PetscFunctionBegin;
5568   PetscCall(DMClone(dm, cdm));
5569   PetscCall(PetscObjectGetOptionsPrefix((PetscObject)dm, &prefix));
5570   PetscCall(PetscObjectSetOptionsPrefix((PetscObject)*cdm, prefix));
5571   PetscCall(PetscObjectAppendOptionsPrefix((PetscObject)*cdm, "cdm_"));
5572   PetscCall(DMPlexGetMaxProjectionHeight(dm, &maxHeight));
5573   PetscCall(DMPlexSetMaxProjectionHeight(*cdm, maxHeight));
5574   PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dm), &section));
5575   PetscCall(DMSetLocalSection(*cdm, section));
5576   PetscCall(PetscSectionDestroy(&section));
5577 
5578   PetscCall(DMSetNumFields(*cdm, 1));
5579   PetscCall(DMCreateDS(*cdm));
5580   (*cdm)->cloneOpts = PETSC_TRUE;
5581   if (dm->setfromoptionscalled) PetscCall(DMSetFromOptions(*cdm));
5582   PetscFunctionReturn(PETSC_SUCCESS);
5583 }
5584 
5585 PetscErrorCode DMCreateCoordinateField_Plex(DM dm, DMField *field)
5586 {
5587   Vec coordsLocal, cellCoordsLocal;
5588   DM  coordsDM, cellCoordsDM;
5589 
5590   PetscFunctionBegin;
5591   *field = NULL;
5592   PetscCall(DMGetCoordinatesLocal(dm, &coordsLocal));
5593   PetscCall(DMGetCoordinateDM(dm, &coordsDM));
5594   PetscCall(DMGetCellCoordinatesLocal(dm, &cellCoordsLocal));
5595   PetscCall(DMGetCellCoordinateDM(dm, &cellCoordsDM));
5596   if (coordsLocal && coordsDM) {
5597     if (cellCoordsLocal && cellCoordsDM) PetscCall(DMFieldCreateDSWithDG(coordsDM, cellCoordsDM, 0, coordsLocal, cellCoordsLocal, field));
5598     else PetscCall(DMFieldCreateDS(coordsDM, 0, coordsLocal, field));
5599   }
5600   PetscFunctionReturn(PETSC_SUCCESS);
5601 }
5602 
5603 /*@
5604   DMPlexGetConeSection - Return a section which describes the layout of cone data
5605 
5606   Not Collective
5607 
5608   Input Parameter:
5609 . dm - The `DMPLEX` object
5610 
5611   Output Parameter:
5612 . section - The `PetscSection` object
5613 
5614   Level: developer
5615 
5616 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetSupportSection()`, `DMPlexGetCones()`, `DMPlexGetConeOrientations()`, `PetscSection`
5617 @*/
5618 PetscErrorCode DMPlexGetConeSection(DM dm, PetscSection *section)
5619 {
5620   DM_Plex *mesh = (DM_Plex *)dm->data;
5621 
5622   PetscFunctionBegin;
5623   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5624   if (section) *section = mesh->coneSection;
5625   PetscFunctionReturn(PETSC_SUCCESS);
5626 }
5627 
5628 /*@
5629   DMPlexGetSupportSection - Return a section which describes the layout of support data
5630 
5631   Not Collective
5632 
5633   Input Parameter:
5634 . dm - The `DMPLEX` object
5635 
5636   Output Parameter:
5637 . section - The `PetscSection` object
5638 
5639   Level: developer
5640 
5641 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetConeSection()`, `PetscSection`
5642 @*/
5643 PetscErrorCode DMPlexGetSupportSection(DM dm, PetscSection *section)
5644 {
5645   DM_Plex *mesh = (DM_Plex *)dm->data;
5646 
5647   PetscFunctionBegin;
5648   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5649   if (section) *section = mesh->supportSection;
5650   PetscFunctionReturn(PETSC_SUCCESS);
5651 }
5652 
5653 /*@C
5654   DMPlexGetCones - Return cone data
5655 
5656   Not Collective
5657 
5658   Input Parameter:
5659 . dm - The `DMPLEX` object
5660 
5661   Output Parameter:
5662 . cones - The cone for each point
5663 
5664   Level: developer
5665 
5666 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetConeSection()`
5667 @*/
5668 PetscErrorCode DMPlexGetCones(DM dm, PetscInt *cones[])
5669 {
5670   DM_Plex *mesh = (DM_Plex *)dm->data;
5671 
5672   PetscFunctionBegin;
5673   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5674   if (cones) *cones = mesh->cones;
5675   PetscFunctionReturn(PETSC_SUCCESS);
5676 }
5677 
5678 /*@C
5679   DMPlexGetConeOrientations - Return cone orientation data
5680 
5681   Not Collective
5682 
5683   Input Parameter:
5684 . dm - The `DMPLEX` object
5685 
5686   Output Parameter:
5687 . coneOrientations - The array of cone orientations for all points
5688 
5689   Level: developer
5690 
5691   Notes:
5692   The `PetscSection` returned by `DMPlexGetConeSection()` partitions coneOrientations into cone orientations of particular points
5693   as returned by `DMPlexGetConeOrientation()`.
5694 
5695   The meaning of coneOrientations values is detailed in `DMPlexGetConeOrientation()`.
5696 
5697 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetConeSection()`, `DMPlexGetConeOrientation()`, `PetscSection`
5698 @*/
5699 PetscErrorCode DMPlexGetConeOrientations(DM dm, PetscInt *coneOrientations[])
5700 {
5701   DM_Plex *mesh = (DM_Plex *)dm->data;
5702 
5703   PetscFunctionBegin;
5704   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
5705   if (coneOrientations) *coneOrientations = mesh->coneOrientations;
5706   PetscFunctionReturn(PETSC_SUCCESS);
5707 }
5708 
5709 /* FEM Support */
5710 
5711 PetscErrorCode DMPlexGetAllCells_Internal(DM plex, IS *cellIS)
5712 {
5713   PetscInt depth;
5714 
5715   PetscFunctionBegin;
5716   PetscCall(DMPlexGetDepth(plex, &depth));
5717   PetscCall(DMGetStratumIS(plex, "dim", depth, cellIS));
5718   if (!*cellIS) PetscCall(DMGetStratumIS(plex, "depth", depth, cellIS));
5719   PetscFunctionReturn(PETSC_SUCCESS);
5720 }
5721 
5722 PetscErrorCode DMPlexGetAllFaces_Internal(DM plex, IS *faceIS)
5723 {
5724   PetscInt depth;
5725 
5726   PetscFunctionBegin;
5727   PetscCall(DMPlexGetDepth(plex, &depth));
5728   PetscCall(DMGetStratumIS(plex, "dim", depth - 1, faceIS));
5729   if (!*faceIS) PetscCall(DMGetStratumIS(plex, "depth", depth - 1, faceIS));
5730   PetscFunctionReturn(PETSC_SUCCESS);
5731 }
5732 
5733 /*
5734  Returns number of components and tensor degree for the field.  For interpolated meshes, line should be a point
5735  representing a line in the section.
5736 */
5737 static PetscErrorCode PetscSectionFieldGetTensorDegree_Private(DM dm, PetscSection section, PetscInt field, PetscInt line, PetscInt *Nc, PetscInt *k, PetscBool *continuous, PetscBool *tensor)
5738 {
5739   PetscObject  obj;
5740   PetscClassId id;
5741   PetscFE      fe = NULL;
5742 
5743   PetscFunctionBeginHot;
5744   PetscCall(PetscSectionGetFieldComponents(section, field, Nc));
5745   PetscCall(DMGetField(dm, field, NULL, &obj));
5746   PetscCall(PetscObjectGetClassId(obj, &id));
5747   if (id == PETSCFE_CLASSID) fe = (PetscFE)obj;
5748 
5749   if (!fe) {
5750     /* Assume the full interpolated mesh is in the chart; lines in particular */
5751     /* An order k SEM disc has k-1 dofs on an edge */
5752     PetscCall(PetscSectionGetFieldDof(section, line, field, k));
5753     *k = *k / *Nc + 1;
5754   } else {
5755     PetscInt       dual_space_size, dim;
5756     PetscDualSpace dsp;
5757 
5758     PetscCall(DMGetDimension(dm, &dim));
5759     PetscCall(PetscFEGetDualSpace(fe, &dsp));
5760     PetscCall(PetscDualSpaceGetDimension(dsp, &dual_space_size));
5761     *k = (PetscInt)PetscCeilReal(PetscPowReal(dual_space_size / *Nc, 1.0 / dim)) - 1;
5762     PetscCall(PetscDualSpaceLagrangeGetContinuity(dsp, continuous));
5763     PetscCall(PetscDualSpaceLagrangeGetTensor(dsp, tensor));
5764   }
5765   PetscFunctionReturn(PETSC_SUCCESS);
5766 }
5767 
5768 static PetscErrorCode GetFieldSize_Private(PetscInt dim, PetscInt k, PetscBool tensor, PetscInt *dof)
5769 {
5770   PetscFunctionBeginHot;
5771   if (tensor) {
5772     *dof = PetscPowInt(k + 1, dim);
5773   } else {
5774     switch (dim) {
5775     case 1:
5776       *dof = k + 1;
5777       break;
5778     case 2:
5779       *dof = ((k + 1) * (k + 2)) / 2;
5780       break;
5781     case 3:
5782       *dof = ((k + 1) * (k + 2) * (k + 3)) / 6;
5783       break;
5784     default:
5785       *dof = 0;
5786     }
5787   }
5788   PetscFunctionReturn(PETSC_SUCCESS);
5789 }
5790 
5791 /*@
5792   DMPlexSetClosurePermutationTensor - Create a permutation from the default (BFS) point ordering in the closure, to a
5793   lexicographic ordering over the tensor product cell (i.e., line, quad, hex, etc.), and set this permutation in the
5794   section provided (or the section of the `DM`).
5795 
5796   Input Parameters:
5797 + dm      - The `DM`
5798 . point   - Either a cell (highest dim point) or an edge (dim 1 point), or `PETSC_DETERMINE`
5799 - section - The `PetscSection` to reorder, or `NULL` for the default section
5800 
5801   Example:
5802   A typical interpolated single-quad mesh might order points as
5803 .vb
5804   [c0, v1, v2, v3, v4, e5, e6, e7, e8]
5805 
5806   v4 -- e6 -- v3
5807   |           |
5808   e7    c0    e8
5809   |           |
5810   v1 -- e5 -- v2
5811 .ve
5812 
5813   (There is no significance to the ordering described here.)  The default section for a Q3 quad might typically assign
5814   dofs in the order of points, e.g.,
5815 .vb
5816     c0 -> [0,1,2,3]
5817     v1 -> [4]
5818     ...
5819     e5 -> [8, 9]
5820 .ve
5821 
5822   which corresponds to the dofs
5823 .vb
5824     6   10  11  7
5825     13  2   3   15
5826     12  0   1   14
5827     4   8   9   5
5828 .ve
5829 
5830   The closure in BFS ordering works through height strata (cells, edges, vertices) to produce the ordering
5831 .vb
5832   0 1 2 3 8 9 14 15 11 10 13 12 4 5 7 6
5833 .ve
5834 
5835   After calling DMPlexSetClosurePermutationTensor(), the closure will be ordered lexicographically,
5836 .vb
5837    4 8 9 5 12 0 1 14 13 2 3 15 6 10 11 7
5838 .ve
5839 
5840   Level: developer
5841 
5842   Notes:
5843   The point is used to determine the number of dofs/field on an edge. For SEM, this is related to the polynomial
5844   degree of the basis.
5845 
5846   This is required to run with libCEED.
5847 
5848 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMGetLocalSection()`, `PetscSectionSetClosurePermutation()`, `DMSetGlobalSection()`
5849 @*/
5850 PetscErrorCode DMPlexSetClosurePermutationTensor(DM dm, PetscInt point, PetscSection section)
5851 {
5852   DMLabel   label;
5853   PetscInt  dim, depth = -1, eStart = -1, Nf;
5854   PetscBool continuous = PETSC_TRUE, tensor = PETSC_TRUE;
5855 
5856   PetscFunctionBegin;
5857   PetscCall(DMGetDimension(dm, &dim));
5858   if (dim < 1) PetscFunctionReturn(PETSC_SUCCESS);
5859   if (point < 0) {
5860     PetscInt sStart, sEnd;
5861 
5862     PetscCall(DMPlexGetDepthStratum(dm, 1, &sStart, &sEnd));
5863     point = sEnd - sStart ? sStart : point;
5864   }
5865   PetscCall(DMPlexGetDepthLabel(dm, &label));
5866   if (point >= 0) PetscCall(DMLabelGetValue(label, point, &depth));
5867   if (!section) PetscCall(DMGetLocalSection(dm, &section));
5868   if (depth == 1) {
5869     eStart = point;
5870   } else if (depth == dim) {
5871     const PetscInt *cone;
5872 
5873     PetscCall(DMPlexGetCone(dm, point, &cone));
5874     if (dim == 2) eStart = cone[0];
5875     else if (dim == 3) {
5876       const PetscInt *cone2;
5877       PetscCall(DMPlexGetCone(dm, cone[0], &cone2));
5878       eStart = cone2[0];
5879     } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Point %" PetscInt_FMT " of depth %" PetscInt_FMT " cannot be used to bootstrap spectral ordering for dim %" PetscInt_FMT, point, depth, dim);
5880   } else PetscCheck(depth < 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Point %" PetscInt_FMT " of depth %" PetscInt_FMT " cannot be used to bootstrap spectral ordering for dim %" PetscInt_FMT, point, depth, dim);
5881 
5882   PetscCall(PetscSectionGetNumFields(section, &Nf));
5883   for (PetscInt d = 1; d <= dim; d++) {
5884     PetscInt  k, f, Nc, c, i, j, size = 0, offset = 0, foffset = 0;
5885     PetscInt *perm;
5886 
5887     for (f = 0; f < Nf; ++f) {
5888       PetscInt dof;
5889 
5890       PetscCall(PetscSectionFieldGetTensorDegree_Private(dm, section, f, eStart, &Nc, &k, &continuous, &tensor));
5891       PetscCheck(dim == 1 || tensor || !continuous, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Continuous field %" PetscInt_FMT " must have a tensor product discretization", f);
5892       if (!continuous && d < dim) continue;
5893       PetscCall(GetFieldSize_Private(d, k, tensor, &dof));
5894       size += dof * Nc;
5895     }
5896     PetscCall(PetscMalloc1(size, &perm));
5897     for (f = 0; f < Nf; ++f) {
5898       switch (d) {
5899       case 1:
5900         PetscCall(PetscSectionFieldGetTensorDegree_Private(dm, section, f, eStart, &Nc, &k, &continuous, &tensor));
5901         if (!continuous && d < dim) continue;
5902         /*
5903          Original ordering is [ edge of length k-1; vtx0; vtx1 ]
5904          We want              [ vtx0; edge of length k-1; vtx1 ]
5905          */
5906         if (continuous) {
5907           for (c = 0; c < Nc; c++, offset++) perm[offset] = (k - 1) * Nc + c + foffset;
5908           for (i = 0; i < k - 1; i++)
5909             for (c = 0; c < Nc; c++, offset++) perm[offset] = i * Nc + c + foffset;
5910           for (c = 0; c < Nc; c++, offset++) perm[offset] = k * Nc + c + foffset;
5911           foffset = offset;
5912         } else {
5913           PetscInt dof;
5914 
5915           PetscCall(GetFieldSize_Private(d, k, tensor, &dof));
5916           for (i = 0; i < dof * Nc; ++i, ++offset) perm[offset] = i + foffset;
5917           foffset = offset;
5918         }
5919         break;
5920       case 2:
5921         /* The original quad closure is oriented clockwise, {f, e_b, e_r, e_t, e_l, v_lb, v_rb, v_tr, v_tl} */
5922         PetscCall(PetscSectionFieldGetTensorDegree_Private(dm, section, f, eStart, &Nc, &k, &continuous, &tensor));
5923         if (!continuous && d < dim) continue;
5924         /* The SEM order is
5925 
5926          v_lb, {e_b}, v_rb,
5927          e^{(k-1)-i}_l, {f^{i*(k-1)}}, e^i_r,
5928          v_lt, reverse {e_t}, v_rt
5929          */
5930         if (continuous) {
5931           const PetscInt of   = 0;
5932           const PetscInt oeb  = of + PetscSqr(k - 1);
5933           const PetscInt oer  = oeb + (k - 1);
5934           const PetscInt oet  = oer + (k - 1);
5935           const PetscInt oel  = oet + (k - 1);
5936           const PetscInt ovlb = oel + (k - 1);
5937           const PetscInt ovrb = ovlb + 1;
5938           const PetscInt ovrt = ovrb + 1;
5939           const PetscInt ovlt = ovrt + 1;
5940           PetscInt       o;
5941 
5942           /* bottom */
5943           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovlb * Nc + c + foffset;
5944           for (o = oeb; o < oer; ++o)
5945             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o * Nc + c + foffset;
5946           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovrb * Nc + c + foffset;
5947           /* middle */
5948           for (i = 0; i < k - 1; ++i) {
5949             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oel + (k - 2) - i) * Nc + c + foffset;
5950             for (o = of + (k - 1) * i; o < of + (k - 1) * (i + 1); ++o)
5951               for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o * Nc + c + foffset;
5952             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oer + i) * Nc + c + foffset;
5953           }
5954           /* top */
5955           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovlt * Nc + c + foffset;
5956           for (o = oel - 1; o >= oet; --o)
5957             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o * Nc + c + foffset;
5958           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovrt * Nc + c + foffset;
5959           foffset = offset;
5960         } else {
5961           PetscInt dof;
5962 
5963           PetscCall(GetFieldSize_Private(d, k, tensor, &dof));
5964           for (i = 0; i < dof * Nc; ++i, ++offset) perm[offset] = i + foffset;
5965           foffset = offset;
5966         }
5967         break;
5968       case 3:
5969         /* The original hex closure is
5970 
5971          {c,
5972          f_b, f_t, f_f, f_b, f_r, f_l,
5973          e_bl, e_bb, e_br, e_bf,  e_tf, e_tr, e_tb, e_tl,  e_rf, e_lf, e_lb, e_rb,
5974          v_blf, v_blb, v_brb, v_brf, v_tlf, v_trf, v_trb, v_tlb}
5975          */
5976         PetscCall(PetscSectionFieldGetTensorDegree_Private(dm, section, f, eStart, &Nc, &k, &continuous, &tensor));
5977         if (!continuous && d < dim) continue;
5978         /* The SEM order is
5979          Bottom Slice
5980          v_blf, {e^{(k-1)-n}_bf}, v_brf,
5981          e^{i}_bl, f^{n*(k-1)+(k-1)-i}_b, e^{(k-1)-i}_br,
5982          v_blb, {e_bb}, v_brb,
5983 
5984          Middle Slice (j)
5985          {e^{(k-1)-j}_lf}, {f^{j*(k-1)+n}_f}, e^j_rf,
5986          f^{i*(k-1)+j}_l, {c^{(j*(k-1) + i)*(k-1)+n}_t}, f^{j*(k-1)+i}_r,
5987          e^j_lb, {f^{j*(k-1)+(k-1)-n}_b}, e^{(k-1)-j}_rb,
5988 
5989          Top Slice
5990          v_tlf, {e_tf}, v_trf,
5991          e^{(k-1)-i}_tl, {f^{i*(k-1)}_t}, e^{i}_tr,
5992          v_tlb, {e^{(k-1)-n}_tb}, v_trb,
5993          */
5994         if (continuous) {
5995           const PetscInt oc    = 0;
5996           const PetscInt ofb   = oc + PetscSqr(k - 1) * (k - 1);
5997           const PetscInt oft   = ofb + PetscSqr(k - 1);
5998           const PetscInt off   = oft + PetscSqr(k - 1);
5999           const PetscInt ofk   = off + PetscSqr(k - 1);
6000           const PetscInt ofr   = ofk + PetscSqr(k - 1);
6001           const PetscInt ofl   = ofr + PetscSqr(k - 1);
6002           const PetscInt oebl  = ofl + PetscSqr(k - 1);
6003           const PetscInt oebb  = oebl + (k - 1);
6004           const PetscInt oebr  = oebb + (k - 1);
6005           const PetscInt oebf  = oebr + (k - 1);
6006           const PetscInt oetf  = oebf + (k - 1);
6007           const PetscInt oetr  = oetf + (k - 1);
6008           const PetscInt oetb  = oetr + (k - 1);
6009           const PetscInt oetl  = oetb + (k - 1);
6010           const PetscInt oerf  = oetl + (k - 1);
6011           const PetscInt oelf  = oerf + (k - 1);
6012           const PetscInt oelb  = oelf + (k - 1);
6013           const PetscInt oerb  = oelb + (k - 1);
6014           const PetscInt ovblf = oerb + (k - 1);
6015           const PetscInt ovblb = ovblf + 1;
6016           const PetscInt ovbrb = ovblb + 1;
6017           const PetscInt ovbrf = ovbrb + 1;
6018           const PetscInt ovtlf = ovbrf + 1;
6019           const PetscInt ovtrf = ovtlf + 1;
6020           const PetscInt ovtrb = ovtrf + 1;
6021           const PetscInt ovtlb = ovtrb + 1;
6022           PetscInt       o, n;
6023 
6024           /* Bottom Slice */
6025           /*   bottom */
6026           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovblf * Nc + c + foffset;
6027           for (o = oetf - 1; o >= oebf; --o)
6028             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o * Nc + c + foffset;
6029           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovbrf * Nc + c + foffset;
6030           /*   middle */
6031           for (i = 0; i < k - 1; ++i) {
6032             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oebl + i) * Nc + c + foffset;
6033             for (n = 0; n < k - 1; ++n) {
6034               o = ofb + n * (k - 1) + i;
6035               for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o * Nc + c + foffset;
6036             }
6037             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oebr + (k - 2) - i) * Nc + c + foffset;
6038           }
6039           /*   top */
6040           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovblb * Nc + c + foffset;
6041           for (o = oebb; o < oebr; ++o)
6042             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o * Nc + c + foffset;
6043           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovbrb * Nc + c + foffset;
6044 
6045           /* Middle Slice */
6046           for (j = 0; j < k - 1; ++j) {
6047             /*   bottom */
6048             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oelf + (k - 2) - j) * Nc + c + foffset;
6049             for (o = off + j * (k - 1); o < off + (j + 1) * (k - 1); ++o)
6050               for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o * Nc + c + foffset;
6051             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oerf + j) * Nc + c + foffset;
6052             /*   middle */
6053             for (i = 0; i < k - 1; ++i) {
6054               for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (ofl + i * (k - 1) + j) * Nc + c + foffset;
6055               for (n = 0; n < k - 1; ++n)
6056                 for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oc + (j * (k - 1) + i) * (k - 1) + n) * Nc + c + foffset;
6057               for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (ofr + j * (k - 1) + i) * Nc + c + foffset;
6058             }
6059             /*   top */
6060             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oelb + j) * Nc + c + foffset;
6061             for (o = ofk + j * (k - 1) + (k - 2); o >= ofk + j * (k - 1); --o)
6062               for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o * Nc + c + foffset;
6063             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oerb + (k - 2) - j) * Nc + c + foffset;
6064           }
6065 
6066           /* Top Slice */
6067           /*   bottom */
6068           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtlf * Nc + c + foffset;
6069           for (o = oetf; o < oetr; ++o)
6070             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o * Nc + c + foffset;
6071           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtrf * Nc + c + foffset;
6072           /*   middle */
6073           for (i = 0; i < k - 1; ++i) {
6074             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oetl + (k - 2) - i) * Nc + c + foffset;
6075             for (n = 0; n < k - 1; ++n)
6076               for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oft + i * (k - 1) + n) * Nc + c + foffset;
6077             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = (oetr + i) * Nc + c + foffset;
6078           }
6079           /*   top */
6080           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtlb * Nc + c + foffset;
6081           for (o = oetl - 1; o >= oetb; --o)
6082             for (c = 0; c < Nc; ++c, ++offset) perm[offset] = o * Nc + c + foffset;
6083           for (c = 0; c < Nc; ++c, ++offset) perm[offset] = ovtrb * Nc + c + foffset;
6084 
6085           foffset = offset;
6086         } else {
6087           PetscInt dof;
6088 
6089           PetscCall(GetFieldSize_Private(d, k, tensor, &dof));
6090           for (i = 0; i < dof * Nc; ++i, ++offset) perm[offset] = i + foffset;
6091           foffset = offset;
6092         }
6093         break;
6094       default:
6095         SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "No spectral ordering for dimension %" PetscInt_FMT, d);
6096       }
6097     }
6098     PetscCheck(offset == size, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Number of permutation entries %" PetscInt_FMT " != %" PetscInt_FMT, offset, size);
6099     /* Check permutation */
6100     {
6101       PetscInt *check;
6102 
6103       PetscCall(PetscMalloc1(size, &check));
6104       for (i = 0; i < size; ++i) {
6105         check[i] = -1;
6106         PetscCheck(perm[i] >= 0 && perm[i] < size, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid permutation index p[%" PetscInt_FMT "] = %" PetscInt_FMT, i, perm[i]);
6107       }
6108       for (i = 0; i < size; ++i) check[perm[i]] = i;
6109       for (i = 0; i < size; ++i) PetscCheck(check[i] >= 0, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Missing permutation index %" PetscInt_FMT, i);
6110       PetscCall(PetscFree(check));
6111     }
6112     PetscCall(PetscSectionSetClosurePermutation_Internal(section, (PetscObject)dm, d, size, PETSC_OWN_POINTER, perm));
6113     if (d == dim) { // Add permutation for localized (in case this is a coordinate DM)
6114       PetscInt *loc_perm;
6115       PetscCall(PetscMalloc1(size * 2, &loc_perm));
6116       for (PetscInt i = 0; i < size; i++) {
6117         loc_perm[i]        = perm[i];
6118         loc_perm[size + i] = size + perm[i];
6119       }
6120       PetscCall(PetscSectionSetClosurePermutation_Internal(section, (PetscObject)dm, d, size * 2, PETSC_OWN_POINTER, loc_perm));
6121     }
6122   }
6123   PetscFunctionReturn(PETSC_SUCCESS);
6124 }
6125 
6126 PetscErrorCode DMPlexGetPointDualSpaceFEM(DM dm, PetscInt point, PetscInt field, PetscDualSpace *dspace)
6127 {
6128   PetscDS  prob;
6129   PetscInt depth, Nf, h;
6130   DMLabel  label;
6131 
6132   PetscFunctionBeginHot;
6133   PetscCall(DMGetDS(dm, &prob));
6134   Nf      = prob->Nf;
6135   label   = dm->depthLabel;
6136   *dspace = NULL;
6137   if (field < Nf) {
6138     PetscObject disc = prob->disc[field];
6139 
6140     if (disc->classid == PETSCFE_CLASSID) {
6141       PetscDualSpace dsp;
6142 
6143       PetscCall(PetscFEGetDualSpace((PetscFE)disc, &dsp));
6144       PetscCall(DMLabelGetNumValues(label, &depth));
6145       PetscCall(DMLabelGetValue(label, point, &h));
6146       h = depth - 1 - h;
6147       if (h) {
6148         PetscCall(PetscDualSpaceGetHeightSubspace(dsp, h, dspace));
6149       } else {
6150         *dspace = dsp;
6151       }
6152     }
6153   }
6154   PetscFunctionReturn(PETSC_SUCCESS);
6155 }
6156 
6157 static inline PetscErrorCode DMPlexVecGetClosure_Depth1_Static(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
6158 {
6159   PetscScalar       *array;
6160   const PetscScalar *vArray;
6161   const PetscInt    *cone, *coneO;
6162   PetscInt           pStart, pEnd, p, numPoints, size = 0, offset = 0;
6163 
6164   PetscFunctionBeginHot;
6165   PetscCall(PetscSectionGetChart(section, &pStart, &pEnd));
6166   PetscCall(DMPlexGetConeSize(dm, point, &numPoints));
6167   PetscCall(DMPlexGetCone(dm, point, &cone));
6168   PetscCall(DMPlexGetConeOrientation(dm, point, &coneO));
6169   if (!values || !*values) {
6170     if ((point >= pStart) && (point < pEnd)) {
6171       PetscInt dof;
6172 
6173       PetscCall(PetscSectionGetDof(section, point, &dof));
6174       size += dof;
6175     }
6176     for (p = 0; p < numPoints; ++p) {
6177       const PetscInt cp = cone[p];
6178       PetscInt       dof;
6179 
6180       if ((cp < pStart) || (cp >= pEnd)) continue;
6181       PetscCall(PetscSectionGetDof(section, cp, &dof));
6182       size += dof;
6183     }
6184     if (!values) {
6185       if (csize) *csize = size;
6186       PetscFunctionReturn(PETSC_SUCCESS);
6187     }
6188     PetscCall(DMGetWorkArray(dm, size, MPIU_SCALAR, &array));
6189   } else {
6190     array = *values;
6191   }
6192   size = 0;
6193   PetscCall(VecGetArrayRead(v, &vArray));
6194   if ((point >= pStart) && (point < pEnd)) {
6195     PetscInt           dof, off, d;
6196     const PetscScalar *varr;
6197 
6198     PetscCall(PetscSectionGetDof(section, point, &dof));
6199     PetscCall(PetscSectionGetOffset(section, point, &off));
6200     varr = PetscSafePointerPlusOffset(vArray, off);
6201     for (d = 0; d < dof; ++d, ++offset) array[offset] = varr[d];
6202     size += dof;
6203   }
6204   for (p = 0; p < numPoints; ++p) {
6205     const PetscInt     cp = cone[p];
6206     PetscInt           o  = coneO[p];
6207     PetscInt           dof, off, d;
6208     const PetscScalar *varr;
6209 
6210     if ((cp < pStart) || (cp >= pEnd)) continue;
6211     PetscCall(PetscSectionGetDof(section, cp, &dof));
6212     PetscCall(PetscSectionGetOffset(section, cp, &off));
6213     varr = PetscSafePointerPlusOffset(vArray, off);
6214     if (o >= 0) {
6215       for (d = 0; d < dof; ++d, ++offset) array[offset] = varr[d];
6216     } else {
6217       for (d = dof - 1; d >= 0; --d, ++offset) array[offset] = varr[d];
6218     }
6219     size += dof;
6220   }
6221   PetscCall(VecRestoreArrayRead(v, &vArray));
6222   if (!*values) {
6223     if (csize) *csize = size;
6224     *values = array;
6225   } else {
6226     PetscCheck(size <= *csize, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Size of input array %" PetscInt_FMT " < actual size %" PetscInt_FMT, *csize, size);
6227     *csize = size;
6228   }
6229   PetscFunctionReturn(PETSC_SUCCESS);
6230 }
6231 
6232 /* Compress out points not in the section */
6233 static inline PetscErrorCode CompressPoints_Private(PetscSection section, PetscInt *numPoints, PetscInt points[])
6234 {
6235   const PetscInt np = *numPoints;
6236   PetscInt       pStart, pEnd, p, q;
6237 
6238   PetscCall(PetscSectionGetChart(section, &pStart, &pEnd));
6239   for (p = 0, q = 0; p < np; ++p) {
6240     const PetscInt r = points[p * 2];
6241     if ((r >= pStart) && (r < pEnd)) {
6242       points[q * 2]     = r;
6243       points[q * 2 + 1] = points[p * 2 + 1];
6244       ++q;
6245     }
6246   }
6247   *numPoints = q;
6248   return PETSC_SUCCESS;
6249 }
6250 
6251 /* Compressed closure does not apply closure permutation */
6252 PetscErrorCode DMPlexGetCompressedClosure(DM dm, PetscSection section, PetscInt point, PetscInt ornt, PetscInt *numPoints, PetscInt **points, PetscSection *clSec, IS *clPoints, const PetscInt **clp)
6253 {
6254   const PetscInt *cla = NULL;
6255   PetscInt        np, *pts = NULL;
6256 
6257   PetscFunctionBeginHot;
6258   PetscCall(PetscSectionGetClosureIndex(section, (PetscObject)dm, clSec, clPoints));
6259   if (!ornt && *clPoints) {
6260     PetscInt dof, off;
6261 
6262     PetscCall(PetscSectionGetDof(*clSec, point, &dof));
6263     PetscCall(PetscSectionGetOffset(*clSec, point, &off));
6264     PetscCall(ISGetIndices(*clPoints, &cla));
6265     np  = dof / 2;
6266     pts = PetscSafePointerPlusOffset((PetscInt *)cla, off);
6267   } else {
6268     PetscCall(DMPlexGetTransitiveClosure_Internal(dm, point, ornt, PETSC_TRUE, &np, &pts));
6269     PetscCall(CompressPoints_Private(section, &np, pts));
6270   }
6271   *numPoints = np;
6272   *points    = pts;
6273   *clp       = cla;
6274   PetscFunctionReturn(PETSC_SUCCESS);
6275 }
6276 
6277 PetscErrorCode DMPlexRestoreCompressedClosure(DM dm, PetscSection section, PetscInt point, PetscInt *numPoints, PetscInt **points, PetscSection *clSec, IS *clPoints, const PetscInt **clp)
6278 {
6279   PetscFunctionBeginHot;
6280   if (!*clPoints) {
6281     PetscCall(DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, numPoints, points));
6282   } else {
6283     PetscCall(ISRestoreIndices(*clPoints, clp));
6284   }
6285   *numPoints = 0;
6286   *points    = NULL;
6287   *clSec     = NULL;
6288   *clPoints  = NULL;
6289   *clp       = NULL;
6290   PetscFunctionReturn(PETSC_SUCCESS);
6291 }
6292 
6293 static inline PetscErrorCode DMPlexVecGetClosure_Static(DM dm, PetscSection section, PetscInt numPoints, const PetscInt points[], const PetscInt clperm[], const PetscScalar vArray[], PetscInt *size, PetscScalar array[])
6294 {
6295   PetscInt            offset = 0, p;
6296   const PetscInt    **perms  = NULL;
6297   const PetscScalar **flips  = NULL;
6298 
6299   PetscFunctionBeginHot;
6300   *size = 0;
6301   PetscCall(PetscSectionGetPointSyms(section, numPoints, points, &perms, &flips));
6302   for (p = 0; p < numPoints; p++) {
6303     const PetscInt     point = points[2 * p];
6304     const PetscInt    *perm  = perms ? perms[p] : NULL;
6305     const PetscScalar *flip  = flips ? flips[p] : NULL;
6306     PetscInt           dof, off, d;
6307     const PetscScalar *varr;
6308 
6309     PetscCall(PetscSectionGetDof(section, point, &dof));
6310     PetscCall(PetscSectionGetOffset(section, point, &off));
6311     varr = PetscSafePointerPlusOffset(vArray, off);
6312     if (clperm) {
6313       if (perm) {
6314         for (d = 0; d < dof; d++) array[clperm[offset + perm[d]]] = varr[d];
6315       } else {
6316         for (d = 0; d < dof; d++) array[clperm[offset + d]] = varr[d];
6317       }
6318       if (flip) {
6319         for (d = 0; d < dof; d++) array[clperm[offset + d]] *= flip[d];
6320       }
6321     } else {
6322       if (perm) {
6323         for (d = 0; d < dof; d++) array[offset + perm[d]] = varr[d];
6324       } else {
6325         for (d = 0; d < dof; d++) array[offset + d] = varr[d];
6326       }
6327       if (flip) {
6328         for (d = 0; d < dof; d++) array[offset + d] *= flip[d];
6329       }
6330     }
6331     offset += dof;
6332   }
6333   PetscCall(PetscSectionRestorePointSyms(section, numPoints, points, &perms, &flips));
6334   *size = offset;
6335   PetscFunctionReturn(PETSC_SUCCESS);
6336 }
6337 
6338 static inline PetscErrorCode DMPlexVecGetClosure_Fields_Static(DM dm, PetscSection section, PetscInt numPoints, const PetscInt points[], PetscInt numFields, const PetscInt clperm[], const PetscScalar vArray[], PetscInt *size, PetscScalar array[])
6339 {
6340   PetscInt offset = 0, f;
6341 
6342   PetscFunctionBeginHot;
6343   *size = 0;
6344   for (f = 0; f < numFields; ++f) {
6345     PetscInt            p;
6346     const PetscInt    **perms = NULL;
6347     const PetscScalar **flips = NULL;
6348 
6349     PetscCall(PetscSectionGetFieldPointSyms(section, f, numPoints, points, &perms, &flips));
6350     for (p = 0; p < numPoints; p++) {
6351       const PetscInt     point = points[2 * p];
6352       PetscInt           fdof, foff, b;
6353       const PetscScalar *varr;
6354       const PetscInt    *perm = perms ? perms[p] : NULL;
6355       const PetscScalar *flip = flips ? flips[p] : NULL;
6356 
6357       PetscCall(PetscSectionGetFieldDof(section, point, f, &fdof));
6358       PetscCall(PetscSectionGetFieldOffset(section, point, f, &foff));
6359       varr = &vArray[foff];
6360       if (clperm) {
6361         if (perm) {
6362           for (b = 0; b < fdof; b++) array[clperm[offset + perm[b]]] = varr[b];
6363         } else {
6364           for (b = 0; b < fdof; b++) array[clperm[offset + b]] = varr[b];
6365         }
6366         if (flip) {
6367           for (b = 0; b < fdof; b++) array[clperm[offset + b]] *= flip[b];
6368         }
6369       } else {
6370         if (perm) {
6371           for (b = 0; b < fdof; b++) array[offset + perm[b]] = varr[b];
6372         } else {
6373           for (b = 0; b < fdof; b++) array[offset + b] = varr[b];
6374         }
6375         if (flip) {
6376           for (b = 0; b < fdof; b++) array[offset + b] *= flip[b];
6377         }
6378       }
6379       offset += fdof;
6380     }
6381     PetscCall(PetscSectionRestoreFieldPointSyms(section, f, numPoints, points, &perms, &flips));
6382   }
6383   *size = offset;
6384   PetscFunctionReturn(PETSC_SUCCESS);
6385 }
6386 
6387 PetscErrorCode DMPlexVecGetOrientedClosure_Internal(DM dm, PetscSection section, PetscBool useClPerm, Vec v, PetscInt point, PetscInt ornt, PetscInt *csize, PetscScalar *values[])
6388 {
6389   PetscSection    clSection;
6390   IS              clPoints;
6391   PetscInt       *points = NULL;
6392   const PetscInt *clp, *perm = NULL;
6393   PetscInt        depth, numFields, numPoints, asize;
6394 
6395   PetscFunctionBeginHot;
6396   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6397   if (!section) PetscCall(DMGetLocalSection(dm, &section));
6398   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
6399   PetscValidHeaderSpecific(v, VEC_CLASSID, 4);
6400   PetscCall(DMPlexGetDepth(dm, &depth));
6401   PetscCall(PetscSectionGetNumFields(section, &numFields));
6402   if (depth == 1 && numFields < 2) {
6403     PetscCall(DMPlexVecGetClosure_Depth1_Static(dm, section, v, point, csize, values));
6404     PetscFunctionReturn(PETSC_SUCCESS);
6405   }
6406   /* Get points */
6407   PetscCall(DMPlexGetCompressedClosure(dm, section, point, ornt, &numPoints, &points, &clSection, &clPoints, &clp));
6408   /* Get sizes */
6409   asize = 0;
6410   for (PetscInt p = 0; p < numPoints * 2; p += 2) {
6411     PetscInt dof;
6412     PetscCall(PetscSectionGetDof(section, points[p], &dof));
6413     asize += dof;
6414   }
6415   if (values) {
6416     const PetscScalar *vArray;
6417     PetscInt           size;
6418 
6419     if (*values) {
6420       PetscCheck(*csize >= asize, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Provided array size %" PetscInt_FMT " not sufficient to hold closure size %" PetscInt_FMT, *csize, asize);
6421     } else PetscCall(DMGetWorkArray(dm, asize, MPIU_SCALAR, values));
6422     if (useClPerm) PetscCall(PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject)dm, depth, asize, &perm));
6423     PetscCall(VecGetArrayRead(v, &vArray));
6424     /* Get values */
6425     if (numFields > 0) PetscCall(DMPlexVecGetClosure_Fields_Static(dm, section, numPoints, points, numFields, perm, vArray, &size, *values));
6426     else PetscCall(DMPlexVecGetClosure_Static(dm, section, numPoints, points, perm, vArray, &size, *values));
6427     PetscCheck(asize == size, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Section size %" PetscInt_FMT " does not match Vec closure size %" PetscInt_FMT, asize, size);
6428     /* Cleanup array */
6429     PetscCall(VecRestoreArrayRead(v, &vArray));
6430   }
6431   if (csize) *csize = asize;
6432   /* Cleanup points */
6433   PetscCall(DMPlexRestoreCompressedClosure(dm, section, point, &numPoints, &points, &clSection, &clPoints, &clp));
6434   PetscFunctionReturn(PETSC_SUCCESS);
6435 }
6436 
6437 /*@C
6438   DMPlexVecGetClosure - Get an array of the values on the closure of 'point'
6439 
6440   Not collective
6441 
6442   Input Parameters:
6443 + dm      - The `DM`
6444 . section - The section describing the layout in `v`, or `NULL` to use the default section
6445 . v       - The local vector
6446 - point   - The point in the `DM`
6447 
6448   Input/Output Parameters:
6449 + csize  - The size of the input values array, or `NULL`; on output the number of values in the closure
6450 - values - An array to use for the values, or *values = `NULL` to have it allocated automatically;
6451            if the user provided `NULL`, it is a borrowed array and should not be freed, use  `DMPlexVecRestoreClosure()` to return it
6452 
6453   Level: intermediate
6454 
6455   Notes:
6456   `DMPlexVecGetClosure()`/`DMPlexVecRestoreClosure()` only allocates the values array if it set to `NULL` in the
6457   calling function. This is because `DMPlexVecGetClosure()` is typically called in the inner loop of a `Vec` or `Mat`
6458   assembly function, and a user may already have allocated storage for this operation.
6459 
6460   A typical use could be
6461 .vb
6462    values = NULL;
6463    PetscCall(DMPlexVecGetClosure(dm, NULL, v, p, &clSize, &values));
6464    for (cl = 0; cl < clSize; ++cl) {
6465      <Compute on closure>
6466    }
6467    PetscCall(DMPlexVecRestoreClosure(dm, NULL, v, p, &clSize, &values));
6468 .ve
6469   or
6470 .vb
6471    PetscMalloc1(clMaxSize, &values);
6472    for (p = pStart; p < pEnd; ++p) {
6473      clSize = clMaxSize;
6474      PetscCall(DMPlexVecGetClosure(dm, NULL, v, p, &clSize, &values));
6475      for (cl = 0; cl < clSize; ++cl) {
6476        <Compute on closure>
6477      }
6478    }
6479    PetscFree(values);
6480 .ve
6481 
6482   Fortran Notes:
6483   The `csize` argument is not present in the Fortran binding.
6484 
6485   `values` must be declared with
6486 .vb
6487   PetscScalar,dimension(:),pointer   :: values
6488 .ve
6489   and it will be allocated internally by PETSc to hold the values returned
6490 
6491 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexVecRestoreClosure()`, `DMPlexVecSetClosure()`, `DMPlexMatSetClosure()`
6492 @*/
6493 PetscErrorCode DMPlexVecGetClosure(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
6494 {
6495   PetscFunctionBeginHot;
6496   PetscCall(DMPlexVecGetOrientedClosure_Internal(dm, section, PETSC_TRUE, v, point, 0, csize, values));
6497   PetscFunctionReturn(PETSC_SUCCESS);
6498 }
6499 
6500 PetscErrorCode DMPlexVecGetClosureAtDepth_Internal(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt depth, PetscInt *csize, PetscScalar *values[])
6501 {
6502   DMLabel            depthLabel;
6503   PetscSection       clSection;
6504   IS                 clPoints;
6505   PetscScalar       *array;
6506   const PetscScalar *vArray;
6507   PetscInt          *points = NULL;
6508   const PetscInt    *clp, *perm = NULL;
6509   PetscInt           mdepth, numFields, numPoints, Np = 0, p, clsize, size;
6510 
6511   PetscFunctionBeginHot;
6512   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
6513   if (!section) PetscCall(DMGetLocalSection(dm, &section));
6514   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
6515   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
6516   PetscCall(DMPlexGetDepth(dm, &mdepth));
6517   PetscCall(DMPlexGetDepthLabel(dm, &depthLabel));
6518   PetscCall(PetscSectionGetNumFields(section, &numFields));
6519   if (mdepth == 1 && numFields < 2) {
6520     PetscCall(DMPlexVecGetClosure_Depth1_Static(dm, section, v, point, csize, values));
6521     PetscFunctionReturn(PETSC_SUCCESS);
6522   }
6523   /* Get points */
6524   PetscCall(DMPlexGetCompressedClosure(dm, section, point, 0, &numPoints, &points, &clSection, &clPoints, &clp));
6525   for (clsize = 0, p = 0; p < Np; p++) {
6526     PetscInt dof;
6527     PetscCall(PetscSectionGetDof(section, points[2 * p], &dof));
6528     clsize += dof;
6529   }
6530   PetscCall(PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject)dm, depth, clsize, &perm));
6531   /* Filter points */
6532   for (p = 0; p < numPoints * 2; p += 2) {
6533     PetscInt dep;
6534 
6535     PetscCall(DMLabelGetValue(depthLabel, points[p], &dep));
6536     if (dep != depth) continue;
6537     points[Np * 2 + 0] = points[p];
6538     points[Np * 2 + 1] = points[p + 1];
6539     ++Np;
6540   }
6541   /* Get array */
6542   if (!values || !*values) {
6543     PetscInt asize = 0, dof;
6544 
6545     for (p = 0; p < Np * 2; p += 2) {
6546       PetscCall(PetscSectionGetDof(section, points[p], &dof));
6547       asize += dof;
6548     }
6549     if (!values) {
6550       PetscCall(DMPlexRestoreCompressedClosure(dm, section, point, &numPoints, &points, &clSection, &clPoints, &clp));
6551       if (csize) *csize = asize;
6552       PetscFunctionReturn(PETSC_SUCCESS);
6553     }
6554     PetscCall(DMGetWorkArray(dm, asize, MPIU_SCALAR, &array));
6555   } else {
6556     array = *values;
6557   }
6558   PetscCall(VecGetArrayRead(v, &vArray));
6559   /* Get values */
6560   if (numFields > 0) PetscCall(DMPlexVecGetClosure_Fields_Static(dm, section, Np, points, numFields, perm, vArray, &size, array));
6561   else PetscCall(DMPlexVecGetClosure_Static(dm, section, Np, points, perm, vArray, &size, array));
6562   /* Cleanup points */
6563   PetscCall(DMPlexRestoreCompressedClosure(dm, section, point, &numPoints, &points, &clSection, &clPoints, &clp));
6564   /* Cleanup array */
6565   PetscCall(VecRestoreArrayRead(v, &vArray));
6566   if (!*values) {
6567     if (csize) *csize = size;
6568     *values = array;
6569   } else {
6570     PetscCheck(size <= *csize, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Size of input array %" PetscInt_FMT " < actual size %" PetscInt_FMT, *csize, size);
6571     *csize = size;
6572   }
6573   PetscFunctionReturn(PETSC_SUCCESS);
6574 }
6575 
6576 /*@C
6577   DMPlexVecRestoreClosure - Restore the array of the values on the closure of 'point' obtained with `DMPlexVecGetClosure()`
6578 
6579   Not collective
6580 
6581   Input Parameters:
6582 + dm      - The `DM`
6583 . section - The section describing the layout in `v`, or `NULL` to use the default section
6584 . v       - The local vector
6585 . point   - The point in the `DM`
6586 . csize   - The number of values in the closure, or `NULL`
6587 - values  - The array of values
6588 
6589   Level: intermediate
6590 
6591   Note:
6592   The array values are discarded and not copied back into `v`. In order to copy values back to `v`, use `DMPlexVecSetClosure()`
6593 
6594   Fortran Note:
6595   The `csize` argument is not present in the Fortran binding since it is internal to the array.
6596 
6597 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexVecGetClosure()`, `DMPlexVecSetClosure()`, `DMPlexMatSetClosure()`
6598 @*/
6599 PetscErrorCode DMPlexVecRestoreClosure(DM dm, PetscSection section, Vec v, PetscInt point, PetscInt *csize, PetscScalar *values[])
6600 {
6601   PetscInt size = 0;
6602 
6603   PetscFunctionBegin;
6604   /* Should work without recalculating size */
6605   PetscCall(DMRestoreWorkArray(dm, size, MPIU_SCALAR, (void *)values));
6606   *values = NULL;
6607   PetscFunctionReturn(PETSC_SUCCESS);
6608 }
6609 
6610 static inline void add(PetscScalar *x, PetscScalar y)
6611 {
6612   *x += y;
6613 }
6614 static inline void insert(PetscScalar *x, PetscScalar y)
6615 {
6616   *x = y;
6617 }
6618 
6619 static inline PetscErrorCode updatePoint_private(PetscSection section, PetscInt point, PetscInt dof, void (*fuse)(PetscScalar *, PetscScalar), PetscBool setBC, const PetscInt perm[], const PetscScalar flip[], const PetscInt clperm[], const PetscScalar values[], PetscInt offset, PetscScalar array[])
6620 {
6621   PetscInt        cdof;  /* The number of constraints on this point */
6622   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
6623   PetscScalar    *a;
6624   PetscInt        off, cind = 0, k;
6625 
6626   PetscFunctionBegin;
6627   PetscCall(PetscSectionGetConstraintDof(section, point, &cdof));
6628   PetscCall(PetscSectionGetOffset(section, point, &off));
6629   a = &array[off];
6630   if (!cdof || setBC) {
6631     if (clperm) {
6632       if (perm) {
6633         for (k = 0; k < dof; ++k) fuse(&a[k], values[clperm[offset + perm[k]]] * (flip ? flip[perm[k]] : 1.));
6634       } else {
6635         for (k = 0; k < dof; ++k) fuse(&a[k], values[clperm[offset + k]] * (flip ? flip[k] : 1.));
6636       }
6637     } else {
6638       if (perm) {
6639         for (k = 0; k < dof; ++k) fuse(&a[k], values[offset + perm[k]] * (flip ? flip[perm[k]] : 1.));
6640       } else {
6641         for (k = 0; k < dof; ++k) fuse(&a[k], values[offset + k] * (flip ? flip[k] : 1.));
6642       }
6643     }
6644   } else {
6645     PetscCall(PetscSectionGetConstraintIndices(section, point, &cdofs));
6646     if (clperm) {
6647       if (perm) {
6648         for (k = 0; k < dof; ++k) {
6649           if ((cind < cdof) && (k == cdofs[cind])) {
6650             ++cind;
6651             continue;
6652           }
6653           fuse(&a[k], values[clperm[offset + perm[k]]] * (flip ? flip[perm[k]] : 1.));
6654         }
6655       } else {
6656         for (k = 0; k < dof; ++k) {
6657           if ((cind < cdof) && (k == cdofs[cind])) {
6658             ++cind;
6659             continue;
6660           }
6661           fuse(&a[k], values[clperm[offset + k]] * (flip ? flip[k] : 1.));
6662         }
6663       }
6664     } else {
6665       if (perm) {
6666         for (k = 0; k < dof; ++k) {
6667           if ((cind < cdof) && (k == cdofs[cind])) {
6668             ++cind;
6669             continue;
6670           }
6671           fuse(&a[k], values[offset + perm[k]] * (flip ? flip[perm[k]] : 1.));
6672         }
6673       } else {
6674         for (k = 0; k < dof; ++k) {
6675           if ((cind < cdof) && (k == cdofs[cind])) {
6676             ++cind;
6677             continue;
6678           }
6679           fuse(&a[k], values[offset + k] * (flip ? flip[k] : 1.));
6680         }
6681       }
6682     }
6683   }
6684   PetscFunctionReturn(PETSC_SUCCESS);
6685 }
6686 
6687 static inline PetscErrorCode updatePointBC_private(PetscSection section, PetscInt point, PetscInt dof, void (*fuse)(PetscScalar *, PetscScalar), const PetscInt perm[], const PetscScalar flip[], const PetscInt clperm[], const PetscScalar values[], PetscInt offset, PetscScalar array[])
6688 {
6689   PetscInt        cdof;  /* The number of constraints on this point */
6690   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
6691   PetscScalar    *a;
6692   PetscInt        off, cind = 0, k;
6693 
6694   PetscFunctionBegin;
6695   PetscCall(PetscSectionGetConstraintDof(section, point, &cdof));
6696   PetscCall(PetscSectionGetOffset(section, point, &off));
6697   a = &array[off];
6698   if (cdof) {
6699     PetscCall(PetscSectionGetConstraintIndices(section, point, &cdofs));
6700     if (clperm) {
6701       if (perm) {
6702         for (k = 0; k < dof; ++k) {
6703           if ((cind < cdof) && (k == cdofs[cind])) {
6704             fuse(&a[k], values[clperm[offset + perm[k]]] * (flip ? flip[perm[k]] : 1.));
6705             cind++;
6706           }
6707         }
6708       } else {
6709         for (k = 0; k < dof; ++k) {
6710           if ((cind < cdof) && (k == cdofs[cind])) {
6711             fuse(&a[k], values[clperm[offset + k]] * (flip ? flip[k] : 1.));
6712             cind++;
6713           }
6714         }
6715       }
6716     } else {
6717       if (perm) {
6718         for (k = 0; k < dof; ++k) {
6719           if ((cind < cdof) && (k == cdofs[cind])) {
6720             fuse(&a[k], values[offset + perm[k]] * (flip ? flip[perm[k]] : 1.));
6721             cind++;
6722           }
6723         }
6724       } else {
6725         for (k = 0; k < dof; ++k) {
6726           if ((cind < cdof) && (k == cdofs[cind])) {
6727             fuse(&a[k], values[offset + k] * (flip ? flip[k] : 1.));
6728             cind++;
6729           }
6730         }
6731       }
6732     }
6733   }
6734   PetscFunctionReturn(PETSC_SUCCESS);
6735 }
6736 
6737 static inline PetscErrorCode updatePointFields_private(PetscSection section, PetscInt point, const PetscInt *perm, const PetscScalar *flip, PetscInt f, void (*fuse)(PetscScalar *, PetscScalar), PetscBool setBC, const PetscInt clperm[], const PetscScalar values[], PetscInt *offset, PetscScalar array[])
6738 {
6739   PetscScalar    *a;
6740   PetscInt        fdof, foff, fcdof, foffset = *offset;
6741   const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
6742   PetscInt        cind = 0, b;
6743 
6744   PetscFunctionBegin;
6745   PetscCall(PetscSectionGetFieldDof(section, point, f, &fdof));
6746   PetscCall(PetscSectionGetFieldConstraintDof(section, point, f, &fcdof));
6747   PetscCall(PetscSectionGetFieldOffset(section, point, f, &foff));
6748   a = &array[foff];
6749   if (!fcdof || setBC) {
6750     if (clperm) {
6751       if (perm) {
6752         for (b = 0; b < fdof; b++) fuse(&a[b], values[clperm[foffset + perm[b]]] * (flip ? flip[perm[b]] : 1.));
6753       } else {
6754         for (b = 0; b < fdof; b++) fuse(&a[b], values[clperm[foffset + b]] * (flip ? flip[b] : 1.));
6755       }
6756     } else {
6757       if (perm) {
6758         for (b = 0; b < fdof; b++) fuse(&a[b], values[foffset + perm[b]] * (flip ? flip[perm[b]] : 1.));
6759       } else {
6760         for (b = 0; b < fdof; b++) fuse(&a[b], values[foffset + b] * (flip ? flip[b] : 1.));
6761       }
6762     }
6763   } else {
6764     PetscCall(PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs));
6765     if (clperm) {
6766       if (perm) {
6767         for (b = 0; b < fdof; b++) {
6768           if ((cind < fcdof) && (b == fcdofs[cind])) {
6769             ++cind;
6770             continue;
6771           }
6772           fuse(&a[b], values[clperm[foffset + perm[b]]] * (flip ? flip[perm[b]] : 1.));
6773         }
6774       } else {
6775         for (b = 0; b < fdof; b++) {
6776           if ((cind < fcdof) && (b == fcdofs[cind])) {
6777             ++cind;
6778             continue;
6779           }
6780           fuse(&a[b], values[clperm[foffset + b]] * (flip ? flip[b] : 1.));
6781         }
6782       }
6783     } else {
6784       if (perm) {
6785         for (b = 0; b < fdof; b++) {
6786           if ((cind < fcdof) && (b == fcdofs[cind])) {
6787             ++cind;
6788             continue;
6789           }
6790           fuse(&a[b], values[foffset + perm[b]] * (flip ? flip[perm[b]] : 1.));
6791         }
6792       } else {
6793         for (b = 0; b < fdof; b++) {
6794           if ((cind < fcdof) && (b == fcdofs[cind])) {
6795             ++cind;
6796             continue;
6797           }
6798           fuse(&a[b], values[foffset + b] * (flip ? flip[b] : 1.));
6799         }
6800       }
6801     }
6802   }
6803   *offset += fdof;
6804   PetscFunctionReturn(PETSC_SUCCESS);
6805 }
6806 
6807 static inline PetscErrorCode updatePointFieldsBC_private(PetscSection section, PetscInt point, const PetscInt perm[], const PetscScalar flip[], PetscInt f, PetscInt Ncc, const PetscInt comps[], void (*fuse)(PetscScalar *, PetscScalar), const PetscInt clperm[], const PetscScalar values[], PetscInt *offset, PetscScalar array[])
6808 {
6809   PetscScalar    *a;
6810   PetscInt        fdof, foff, fcdof, foffset = *offset;
6811   const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
6812   PetscInt        Nc, cind = 0, ncind = 0, b;
6813   PetscBool       ncSet, fcSet;
6814 
6815   PetscFunctionBegin;
6816   PetscCall(PetscSectionGetFieldComponents(section, f, &Nc));
6817   PetscCall(PetscSectionGetFieldDof(section, point, f, &fdof));
6818   PetscCall(PetscSectionGetFieldConstraintDof(section, point, f, &fcdof));
6819   PetscCall(PetscSectionGetFieldOffset(section, point, f, &foff));
6820   a = &array[foff];
6821   if (fcdof) {
6822     /* We just override fcdof and fcdofs with Ncc and comps */
6823     PetscCall(PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs));
6824     if (clperm) {
6825       if (perm) {
6826         if (comps) {
6827           for (b = 0; b < fdof; b++) {
6828             ncSet = fcSet = PETSC_FALSE;
6829             if (b % Nc == comps[ncind]) {
6830               ncind = (ncind + 1) % Ncc;
6831               ncSet = PETSC_TRUE;
6832             }
6833             if ((cind < fcdof) && (b == fcdofs[cind])) {
6834               ++cind;
6835               fcSet = PETSC_TRUE;
6836             }
6837             if (ncSet && fcSet) fuse(&a[b], values[clperm[foffset + perm[b]]] * (flip ? flip[perm[b]] : 1.));
6838           }
6839         } else {
6840           for (b = 0; b < fdof; b++) {
6841             if ((cind < fcdof) && (b == fcdofs[cind])) {
6842               fuse(&a[b], values[clperm[foffset + perm[b]]] * (flip ? flip[perm[b]] : 1.));
6843               ++cind;
6844             }
6845           }
6846         }
6847       } else {
6848         if (comps) {
6849           for (b = 0; b < fdof; b++) {
6850             ncSet = fcSet = PETSC_FALSE;
6851             if (b % Nc == comps[ncind]) {
6852               ncind = (ncind + 1) % Ncc;
6853               ncSet = PETSC_TRUE;
6854             }
6855             if ((cind < fcdof) && (b == fcdofs[cind])) {
6856               ++cind;
6857               fcSet = PETSC_TRUE;
6858             }
6859             if (ncSet && fcSet) fuse(&a[b], values[clperm[foffset + b]] * (flip ? flip[b] : 1.));
6860           }
6861         } else {
6862           for (b = 0; b < fdof; b++) {
6863             if ((cind < fcdof) && (b == fcdofs[cind])) {
6864               fuse(&a[b], values[clperm[foffset + b]] * (flip ? flip[b] : 1.));
6865               ++cind;
6866             }
6867           }
6868         }
6869       }
6870     } else {
6871       if (perm) {
6872         if (comps) {
6873           for (b = 0; b < fdof; b++) {
6874             ncSet = fcSet = PETSC_FALSE;
6875             if (b % Nc == comps[ncind]) {
6876               ncind = (ncind + 1) % Ncc;
6877               ncSet = PETSC_TRUE;
6878             }
6879             if ((cind < fcdof) && (b == fcdofs[cind])) {
6880               ++cind;
6881               fcSet = PETSC_TRUE;
6882             }
6883             if (ncSet && fcSet) fuse(&a[b], values[foffset + perm[b]] * (flip ? flip[perm[b]] : 1.));
6884           }
6885         } else {
6886           for (b = 0; b < fdof; b++) {
6887             if ((cind < fcdof) && (b == fcdofs[cind])) {
6888               fuse(&a[b], values[foffset + perm[b]] * (flip ? flip[perm[b]] : 1.));
6889               ++cind;
6890             }
6891           }
6892         }
6893       } else {
6894         if (comps) {
6895           for (b = 0; b < fdof; b++) {
6896             ncSet = fcSet = PETSC_FALSE;
6897             if (b % Nc == comps[ncind]) {
6898               ncind = (ncind + 1) % Ncc;
6899               ncSet = PETSC_TRUE;
6900             }
6901             if ((cind < fcdof) && (b == fcdofs[cind])) {
6902               ++cind;
6903               fcSet = PETSC_TRUE;
6904             }
6905             if (ncSet && fcSet) fuse(&a[b], values[foffset + b] * (flip ? flip[b] : 1.));
6906           }
6907         } else {
6908           for (b = 0; b < fdof; b++) {
6909             if ((cind < fcdof) && (b == fcdofs[cind])) {
6910               fuse(&a[b], values[foffset + b] * (flip ? flip[b] : 1.));
6911               ++cind;
6912             }
6913           }
6914         }
6915       }
6916     }
6917   }
6918   *offset += fdof;
6919   PetscFunctionReturn(PETSC_SUCCESS);
6920 }
6921 
6922 static inline PetscErrorCode DMPlexVecSetClosure_Depth1_Static(DM dm, PetscSection section, Vec v, PetscInt point, const PetscScalar values[], InsertMode mode)
6923 {
6924   PetscScalar    *array;
6925   const PetscInt *cone, *coneO;
6926   PetscInt        pStart, pEnd, p, numPoints, off, dof;
6927 
6928   PetscFunctionBeginHot;
6929   PetscCall(PetscSectionGetChart(section, &pStart, &pEnd));
6930   PetscCall(DMPlexGetConeSize(dm, point, &numPoints));
6931   PetscCall(DMPlexGetCone(dm, point, &cone));
6932   PetscCall(DMPlexGetConeOrientation(dm, point, &coneO));
6933   PetscCall(VecGetArray(v, &array));
6934   for (p = 0, off = 0; p <= numPoints; ++p, off += dof) {
6935     const PetscInt cp = !p ? point : cone[p - 1];
6936     const PetscInt o  = !p ? 0 : coneO[p - 1];
6937 
6938     if ((cp < pStart) || (cp >= pEnd)) {
6939       dof = 0;
6940       continue;
6941     }
6942     PetscCall(PetscSectionGetDof(section, cp, &dof));
6943     /* ADD_VALUES */
6944     {
6945       const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
6946       PetscScalar    *a;
6947       PetscInt        cdof, coff, cind = 0, k;
6948 
6949       PetscCall(PetscSectionGetConstraintDof(section, cp, &cdof));
6950       PetscCall(PetscSectionGetOffset(section, cp, &coff));
6951       a = &array[coff];
6952       if (!cdof) {
6953         if (o >= 0) {
6954           for (k = 0; k < dof; ++k) a[k] += values[off + k];
6955         } else {
6956           for (k = 0; k < dof; ++k) a[k] += values[off + dof - k - 1];
6957         }
6958       } else {
6959         PetscCall(PetscSectionGetConstraintIndices(section, cp, &cdofs));
6960         if (o >= 0) {
6961           for (k = 0; k < dof; ++k) {
6962             if ((cind < cdof) && (k == cdofs[cind])) {
6963               ++cind;
6964               continue;
6965             }
6966             a[k] += values[off + k];
6967           }
6968         } else {
6969           for (k = 0; k < dof; ++k) {
6970             if ((cind < cdof) && (k == cdofs[cind])) {
6971               ++cind;
6972               continue;
6973             }
6974             a[k] += values[off + dof - k - 1];
6975           }
6976         }
6977       }
6978     }
6979   }
6980   PetscCall(VecRestoreArray(v, &array));
6981   PetscFunctionReturn(PETSC_SUCCESS);
6982 }
6983 
6984 /*@C
6985   DMPlexVecSetClosure - Set an array of the values on the closure of `point`
6986 
6987   Not collective
6988 
6989   Input Parameters:
6990 + dm      - The `DM`
6991 . section - The section describing the layout in `v`, or `NULL` to use the default section
6992 . v       - The local vector
6993 . point   - The point in the `DM`
6994 . values  - The array of values
6995 - mode    - The insert mode. One of `INSERT_ALL_VALUES`, `ADD_ALL_VALUES`, `INSERT_VALUES`, `ADD_VALUES`, `INSERT_BC_VALUES`, and `ADD_BC_VALUES`,
6996             where `INSERT_ALL_VALUES` and `ADD_ALL_VALUES` also overwrite boundary conditions.
6997 
6998   Level: intermediate
6999 
7000   Note:
7001   Usually the input arrays were obtained with `DMPlexVecGetClosure()`
7002 
7003   Fortran Note:
7004   `values` must be declared with
7005 .vb
7006   PetscScalar,dimension(:),pointer   :: values
7007 .ve
7008 
7009 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexVecGetClosure()`, `DMPlexMatSetClosure()`
7010 @*/
7011 PetscErrorCode DMPlexVecSetClosure(DM dm, PetscSection section, Vec v, PetscInt point, const PetscScalar values[], InsertMode mode)
7012 {
7013   PetscSection    clSection;
7014   IS              clPoints;
7015   PetscScalar    *array;
7016   PetscInt       *points = NULL;
7017   const PetscInt *clp, *clperm = NULL;
7018   PetscInt        depth, numFields, numPoints, p, clsize;
7019 
7020   PetscFunctionBeginHot;
7021   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7022   if (!section) PetscCall(DMGetLocalSection(dm, &section));
7023   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
7024   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
7025   PetscCall(DMPlexGetDepth(dm, &depth));
7026   PetscCall(PetscSectionGetNumFields(section, &numFields));
7027   if (depth == 1 && numFields < 2 && mode == ADD_VALUES) {
7028     PetscCall(DMPlexVecSetClosure_Depth1_Static(dm, section, v, point, values, mode));
7029     PetscFunctionReturn(PETSC_SUCCESS);
7030   }
7031   /* Get points */
7032   PetscCall(DMPlexGetCompressedClosure(dm, section, point, 0, &numPoints, &points, &clSection, &clPoints, &clp));
7033   for (clsize = 0, p = 0; p < numPoints; p++) {
7034     PetscInt dof;
7035     PetscCall(PetscSectionGetDof(section, points[2 * p], &dof));
7036     clsize += dof;
7037   }
7038   PetscCall(PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject)dm, depth, clsize, &clperm));
7039   /* Get array */
7040   PetscCall(VecGetArray(v, &array));
7041   /* Get values */
7042   if (numFields > 0) {
7043     PetscInt offset = 0, f;
7044     for (f = 0; f < numFields; ++f) {
7045       const PetscInt    **perms = NULL;
7046       const PetscScalar **flips = NULL;
7047 
7048       PetscCall(PetscSectionGetFieldPointSyms(section, f, numPoints, points, &perms, &flips));
7049       switch (mode) {
7050       case INSERT_VALUES:
7051         for (p = 0; p < numPoints; p++) {
7052           const PetscInt     point = points[2 * p];
7053           const PetscInt    *perm  = perms ? perms[p] : NULL;
7054           const PetscScalar *flip  = flips ? flips[p] : NULL;
7055           PetscCall(updatePointFields_private(section, point, perm, flip, f, insert, PETSC_FALSE, clperm, values, &offset, array));
7056         }
7057         break;
7058       case INSERT_ALL_VALUES:
7059         for (p = 0; p < numPoints; p++) {
7060           const PetscInt     point = points[2 * p];
7061           const PetscInt    *perm  = perms ? perms[p] : NULL;
7062           const PetscScalar *flip  = flips ? flips[p] : NULL;
7063           PetscCall(updatePointFields_private(section, point, perm, flip, f, insert, PETSC_TRUE, clperm, values, &offset, array));
7064         }
7065         break;
7066       case INSERT_BC_VALUES:
7067         for (p = 0; p < numPoints; p++) {
7068           const PetscInt     point = points[2 * p];
7069           const PetscInt    *perm  = perms ? perms[p] : NULL;
7070           const PetscScalar *flip  = flips ? flips[p] : NULL;
7071           PetscCall(updatePointFieldsBC_private(section, point, perm, flip, f, -1, NULL, insert, clperm, values, &offset, array));
7072         }
7073         break;
7074       case ADD_VALUES:
7075         for (p = 0; p < numPoints; p++) {
7076           const PetscInt     point = points[2 * p];
7077           const PetscInt    *perm  = perms ? perms[p] : NULL;
7078           const PetscScalar *flip  = flips ? flips[p] : NULL;
7079           PetscCall(updatePointFields_private(section, point, perm, flip, f, add, PETSC_FALSE, clperm, values, &offset, array));
7080         }
7081         break;
7082       case ADD_ALL_VALUES:
7083         for (p = 0; p < numPoints; p++) {
7084           const PetscInt     point = points[2 * p];
7085           const PetscInt    *perm  = perms ? perms[p] : NULL;
7086           const PetscScalar *flip  = flips ? flips[p] : NULL;
7087           PetscCall(updatePointFields_private(section, point, perm, flip, f, add, PETSC_TRUE, clperm, values, &offset, array));
7088         }
7089         break;
7090       case ADD_BC_VALUES:
7091         for (p = 0; p < numPoints; p++) {
7092           const PetscInt     point = points[2 * p];
7093           const PetscInt    *perm  = perms ? perms[p] : NULL;
7094           const PetscScalar *flip  = flips ? flips[p] : NULL;
7095           PetscCall(updatePointFieldsBC_private(section, point, perm, flip, f, -1, NULL, add, clperm, values, &offset, array));
7096         }
7097         break;
7098       default:
7099         SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %d", mode);
7100       }
7101       PetscCall(PetscSectionRestoreFieldPointSyms(section, f, numPoints, points, &perms, &flips));
7102     }
7103   } else {
7104     PetscInt            dof, off;
7105     const PetscInt    **perms = NULL;
7106     const PetscScalar **flips = NULL;
7107 
7108     PetscCall(PetscSectionGetPointSyms(section, numPoints, points, &perms, &flips));
7109     switch (mode) {
7110     case INSERT_VALUES:
7111       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
7112         const PetscInt     point = points[2 * p];
7113         const PetscInt    *perm  = perms ? perms[p] : NULL;
7114         const PetscScalar *flip  = flips ? flips[p] : NULL;
7115         PetscCall(PetscSectionGetDof(section, point, &dof));
7116         PetscCall(updatePoint_private(section, point, dof, insert, PETSC_FALSE, perm, flip, clperm, values, off, array));
7117       }
7118       break;
7119     case INSERT_ALL_VALUES:
7120       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
7121         const PetscInt     point = points[2 * p];
7122         const PetscInt    *perm  = perms ? perms[p] : NULL;
7123         const PetscScalar *flip  = flips ? flips[p] : NULL;
7124         PetscCall(PetscSectionGetDof(section, point, &dof));
7125         PetscCall(updatePoint_private(section, point, dof, insert, PETSC_TRUE, perm, flip, clperm, values, off, array));
7126       }
7127       break;
7128     case INSERT_BC_VALUES:
7129       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
7130         const PetscInt     point = points[2 * p];
7131         const PetscInt    *perm  = perms ? perms[p] : NULL;
7132         const PetscScalar *flip  = flips ? flips[p] : NULL;
7133         PetscCall(PetscSectionGetDof(section, point, &dof));
7134         PetscCall(updatePointBC_private(section, point, dof, insert, perm, flip, clperm, values, off, array));
7135       }
7136       break;
7137     case ADD_VALUES:
7138       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
7139         const PetscInt     point = points[2 * p];
7140         const PetscInt    *perm  = perms ? perms[p] : NULL;
7141         const PetscScalar *flip  = flips ? flips[p] : NULL;
7142         PetscCall(PetscSectionGetDof(section, point, &dof));
7143         PetscCall(updatePoint_private(section, point, dof, add, PETSC_FALSE, perm, flip, clperm, values, off, array));
7144       }
7145       break;
7146     case ADD_ALL_VALUES:
7147       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
7148         const PetscInt     point = points[2 * p];
7149         const PetscInt    *perm  = perms ? perms[p] : NULL;
7150         const PetscScalar *flip  = flips ? flips[p] : NULL;
7151         PetscCall(PetscSectionGetDof(section, point, &dof));
7152         PetscCall(updatePoint_private(section, point, dof, add, PETSC_TRUE, perm, flip, clperm, values, off, array));
7153       }
7154       break;
7155     case ADD_BC_VALUES:
7156       for (p = 0, off = 0; p < numPoints; p++, off += dof) {
7157         const PetscInt     point = points[2 * p];
7158         const PetscInt    *perm  = perms ? perms[p] : NULL;
7159         const PetscScalar *flip  = flips ? flips[p] : NULL;
7160         PetscCall(PetscSectionGetDof(section, point, &dof));
7161         PetscCall(updatePointBC_private(section, point, dof, add, perm, flip, clperm, values, off, array));
7162       }
7163       break;
7164     default:
7165       SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %d", mode);
7166     }
7167     PetscCall(PetscSectionRestorePointSyms(section, numPoints, points, &perms, &flips));
7168   }
7169   /* Cleanup points */
7170   PetscCall(DMPlexRestoreCompressedClosure(dm, section, point, &numPoints, &points, &clSection, &clPoints, &clp));
7171   /* Cleanup array */
7172   PetscCall(VecRestoreArray(v, &array));
7173   PetscFunctionReturn(PETSC_SUCCESS);
7174 }
7175 
7176 /* Check whether the given point is in the label. If not, update the offset to skip this point */
7177 static inline PetscErrorCode CheckPoint_Private(DMLabel label, PetscInt labelId, PetscSection section, PetscInt point, PetscInt f, PetscInt *offset, PetscBool *contains)
7178 {
7179   PetscFunctionBegin;
7180   *contains = PETSC_TRUE;
7181   if (label) {
7182     PetscInt fdof;
7183 
7184     PetscCall(DMLabelStratumHasPoint(label, labelId, point, contains));
7185     if (!*contains) {
7186       PetscCall(PetscSectionGetFieldDof(section, point, f, &fdof));
7187       *offset += fdof;
7188       PetscFunctionReturn(PETSC_SUCCESS);
7189     }
7190   }
7191   PetscFunctionReturn(PETSC_SUCCESS);
7192 }
7193 
7194 /* Unlike DMPlexVecSetClosure(), this uses plex-native closure permutation, not a user-specified permutation such as DMPlexSetClosurePermutationTensor(). */
7195 PetscErrorCode DMPlexVecSetFieldClosure_Internal(DM dm, PetscSection section, Vec v, PetscBool fieldActive[], PetscInt point, PetscInt Ncc, const PetscInt comps[], DMLabel label, PetscInt labelId, const PetscScalar values[], InsertMode mode)
7196 {
7197   PetscSection    clSection;
7198   IS              clPoints;
7199   PetscScalar    *array;
7200   PetscInt       *points = NULL;
7201   const PetscInt *clp;
7202   PetscInt        numFields, numPoints, p;
7203   PetscInt        offset = 0, f;
7204 
7205   PetscFunctionBeginHot;
7206   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7207   if (!section) PetscCall(DMGetLocalSection(dm, &section));
7208   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
7209   PetscValidHeaderSpecific(v, VEC_CLASSID, 3);
7210   PetscCall(PetscSectionGetNumFields(section, &numFields));
7211   /* Get points */
7212   PetscCall(DMPlexGetCompressedClosure(dm, section, point, 0, &numPoints, &points, &clSection, &clPoints, &clp));
7213   /* Get array */
7214   PetscCall(VecGetArray(v, &array));
7215   /* Get values */
7216   for (f = 0; f < numFields; ++f) {
7217     const PetscInt    **perms = NULL;
7218     const PetscScalar **flips = NULL;
7219     PetscBool           contains;
7220 
7221     if (!fieldActive[f]) {
7222       for (p = 0; p < numPoints * 2; p += 2) {
7223         PetscInt fdof;
7224         PetscCall(PetscSectionGetFieldDof(section, points[p], f, &fdof));
7225         offset += fdof;
7226       }
7227       continue;
7228     }
7229     PetscCall(PetscSectionGetFieldPointSyms(section, f, numPoints, points, &perms, &flips));
7230     switch (mode) {
7231     case INSERT_VALUES:
7232       for (p = 0; p < numPoints; p++) {
7233         const PetscInt     point = points[2 * p];
7234         const PetscInt    *perm  = perms ? perms[p] : NULL;
7235         const PetscScalar *flip  = flips ? flips[p] : NULL;
7236         PetscCall(CheckPoint_Private(label, labelId, section, point, f, &offset, &contains));
7237         if (!contains) continue;
7238         PetscCall(updatePointFields_private(section, point, perm, flip, f, insert, PETSC_FALSE, NULL, values, &offset, array));
7239       }
7240       break;
7241     case INSERT_ALL_VALUES:
7242       for (p = 0; p < numPoints; p++) {
7243         const PetscInt     point = points[2 * p];
7244         const PetscInt    *perm  = perms ? perms[p] : NULL;
7245         const PetscScalar *flip  = flips ? flips[p] : NULL;
7246         PetscCall(CheckPoint_Private(label, labelId, section, point, f, &offset, &contains));
7247         if (!contains) continue;
7248         PetscCall(updatePointFields_private(section, point, perm, flip, f, insert, PETSC_TRUE, NULL, values, &offset, array));
7249       }
7250       break;
7251     case INSERT_BC_VALUES:
7252       for (p = 0; p < numPoints; p++) {
7253         const PetscInt     point = points[2 * p];
7254         const PetscInt    *perm  = perms ? perms[p] : NULL;
7255         const PetscScalar *flip  = flips ? flips[p] : NULL;
7256         PetscCall(CheckPoint_Private(label, labelId, section, point, f, &offset, &contains));
7257         if (!contains) continue;
7258         PetscCall(updatePointFieldsBC_private(section, point, perm, flip, f, Ncc, comps, insert, NULL, values, &offset, array));
7259       }
7260       break;
7261     case ADD_VALUES:
7262       for (p = 0; p < numPoints; p++) {
7263         const PetscInt     point = points[2 * p];
7264         const PetscInt    *perm  = perms ? perms[p] : NULL;
7265         const PetscScalar *flip  = flips ? flips[p] : NULL;
7266         PetscCall(CheckPoint_Private(label, labelId, section, point, f, &offset, &contains));
7267         if (!contains) continue;
7268         PetscCall(updatePointFields_private(section, point, perm, flip, f, add, PETSC_FALSE, NULL, values, &offset, array));
7269       }
7270       break;
7271     case ADD_ALL_VALUES:
7272       for (p = 0; p < numPoints; p++) {
7273         const PetscInt     point = points[2 * p];
7274         const PetscInt    *perm  = perms ? perms[p] : NULL;
7275         const PetscScalar *flip  = flips ? flips[p] : NULL;
7276         PetscCall(CheckPoint_Private(label, labelId, section, point, f, &offset, &contains));
7277         if (!contains) continue;
7278         PetscCall(updatePointFields_private(section, point, perm, flip, f, add, PETSC_TRUE, NULL, values, &offset, array));
7279       }
7280       break;
7281     default:
7282       SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Invalid insert mode %d", mode);
7283     }
7284     PetscCall(PetscSectionRestoreFieldPointSyms(section, f, numPoints, points, &perms, &flips));
7285   }
7286   /* Cleanup points */
7287   PetscCall(DMPlexRestoreCompressedClosure(dm, section, point, &numPoints, &points, &clSection, &clPoints, &clp));
7288   /* Cleanup array */
7289   PetscCall(VecRestoreArray(v, &array));
7290   PetscFunctionReturn(PETSC_SUCCESS);
7291 }
7292 
7293 static PetscErrorCode DMPlexPrintMatSetValues(PetscViewer viewer, Mat A, PetscInt point, PetscInt numRIndices, const PetscInt rindices[], PetscInt numCIndices, const PetscInt cindices[], const PetscScalar values[])
7294 {
7295   PetscMPIInt rank;
7296   PetscInt    i, j;
7297 
7298   PetscFunctionBegin;
7299   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank));
7300   PetscCall(PetscViewerASCIIPrintf(viewer, "[%d]mat for point %" PetscInt_FMT "\n", rank, point));
7301   for (i = 0; i < numRIndices; i++) PetscCall(PetscViewerASCIIPrintf(viewer, "[%d]mat row indices[%" PetscInt_FMT "] = %" PetscInt_FMT "\n", rank, i, rindices[i]));
7302   for (i = 0; i < numCIndices; i++) PetscCall(PetscViewerASCIIPrintf(viewer, "[%d]mat col indices[%" PetscInt_FMT "] = %" PetscInt_FMT "\n", rank, i, cindices[i]));
7303   numCIndices = numCIndices ? numCIndices : numRIndices;
7304   if (!values) PetscFunctionReturn(PETSC_SUCCESS);
7305   for (i = 0; i < numRIndices; i++) {
7306     PetscCall(PetscViewerASCIIPrintf(viewer, "[%d]", rank));
7307     for (j = 0; j < numCIndices; j++) {
7308 #if defined(PETSC_USE_COMPLEX)
7309       PetscCall(PetscViewerASCIIPrintf(viewer, " (%g,%g)", (double)PetscRealPart(values[i * numCIndices + j]), (double)PetscImaginaryPart(values[i * numCIndices + j])));
7310 #else
7311       PetscCall(PetscViewerASCIIPrintf(viewer, " %g", (double)values[i * numCIndices + j]));
7312 #endif
7313     }
7314     PetscCall(PetscViewerASCIIPrintf(viewer, "\n"));
7315   }
7316   PetscFunctionReturn(PETSC_SUCCESS);
7317 }
7318 
7319 /*
7320   DMPlexGetIndicesPoint_Internal - Add the indices for dofs on a point to an index array
7321 
7322   Input Parameters:
7323 + section - The section for this data layout
7324 . islocal - Is the section (and thus indices being requested) local or global?
7325 . point   - The point contributing dofs with these indices
7326 . off     - The global offset of this point
7327 . loff    - The local offset of each field
7328 . setBC   - The flag determining whether to include indices of boundary values
7329 . perm    - A permutation of the dofs on this point, or NULL
7330 - indperm - A permutation of the entire indices array, or NULL
7331 
7332   Output Parameter:
7333 . indices - Indices for dofs on this point
7334 
7335   Level: developer
7336 
7337   Note: The indices could be local or global, depending on the value of 'off'.
7338 */
7339 PetscErrorCode DMPlexGetIndicesPoint_Internal(PetscSection section, PetscBool islocal, PetscInt point, PetscInt off, PetscInt *loff, PetscBool setBC, const PetscInt perm[], const PetscInt indperm[], PetscInt indices[])
7340 {
7341   PetscInt        dof;   /* The number of unknowns on this point */
7342   PetscInt        cdof;  /* The number of constraints on this point */
7343   const PetscInt *cdofs; /* The indices of the constrained dofs on this point */
7344   PetscInt        cind = 0, k;
7345 
7346   PetscFunctionBegin;
7347   PetscCheck(islocal || !setBC, PetscObjectComm((PetscObject)section), PETSC_ERR_ARG_INCOMP, "setBC incompatible with global indices; use a local section or disable setBC");
7348   PetscCall(PetscSectionGetDof(section, point, &dof));
7349   PetscCall(PetscSectionGetConstraintDof(section, point, &cdof));
7350   if (!cdof || setBC) {
7351     for (k = 0; k < dof; ++k) {
7352       const PetscInt preind = perm ? *loff + perm[k] : *loff + k;
7353       const PetscInt ind    = indperm ? indperm[preind] : preind;
7354 
7355       indices[ind] = off + k;
7356     }
7357   } else {
7358     PetscCall(PetscSectionGetConstraintIndices(section, point, &cdofs));
7359     for (k = 0; k < dof; ++k) {
7360       const PetscInt preind = perm ? *loff + perm[k] : *loff + k;
7361       const PetscInt ind    = indperm ? indperm[preind] : preind;
7362 
7363       if ((cind < cdof) && (k == cdofs[cind])) {
7364         /* Insert check for returning constrained indices */
7365         indices[ind] = -(off + k + 1);
7366         ++cind;
7367       } else {
7368         indices[ind] = off + k - (islocal ? 0 : cind);
7369       }
7370     }
7371   }
7372   *loff += dof;
7373   PetscFunctionReturn(PETSC_SUCCESS);
7374 }
7375 
7376 /*
7377  DMPlexGetIndicesPointFields_Internal - gets section indices for a point in its canonical ordering.
7378 
7379  Input Parameters:
7380 + section - a section (global or local)
7381 - islocal - `PETSC_TRUE` if requesting local indices (i.e., section is local); `PETSC_FALSE` for global
7382 . point - point within section
7383 . off - The offset of this point in the (local or global) indexed space - should match islocal and (usually) the section
7384 . foffs - array of length numFields containing the offset in canonical point ordering (the location in indices) of each field
7385 . setBC - identify constrained (boundary condition) points via involution.
7386 . perms - perms[f][permsoff][:] is a permutation of dofs within each field
7387 . permsoff - offset
7388 - indperm - index permutation
7389 
7390  Output Parameter:
7391 . foffs - each entry is incremented by the number of (unconstrained if setBC=FALSE) dofs in that field
7392 . indices - array to hold indices (as defined by section) of each dof associated with point
7393 
7394  Notes:
7395  If section is local and setBC=true, there is no distinction between constrained and unconstrained dofs.
7396  If section is local and setBC=false, the indices for constrained points are the involution -(i+1) of their position
7397  in the local vector.
7398 
7399  If section is global and setBC=false, the indices for constrained points are negative (and their value is not
7400  significant).  It is invalid to call with a global section and setBC=true.
7401 
7402  Developer Note:
7403  The section is only used for field layout, so islocal is technically a statement about the offset (off).  At some point
7404  in the future, global sections may have fields set, in which case we could pass the global section and obtain the
7405  offset could be obtained from the section instead of passing it explicitly as we do now.
7406 
7407  Example:
7408  Suppose a point contains one field with three components, and for which the unconstrained indices are {10, 11, 12}.
7409  When the middle component is constrained, we get the array {10, -12, 12} for (islocal=TRUE, setBC=FALSE).
7410  Note that -12 is the involution of 11, so the user can involute negative indices to recover local indices.
7411  The global vector does not store constrained dofs, so when this function returns global indices, say {110, -112, 111}, the value of -112 is an arbitrary flag that should not be interpreted beyond its sign.
7412 
7413  Level: developer
7414 */
7415 PetscErrorCode DMPlexGetIndicesPointFields_Internal(PetscSection section, PetscBool islocal, PetscInt point, PetscInt off, PetscInt foffs[], PetscBool setBC, const PetscInt ***perms, PetscInt permsoff, const PetscInt indperm[], PetscInt indices[])
7416 {
7417   PetscInt numFields, foff, f;
7418 
7419   PetscFunctionBegin;
7420   PetscCheck(islocal || !setBC, PetscObjectComm((PetscObject)section), PETSC_ERR_ARG_INCOMP, "setBC incompatible with global indices; use a local section or disable setBC");
7421   PetscCall(PetscSectionGetNumFields(section, &numFields));
7422   for (f = 0, foff = 0; f < numFields; ++f) {
7423     PetscInt        fdof, cfdof;
7424     const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
7425     PetscInt        cind = 0, b;
7426     const PetscInt *perm = (perms && perms[f]) ? perms[f][permsoff] : NULL;
7427 
7428     PetscCall(PetscSectionGetFieldDof(section, point, f, &fdof));
7429     PetscCall(PetscSectionGetFieldConstraintDof(section, point, f, &cfdof));
7430     if (!cfdof || setBC) {
7431       for (b = 0; b < fdof; ++b) {
7432         const PetscInt preind = perm ? foffs[f] + perm[b] : foffs[f] + b;
7433         const PetscInt ind    = indperm ? indperm[preind] : preind;
7434 
7435         indices[ind] = off + foff + b;
7436       }
7437     } else {
7438       PetscCall(PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs));
7439       for (b = 0; b < fdof; ++b) {
7440         const PetscInt preind = perm ? foffs[f] + perm[b] : foffs[f] + b;
7441         const PetscInt ind    = indperm ? indperm[preind] : preind;
7442 
7443         if ((cind < cfdof) && (b == fcdofs[cind])) {
7444           indices[ind] = -(off + foff + b + 1);
7445           ++cind;
7446         } else {
7447           indices[ind] = off + foff + b - (islocal ? 0 : cind);
7448         }
7449       }
7450     }
7451     foff += (setBC || islocal ? fdof : (fdof - cfdof));
7452     foffs[f] += fdof;
7453   }
7454   PetscFunctionReturn(PETSC_SUCCESS);
7455 }
7456 
7457 /*
7458   This version believes the globalSection offsets for each field, rather than just the point offset
7459 
7460  . foffs - The offset into 'indices' for each field, since it is segregated by field
7461 
7462  Notes:
7463  The semantics of this function relate to that of setBC=FALSE in DMPlexGetIndicesPointFields_Internal.
7464  Since this function uses global indices, setBC=TRUE would be invalid, so no such argument exists.
7465 */
7466 static PetscErrorCode DMPlexGetIndicesPointFieldsSplit_Internal(PetscSection section, PetscSection globalSection, PetscInt point, PetscInt foffs[], const PetscInt ***perms, PetscInt permsoff, const PetscInt indperm[], PetscInt indices[])
7467 {
7468   PetscInt numFields, foff, f;
7469 
7470   PetscFunctionBegin;
7471   PetscCall(PetscSectionGetNumFields(section, &numFields));
7472   for (f = 0; f < numFields; ++f) {
7473     PetscInt        fdof, cfdof;
7474     const PetscInt *fcdofs; /* The indices of the constrained dofs for field f on this point */
7475     PetscInt        cind = 0, b;
7476     const PetscInt *perm = (perms && perms[f]) ? perms[f][permsoff] : NULL;
7477 
7478     PetscCall(PetscSectionGetFieldDof(section, point, f, &fdof));
7479     PetscCall(PetscSectionGetFieldConstraintDof(section, point, f, &cfdof));
7480     PetscCall(PetscSectionGetFieldOffset(globalSection, point, f, &foff));
7481     if (!cfdof) {
7482       for (b = 0; b < fdof; ++b) {
7483         const PetscInt preind = perm ? foffs[f] + perm[b] : foffs[f] + b;
7484         const PetscInt ind    = indperm ? indperm[preind] : preind;
7485 
7486         indices[ind] = foff + b;
7487       }
7488     } else {
7489       PetscCall(PetscSectionGetFieldConstraintIndices(section, point, f, &fcdofs));
7490       for (b = 0; b < fdof; ++b) {
7491         const PetscInt preind = perm ? foffs[f] + perm[b] : foffs[f] + b;
7492         const PetscInt ind    = indperm ? indperm[preind] : preind;
7493 
7494         if ((cind < cfdof) && (b == fcdofs[cind])) {
7495           indices[ind] = -(foff + b + 1);
7496           ++cind;
7497         } else {
7498           indices[ind] = foff + b - cind;
7499         }
7500       }
7501     }
7502     foffs[f] += fdof;
7503   }
7504   PetscFunctionReturn(PETSC_SUCCESS);
7505 }
7506 
7507 static PetscErrorCode DMPlexAnchorsGetSubMatIndices(PetscInt nPoints, const PetscInt pnts[], PetscSection section, PetscSection cSec, PetscInt tmpIndices[], PetscInt fieldOffsets[], PetscInt indices[], const PetscInt ***perms)
7508 {
7509   PetscInt numFields, sStart, sEnd, cStart, cEnd;
7510 
7511   PetscFunctionBegin;
7512   PetscCall(PetscSectionGetNumFields(section, &numFields));
7513   PetscCall(PetscSectionGetChart(section, &sStart, &sEnd));
7514   PetscCall(PetscSectionGetChart(cSec, &cStart, &cEnd));
7515   for (PetscInt p = 0; p < nPoints; p++) {
7516     PetscInt     b       = pnts[2 * p];
7517     PetscInt     bSecDof = 0, bOff;
7518     PetscInt     cSecDof = 0;
7519     PetscSection indices_section;
7520 
7521     if (b >= sStart && b < sEnd) PetscCall(PetscSectionGetDof(section, b, &bSecDof));
7522     if (!bSecDof) continue;
7523     if (b >= cStart && b < cEnd) PetscCall(PetscSectionGetDof(cSec, b, &cSecDof));
7524     indices_section = cSecDof > 0 ? cSec : section;
7525     if (numFields) {
7526       PetscInt fStart[32], fEnd[32];
7527 
7528       fStart[0] = 0;
7529       fEnd[0]   = 0;
7530       for (PetscInt f = 0; f < numFields; f++) {
7531         PetscInt fDof = 0;
7532 
7533         PetscCall(PetscSectionGetFieldDof(indices_section, b, f, &fDof));
7534         fStart[f + 1] = fStart[f] + fDof;
7535         fEnd[f + 1]   = fStart[f + 1];
7536       }
7537       PetscCall(PetscSectionGetOffset(indices_section, b, &bOff));
7538       // only apply permutations on one side
7539       PetscCall(DMPlexGetIndicesPointFields_Internal(indices_section, PETSC_TRUE, b, bOff, fEnd, PETSC_TRUE, perms, perms ? p : -1, NULL, tmpIndices));
7540       for (PetscInt f = 0; f < numFields; f++) {
7541         for (PetscInt i = fStart[f]; i < fEnd[f]; i++) { indices[fieldOffsets[f]++] = (cSecDof > 0) ? tmpIndices[i] : -(tmpIndices[i] + 1); }
7542       }
7543     } else {
7544       PetscInt bEnd = 0;
7545 
7546       PetscCall(PetscSectionGetOffset(indices_section, b, &bOff));
7547       PetscCall(DMPlexGetIndicesPoint_Internal(indices_section, PETSC_TRUE, b, bOff, &bEnd, PETSC_TRUE, (perms && perms[0]) ? perms[0][p] : NULL, NULL, tmpIndices));
7548 
7549       for (PetscInt i = 0; i < bEnd; i++) indices[fieldOffsets[0]++] = (cSecDof > 0) ? tmpIndices[i] : -(tmpIndices[i] + 1);
7550     }
7551   }
7552   PetscFunctionReturn(PETSC_SUCCESS);
7553 }
7554 
7555 PETSC_INTERN PetscErrorCode DMPlexAnchorsGetSubMatModification(DM dm, PetscSection section, PetscInt numPoints, PetscInt numIndices, const PetscInt points[], const PetscInt ***perms, PetscInt *outNumPoints, PetscInt *outNumIndices, PetscInt *outPoints[], PetscInt offsets[], PetscScalar *outMat[])
7556 {
7557   Mat             cMat;
7558   PetscSection    aSec, cSec;
7559   IS              aIS;
7560   PetscInt        aStart = -1, aEnd = -1;
7561   PetscInt        sStart = -1, sEnd = -1;
7562   PetscInt        cStart = -1, cEnd = -1;
7563   const PetscInt *anchors;
7564   PetscInt        numFields, p;
7565   PetscInt        newNumPoints = 0, newNumIndices = 0;
7566   PetscInt       *newPoints, *indices, *newIndices, *tmpIndices, *tmpNewIndices;
7567   PetscInt        oldOffsets[32];
7568   PetscInt        newOffsets[32];
7569   PetscInt        oldOffsetsCopy[32];
7570   PetscInt        newOffsetsCopy[32];
7571   PetscScalar    *modMat         = NULL;
7572   PetscBool       anyConstrained = PETSC_FALSE;
7573 
7574   PetscFunctionBegin;
7575   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7576   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
7577   PetscCall(PetscSectionGetNumFields(section, &numFields));
7578 
7579   PetscCall(DMPlexGetAnchors(dm, &aSec, &aIS));
7580   /* if there are point-to-point constraints */
7581   if (aSec) {
7582     PetscCall(PetscArrayzero(newOffsets, 32));
7583     PetscCall(PetscArrayzero(oldOffsets, 32));
7584     PetscCall(ISGetIndices(aIS, &anchors));
7585     PetscCall(PetscSectionGetChart(aSec, &aStart, &aEnd));
7586     PetscCall(PetscSectionGetChart(section, &sStart, &sEnd));
7587     /* figure out how many points are going to be in the new element matrix
7588      * (we allow double counting, because it's all just going to be summed
7589      * into the global matrix anyway) */
7590     for (p = 0; p < 2 * numPoints; p += 2) {
7591       PetscInt b    = points[p];
7592       PetscInt bDof = 0, bSecDof = 0;
7593 
7594       if (b >= sStart && b < sEnd) PetscCall(PetscSectionGetDof(section, b, &bSecDof));
7595       if (!bSecDof) continue;
7596 
7597       for (PetscInt f = 0; f < numFields; f++) {
7598         PetscInt fDof = 0;
7599 
7600         PetscCall(PetscSectionGetFieldDof(section, b, f, &fDof));
7601         oldOffsets[f + 1] += fDof;
7602       }
7603       if (b >= aStart && b < aEnd) PetscCall(PetscSectionGetDof(aSec, b, &bDof));
7604       if (bDof) {
7605         /* this point is constrained */
7606         /* it is going to be replaced by its anchors */
7607         PetscInt bOff, q;
7608 
7609         PetscCall(PetscSectionGetOffset(aSec, b, &bOff));
7610         for (q = 0; q < bDof; q++) {
7611           PetscInt a    = anchors[bOff + q];
7612           PetscInt aDof = 0;
7613 
7614           if (a >= sStart && a < sEnd) PetscCall(PetscSectionGetDof(section, a, &aDof));
7615           if (aDof) {
7616             anyConstrained = PETSC_TRUE;
7617             newNumPoints += 1;
7618           }
7619           newNumIndices += aDof;
7620           for (PetscInt f = 0; f < numFields; ++f) {
7621             PetscInt fDof = 0;
7622 
7623             if (a >= sStart && a < sEnd) PetscCall(PetscSectionGetFieldDof(section, a, f, &fDof));
7624             newOffsets[f + 1] += fDof;
7625           }
7626         }
7627       } else {
7628         /* this point is not constrained */
7629         newNumPoints++;
7630         newNumIndices += bSecDof;
7631         for (PetscInt f = 0; f < numFields; ++f) {
7632           PetscInt fDof;
7633 
7634           PetscCall(PetscSectionGetFieldDof(section, b, f, &fDof));
7635           newOffsets[f + 1] += fDof;
7636         }
7637       }
7638     }
7639   }
7640   if (!anyConstrained) {
7641     if (outNumPoints) *outNumPoints = 0;
7642     if (outNumIndices) *outNumIndices = 0;
7643     if (outPoints) *outPoints = NULL;
7644     if (outMat) *outMat = NULL;
7645     if (aSec) PetscCall(ISRestoreIndices(aIS, &anchors));
7646     PetscFunctionReturn(PETSC_SUCCESS);
7647   }
7648 
7649   if (outNumPoints) *outNumPoints = newNumPoints;
7650   if (outNumIndices) *outNumIndices = newNumIndices;
7651 
7652   for (PetscInt f = 0; f < numFields; ++f) newOffsets[f + 1] += newOffsets[f];
7653   for (PetscInt f = 0; f < numFields; ++f) oldOffsets[f + 1] += oldOffsets[f];
7654 
7655   if (!outPoints && !outMat) {
7656     if (offsets) {
7657       for (PetscInt f = 0; f <= numFields; f++) offsets[f] = newOffsets[f];
7658     }
7659     if (aSec) PetscCall(ISRestoreIndices(aIS, &anchors));
7660     PetscFunctionReturn(PETSC_SUCCESS);
7661   }
7662 
7663   PetscCheck(!numFields || newOffsets[numFields] == newNumIndices, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid size for closure %" PetscInt_FMT " should be %" PetscInt_FMT, newOffsets[numFields], newNumIndices);
7664   PetscCheck(!numFields || oldOffsets[numFields] == numIndices, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid size for closure %" PetscInt_FMT " should be %" PetscInt_FMT, oldOffsets[numFields], numIndices);
7665 
7666   PetscCall(DMGetDefaultConstraints(dm, &cSec, &cMat, NULL));
7667   PetscCall(PetscSectionGetChart(cSec, &cStart, &cEnd));
7668 
7669   /* output arrays */
7670   PetscCall(DMGetWorkArray(dm, 2 * newNumPoints, MPIU_INT, &newPoints));
7671   PetscCall(PetscArrayzero(newPoints, 2 * newNumPoints));
7672 
7673   // get the new Points
7674   for (PetscInt p = 0, newP = 0; p < numPoints; p++) {
7675     PetscInt b    = points[2 * p];
7676     PetscInt bDof = 0, bSecDof = 0, bOff;
7677 
7678     if (b >= sStart && b < sEnd) PetscCall(PetscSectionGetDof(section, b, &bSecDof));
7679     if (!bSecDof) continue;
7680     if (b >= aStart && b < aEnd) PetscCall(PetscSectionGetDof(aSec, b, &bDof));
7681     if (bDof) {
7682       PetscCall(PetscSectionGetOffset(aSec, b, &bOff));
7683       for (PetscInt q = 0; q < bDof; q++) {
7684         PetscInt a = anchors[bOff + q], aDof = 0;
7685 
7686         if (a >= sStart && a < sEnd) PetscCall(PetscSectionGetDof(section, a, &aDof));
7687         if (aDof) {
7688           newPoints[2 * newP]     = a;
7689           newPoints[2 * newP + 1] = 0; // orientations are accounted for in constructing the matrix, newly added points are in default orientation
7690           newP++;
7691         }
7692       }
7693     } else {
7694       newPoints[2 * newP]     = b;
7695       newPoints[2 * newP + 1] = points[2 * p + 1];
7696       newP++;
7697     }
7698   }
7699 
7700   if (outMat) {
7701     PetscScalar *tmpMat;
7702     PetscCall(PetscArraycpy(oldOffsetsCopy, oldOffsets, 32));
7703     PetscCall(PetscArraycpy(newOffsetsCopy, newOffsets, 32));
7704 
7705     PetscCall(DMGetWorkArray(dm, numIndices, MPIU_INT, &indices));
7706     PetscCall(DMGetWorkArray(dm, numIndices, MPIU_INT, &tmpIndices));
7707     PetscCall(DMGetWorkArray(dm, newNumIndices, MPIU_INT, &newIndices));
7708     PetscCall(DMGetWorkArray(dm, newNumIndices, MPIU_INT, &tmpNewIndices));
7709 
7710     for (PetscInt i = 0; i < numIndices; i++) indices[i] = -1;
7711     for (PetscInt i = 0; i < newNumIndices; i++) newIndices[i] = -1;
7712 
7713     PetscCall(DMPlexAnchorsGetSubMatIndices(numPoints, points, section, cSec, tmpIndices, oldOffsetsCopy, indices, perms));
7714     PetscCall(DMPlexAnchorsGetSubMatIndices(newNumPoints, newPoints, section, section, tmpNewIndices, newOffsetsCopy, newIndices, NULL));
7715 
7716     PetscCall(DMGetWorkArray(dm, numIndices * newNumIndices, MPIU_SCALAR, &modMat));
7717     PetscCall(DMGetWorkArray(dm, numIndices * newNumIndices, MPIU_SCALAR, &tmpMat));
7718     PetscCall(PetscArrayzero(modMat, newNumIndices * numIndices));
7719     // for each field, insert the anchor modification into modMat
7720     for (PetscInt f = 0; f < PetscMax(1, numFields); f++) {
7721       PetscInt fStart    = oldOffsets[f];
7722       PetscInt fNewStart = newOffsets[f];
7723       for (PetscInt p = 0, newP = 0, o = fStart, oNew = fNewStart; p < numPoints; p++) {
7724         PetscInt b    = points[2 * p];
7725         PetscInt bDof = 0, bSecDof = 0, bOff;
7726 
7727         if (b >= sStart && b < sEnd) {
7728           if (numFields) {
7729             PetscCall(PetscSectionGetFieldDof(section, b, f, &bSecDof));
7730           } else {
7731             PetscCall(PetscSectionGetDof(section, b, &bSecDof));
7732           }
7733         }
7734         if (!bSecDof) continue;
7735         if (b >= aStart && b < aEnd) PetscCall(PetscSectionGetDof(aSec, b, &bDof));
7736         if (bDof) {
7737           PetscCall(PetscSectionGetOffset(aSec, b, &bOff));
7738           for (PetscInt q = 0; q < bDof; q++, newP++) {
7739             PetscInt a = anchors[bOff + q], aDof = 0;
7740 
7741             if (a >= sStart && a < sEnd) {
7742               if (numFields) {
7743                 PetscCall(PetscSectionGetFieldDof(section, a, f, &aDof));
7744               } else {
7745                 PetscCall(PetscSectionGetDof(section, a, &aDof));
7746               }
7747             }
7748             if (aDof) {
7749               PetscCall(MatGetValues(cMat, bSecDof, &indices[o], aDof, &newIndices[oNew], tmpMat));
7750               for (PetscInt d = 0; d < bSecDof; d++) {
7751                 for (PetscInt e = 0; e < aDof; e++) modMat[(o + d) * newNumIndices + oNew + e] = tmpMat[d * aDof + e];
7752               }
7753             }
7754             oNew += aDof;
7755           }
7756         } else {
7757           // Insert the identity matrix in this block
7758           for (PetscInt d = 0; d < bSecDof; d++) modMat[(o + d) * newNumIndices + oNew + d] = 1;
7759           oNew += bSecDof;
7760           newP++;
7761         }
7762         o += bSecDof;
7763       }
7764     }
7765 
7766     *outMat = modMat;
7767 
7768     PetscCall(DMRestoreWorkArray(dm, numIndices * newNumIndices, MPIU_SCALAR, &tmpMat));
7769     PetscCall(DMRestoreWorkArray(dm, newNumIndices, MPIU_INT, &tmpNewIndices));
7770     PetscCall(DMRestoreWorkArray(dm, newNumIndices, MPIU_INT, &newIndices));
7771     PetscCall(DMRestoreWorkArray(dm, numIndices, MPIU_INT, &tmpIndices));
7772     PetscCall(DMRestoreWorkArray(dm, numIndices, MPIU_INT, &indices));
7773   }
7774   PetscCall(ISRestoreIndices(aIS, &anchors));
7775 
7776   /* output */
7777   if (outPoints) {
7778     *outPoints = newPoints;
7779   } else {
7780     PetscCall(DMRestoreWorkArray(dm, 2 * newNumPoints, MPIU_INT, &newPoints));
7781   }
7782   for (PetscInt f = 0; f <= numFields; f++) offsets[f] = newOffsets[f];
7783   PetscFunctionReturn(PETSC_SUCCESS);
7784 }
7785 
7786 PETSC_INTERN PetscErrorCode DMPlexAnchorsModifyMat_Internal(DM dm, PetscSection section, PetscInt numPoints, PetscInt numIndices, const PetscInt points[], const PetscInt ***perms, PetscInt numRows, PetscInt numCols, const PetscScalar values[], PetscInt *outNumPoints, PetscInt *outNumIndices, PetscInt *outPoints[], PetscScalar *outValues[], PetscInt offsets[], PetscBool multiplyRight, PetscBool multiplyLeft)
7787 {
7788   PetscScalar *modMat        = NULL;
7789   PetscInt     newNumIndices = -1;
7790 
7791   PetscFunctionBegin;
7792   /* If M is the matrix represented by values, get the matrix C such that we will add M * C (or, if multiplyLeft, C^T * M * C) into the global matrix.
7793      modMat is that matrix C */
7794   PetscCall(DMPlexAnchorsGetSubMatModification(dm, section, numPoints, numIndices, points, perms, outNumPoints, &newNumIndices, outPoints, offsets, outValues ? &modMat : NULL));
7795   if (outNumIndices) *outNumIndices = newNumIndices;
7796   if (modMat) {
7797     const PetscScalar *newValues = values;
7798 
7799     if (multiplyRight) {
7800       PetscScalar *newNewValues = NULL;
7801       PetscBLASInt M, N, K;
7802       PetscScalar  a = 1.0, b = 0.0;
7803 
7804       PetscCheck(numCols == numIndices, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_SIZ, "values matrix has the wrong number of columns: %" PetscInt_FMT ", expected %" PetscInt_FMT, numCols, numIndices);
7805 
7806       PetscCall(PetscBLASIntCast(newNumIndices, &M));
7807       PetscCall(PetscBLASIntCast(numRows, &N));
7808       PetscCall(PetscBLASIntCast(numIndices, &K));
7809       PetscCall(DMGetWorkArray(dm, numRows * newNumIndices, MPIU_SCALAR, &newNewValues));
7810       // row-major to column-major conversion, right multiplication becomes left multiplication
7811       PetscCallBLAS("BLASgemm", BLASgemm_("N", "N", &M, &N, &K, &a, modMat, &M, newValues, &K, &b, newNewValues, &M));
7812       numCols   = newNumIndices;
7813       newValues = newNewValues;
7814     }
7815 
7816     if (multiplyLeft) {
7817       PetscScalar *newNewValues = NULL;
7818       PetscBLASInt M, N, K;
7819       PetscScalar  a = 1.0, b = 0.0;
7820 
7821       PetscCheck(numRows == numIndices, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_SIZ, "values matrix has the wrong number of rows: %" PetscInt_FMT ", expected %" PetscInt_FMT, numRows, numIndices);
7822 
7823       PetscCall(PetscBLASIntCast(numCols, &M));
7824       PetscCall(PetscBLASIntCast(newNumIndices, &N));
7825       PetscCall(PetscBLASIntCast(numIndices, &K));
7826       PetscCall(DMGetWorkArray(dm, newNumIndices * numCols, MPIU_SCALAR, &newNewValues));
7827       // row-major to column-major conversion, left multiplication becomes right multiplication
7828       PetscCallBLAS("BLASgemm", BLASgemm_("N", "T", &M, &N, &K, &a, newValues, &M, modMat, &N, &b, newNewValues, &M));
7829       if (newValues != values) PetscCall(DMRestoreWorkArray(dm, numIndices * newNumIndices, MPIU_SCALAR, &newValues));
7830       newValues = newNewValues;
7831     }
7832     *outValues = (PetscScalar *)newValues;
7833     PetscCall(DMRestoreWorkArray(dm, numIndices * newNumIndices, MPIU_SCALAR, &modMat));
7834   }
7835   PetscFunctionReturn(PETSC_SUCCESS);
7836 }
7837 
7838 PETSC_INTERN PetscErrorCode DMPlexAnchorsModifyMat(DM dm, PetscSection section, PetscInt numPoints, PetscInt numIndices, const PetscInt points[], const PetscInt ***perms, const PetscScalar values[], PetscInt *outNumPoints, PetscInt *outNumIndices, PetscInt *outPoints[], PetscScalar *outValues[], PetscInt offsets[], PetscBool multiplyLeft)
7839 {
7840   PetscFunctionBegin;
7841   PetscCall(DMPlexAnchorsModifyMat_Internal(dm, section, numPoints, numIndices, points, perms, numIndices, numIndices, values, outNumPoints, outNumIndices, outPoints, outValues, offsets, PETSC_TRUE, multiplyLeft));
7842   PetscFunctionReturn(PETSC_SUCCESS);
7843 }
7844 
7845 static PetscErrorCode DMPlexGetClosureIndicesSize_Internal(DM dm, PetscSection section, PetscInt point, PetscInt *closureSize)
7846 {
7847   /* Closure ordering */
7848   PetscSection    clSection;
7849   IS              clPoints;
7850   const PetscInt *clp;
7851   PetscInt       *points;
7852   PetscInt        Ncl, Ni = 0;
7853 
7854   PetscFunctionBeginHot;
7855   PetscCall(DMPlexGetCompressedClosure(dm, section, point, 0, &Ncl, &points, &clSection, &clPoints, &clp));
7856   for (PetscInt p = 0; p < Ncl * 2; p += 2) {
7857     PetscInt dof;
7858 
7859     PetscCall(PetscSectionGetDof(section, points[p], &dof));
7860     Ni += dof;
7861   }
7862   PetscCall(DMPlexRestoreCompressedClosure(dm, section, point, &Ncl, &points, &clSection, &clPoints, &clp));
7863   *closureSize = Ni;
7864   PetscFunctionReturn(PETSC_SUCCESS);
7865 }
7866 
7867 static PetscErrorCode DMPlexGetClosureIndices_Internal(DM dm, PetscSection section, PetscSection idxSection, PetscInt point, PetscBool useClPerm, PetscInt *numRows, PetscInt *numCols, PetscInt *indices[], PetscInt outOffsets[], PetscScalar *values[], PetscBool multiplyRight, PetscBool multiplyLeft)
7868 {
7869   /* Closure ordering */
7870   PetscSection    clSection;
7871   IS              clPoints;
7872   const PetscInt *clp;
7873   PetscInt       *points;
7874   const PetscInt *clperm = NULL;
7875   /* Dof permutation and sign flips */
7876   const PetscInt    **perms[32] = {NULL};
7877   const PetscScalar **flips[32] = {NULL};
7878   PetscScalar        *valCopy   = NULL;
7879   /* Hanging node constraints */
7880   PetscInt    *pointsC = NULL;
7881   PetscScalar *valuesC = NULL;
7882   PetscInt     NclC, NiC;
7883 
7884   PetscInt *idx;
7885   PetscInt  Nf, Ncl, Ni = 0, offsets[32], p, f;
7886   PetscBool isLocal = (section == idxSection) ? PETSC_TRUE : PETSC_FALSE;
7887   PetscInt  idxStart, idxEnd;
7888   PetscInt  nRows, nCols;
7889 
7890   PetscFunctionBeginHot;
7891   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
7892   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
7893   PetscValidHeaderSpecific(idxSection, PETSC_SECTION_CLASSID, 3);
7894   PetscAssertPointer(numRows, 6);
7895   PetscAssertPointer(numCols, 7);
7896   if (indices) PetscAssertPointer(indices, 8);
7897   if (outOffsets) PetscAssertPointer(outOffsets, 9);
7898   if (values) PetscAssertPointer(values, 10);
7899   PetscCall(PetscSectionGetNumFields(section, &Nf));
7900   PetscCheck(Nf <= 31, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %" PetscInt_FMT " limited to 31", Nf);
7901   PetscCall(PetscArrayzero(offsets, 32));
7902   /* 1) Get points in closure */
7903   PetscCall(DMPlexGetCompressedClosure(dm, section, point, 0, &Ncl, &points, &clSection, &clPoints, &clp));
7904   if (useClPerm) {
7905     PetscInt depth, clsize;
7906     PetscCall(DMPlexGetPointDepth(dm, point, &depth));
7907     for (clsize = 0, p = 0; p < Ncl; p++) {
7908       PetscInt dof;
7909       PetscCall(PetscSectionGetDof(section, points[2 * p], &dof));
7910       clsize += dof;
7911     }
7912     PetscCall(PetscSectionGetClosureInversePermutation_Internal(section, (PetscObject)dm, depth, clsize, &clperm));
7913   }
7914   /* 2) Get number of indices on these points and field offsets from section */
7915   for (p = 0; p < Ncl * 2; p += 2) {
7916     PetscInt dof, fdof;
7917 
7918     PetscCall(PetscSectionGetDof(section, points[p], &dof));
7919     for (f = 0; f < Nf; ++f) {
7920       PetscCall(PetscSectionGetFieldDof(section, points[p], f, &fdof));
7921       offsets[f + 1] += fdof;
7922     }
7923     Ni += dof;
7924   }
7925   if (*numRows == -1) *numRows = Ni;
7926   if (*numCols == -1) *numCols = Ni;
7927   nRows = *numRows;
7928   nCols = *numCols;
7929   for (f = 1; f < Nf; ++f) offsets[f + 1] += offsets[f];
7930   PetscCheck(!Nf || offsets[Nf] == Ni, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Invalid size for closure %" PetscInt_FMT " should be %" PetscInt_FMT, offsets[Nf], Ni);
7931   /* 3) Get symmetries and sign flips. Apply sign flips to values if passed in (only works for square values matrix) */
7932   if (multiplyRight) PetscCheck(nCols == Ni, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_SIZ, "Expected %" PetscInt_FMT " columns, got %" PetscInt_FMT, Ni, nCols);
7933   if (multiplyLeft) PetscCheck(nRows == Ni, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_SIZ, "Expected %" PetscInt_FMT " rows, got %" PetscInt_FMT, Ni, nRows);
7934   for (f = 0; f < PetscMax(1, Nf); ++f) {
7935     if (Nf) PetscCall(PetscSectionGetFieldPointSyms(section, f, Ncl, points, &perms[f], &flips[f]));
7936     else PetscCall(PetscSectionGetPointSyms(section, Ncl, points, &perms[f], &flips[f]));
7937     /* may need to apply sign changes to the element matrix */
7938     if (values && flips[f]) {
7939       PetscInt foffset = offsets[f];
7940 
7941       for (p = 0; p < Ncl; ++p) {
7942         PetscInt           pnt  = points[2 * p], fdof;
7943         const PetscScalar *flip = flips[f] ? flips[f][p] : NULL;
7944 
7945         if (!Nf) PetscCall(PetscSectionGetDof(section, pnt, &fdof));
7946         else PetscCall(PetscSectionGetFieldDof(section, pnt, f, &fdof));
7947         if (flip) {
7948           PetscInt i, j, k;
7949 
7950           if (!valCopy) {
7951             PetscCall(DMGetWorkArray(dm, Ni * Ni, MPIU_SCALAR, &valCopy));
7952             for (j = 0; j < Ni * Ni; ++j) valCopy[j] = (*values)[j];
7953             *values = valCopy;
7954           }
7955           for (i = 0; i < fdof; ++i) {
7956             PetscScalar fval = flip[i];
7957 
7958             if (multiplyRight) {
7959               for (k = 0; k < nRows; ++k) { valCopy[Ni * k + (foffset + i)] *= fval; }
7960             }
7961             if (multiplyLeft) {
7962               for (k = 0; k < nCols; ++k) { valCopy[nCols * (foffset + i) + k] *= fval; }
7963             }
7964           }
7965         }
7966         foffset += fdof;
7967       }
7968     }
7969   }
7970   /* 4) Apply hanging node constraints. Get new symmetries and replace all storage with constrained storage */
7971   PetscCall(DMPlexAnchorsModifyMat_Internal(dm, section, Ncl, Ni, points, perms, nRows, nCols, values ? *values : NULL, &NclC, &NiC, &pointsC, values ? &valuesC : NULL, offsets, multiplyRight, multiplyLeft));
7972   if (NclC) {
7973     if (multiplyRight) *numCols = NiC;
7974     if (multiplyLeft) *numRows = NiC;
7975     if (valCopy) PetscCall(DMRestoreWorkArray(dm, Ni * Ni, MPIU_SCALAR, &valCopy));
7976     for (f = 0; f < PetscMax(1, Nf); ++f) {
7977       if (Nf) PetscCall(PetscSectionRestoreFieldPointSyms(section, f, Ncl, points, &perms[f], &flips[f]));
7978       else PetscCall(PetscSectionRestorePointSyms(section, Ncl, points, &perms[f], &flips[f]));
7979     }
7980     for (f = 0; f < PetscMax(1, Nf); ++f) {
7981       if (Nf) PetscCall(PetscSectionGetFieldPointSyms(section, f, NclC, pointsC, &perms[f], &flips[f]));
7982       else PetscCall(PetscSectionGetPointSyms(section, NclC, pointsC, &perms[f], &flips[f]));
7983     }
7984     PetscCall(DMPlexRestoreCompressedClosure(dm, section, point, &Ncl, &points, &clSection, &clPoints, &clp));
7985     Ncl    = NclC;
7986     Ni     = NiC;
7987     points = pointsC;
7988     if (values) *values = valuesC;
7989   }
7990   /* 5) Calculate indices */
7991   PetscCall(DMGetWorkArray(dm, Ni, MPIU_INT, &idx));
7992   PetscCall(PetscSectionGetChart(idxSection, &idxStart, &idxEnd));
7993   if (Nf) {
7994     PetscInt  idxOff;
7995     PetscBool useFieldOffsets;
7996 
7997     if (outOffsets) {
7998       for (f = 0; f <= Nf; f++) outOffsets[f] = offsets[f];
7999     }
8000     PetscCall(PetscSectionGetUseFieldOffsets(idxSection, &useFieldOffsets));
8001     if (useFieldOffsets) {
8002       for (p = 0; p < Ncl; ++p) {
8003         const PetscInt pnt = points[p * 2];
8004 
8005         PetscCall(DMPlexGetIndicesPointFieldsSplit_Internal(section, idxSection, pnt, offsets, perms, p, clperm, idx));
8006       }
8007     } else {
8008       for (p = 0; p < Ncl; ++p) {
8009         const PetscInt pnt = points[p * 2];
8010 
8011         if (pnt < idxStart || pnt >= idxEnd) continue;
8012         PetscCall(PetscSectionGetOffset(idxSection, pnt, &idxOff));
8013         /* Note that we pass a local section even though we're using global offsets.  This is because global sections do
8014          * not (at the time of this writing) have fields set. They probably should, in which case we would pass the
8015          * global section. */
8016         PetscCall(DMPlexGetIndicesPointFields_Internal(section, isLocal, pnt, idxOff < 0 ? -(idxOff + 1) : idxOff, offsets, PETSC_FALSE, perms, p, clperm, idx));
8017       }
8018     }
8019   } else {
8020     PetscInt off = 0, idxOff;
8021 
8022     for (p = 0; p < Ncl; ++p) {
8023       const PetscInt  pnt  = points[p * 2];
8024       const PetscInt *perm = perms[0] ? perms[0][p] : NULL;
8025 
8026       if (pnt < idxStart || pnt >= idxEnd) continue;
8027       PetscCall(PetscSectionGetOffset(idxSection, pnt, &idxOff));
8028       /* Note that we pass a local section even though we're using global offsets.  This is because global sections do
8029        * not (at the time of this writing) have fields set. They probably should, in which case we would pass the global section. */
8030       PetscCall(DMPlexGetIndicesPoint_Internal(section, isLocal, pnt, idxOff < 0 ? -(idxOff + 1) : idxOff, &off, PETSC_FALSE, perm, clperm, idx));
8031     }
8032   }
8033   /* 6) Cleanup */
8034   for (f = 0; f < PetscMax(1, Nf); ++f) {
8035     if (Nf) PetscCall(PetscSectionRestoreFieldPointSyms(section, f, Ncl, points, &perms[f], &flips[f]));
8036     else PetscCall(PetscSectionRestorePointSyms(section, Ncl, points, &perms[f], &flips[f]));
8037   }
8038   if (NclC) {
8039     PetscCall(DMRestoreWorkArray(dm, NclC * 2, MPIU_INT, &pointsC));
8040   } else {
8041     PetscCall(DMPlexRestoreCompressedClosure(dm, section, point, &Ncl, &points, &clSection, &clPoints, &clp));
8042   }
8043 
8044   if (indices) *indices = idx;
8045   PetscFunctionReturn(PETSC_SUCCESS);
8046 }
8047 
8048 /*@C
8049   DMPlexGetClosureIndices - Gets the global dof indices associated with the closure of the given point within the provided sections.
8050 
8051   Not collective
8052 
8053   Input Parameters:
8054 + dm         - The `DM`
8055 . section    - The `PetscSection` describing the points (a local section)
8056 . idxSection - The `PetscSection` from which to obtain indices (may be local or global)
8057 . point      - The point defining the closure
8058 - useClPerm  - Use the closure point permutation if available
8059 
8060   Output Parameters:
8061 + numIndices - The number of dof indices in the closure of point with the input sections
8062 . indices    - The dof indices
8063 . outOffsets - Array to write the field offsets into, or `NULL`
8064 - values     - The input values, which may be modified if sign flips are induced by the point symmetries, or `NULL`
8065 
8066   Level: advanced
8067 
8068   Notes:
8069   Call `DMPlexRestoreClosureIndices()` to free allocated memory
8070 
8071   If `idxSection` is global, any constrained dofs (see `DMAddBoundary()`, for example) will get negative indices.  The value
8072   of those indices is not significant.  If `idxSection` is local, the constrained dofs will yield the involution -(idx+1)
8073   of their index in a local vector.  A caller who does not wish to distinguish those points may recover the nonnegative
8074   indices via involution, -(-(idx+1)+1)==idx.  Local indices are provided when `idxSection` == section, otherwise global
8075   indices (with the above semantics) are implied.
8076 
8077 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexRestoreClosureIndices()`, `DMPlexVecGetClosure()`, `DMPlexMatSetClosure()`, `DMGetLocalSection()`,
8078           `PetscSection`, `DMGetGlobalSection()`
8079 @*/
8080 PetscErrorCode DMPlexGetClosureIndices(DM dm, PetscSection section, PetscSection idxSection, PetscInt point, PetscBool useClPerm, PetscInt *numIndices, PetscInt *indices[], PetscInt outOffsets[], PetscScalar *values[])
8081 {
8082   PetscInt numRows = -1, numCols = -1;
8083 
8084   PetscFunctionBeginHot;
8085   PetscCall(DMPlexGetClosureIndices_Internal(dm, section, idxSection, point, useClPerm, &numRows, &numCols, indices, outOffsets, values, PETSC_TRUE, PETSC_TRUE));
8086   PetscCheck(numRows == numCols, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Symmetric matrix transformation produces rectangular dimensions (%" PetscInt_FMT ", %" PetscInt_FMT ")", numRows, numCols);
8087   *numIndices = numRows;
8088   PetscFunctionReturn(PETSC_SUCCESS);
8089 }
8090 
8091 /*@C
8092   DMPlexRestoreClosureIndices - Restores the global dof indices associated with the closure of the given point within the provided sections.
8093 
8094   Not collective
8095 
8096   Input Parameters:
8097 + dm         - The `DM`
8098 . section    - The `PetscSection` describing the points (a local section)
8099 . idxSection - The `PetscSection` from which to obtain indices (may be local or global)
8100 . point      - The point defining the closure
8101 - useClPerm  - Use the closure point permutation if available
8102 
8103   Output Parameters:
8104 + numIndices - The number of dof indices in the closure of point with the input sections
8105 . indices    - The dof indices
8106 . outOffsets - Array to write the field offsets into, or `NULL`
8107 - values     - The input values, which may be modified if sign flips are induced by the point symmetries, or `NULL`
8108 
8109   Level: advanced
8110 
8111   Notes:
8112   If values were modified, the user is responsible for calling `DMRestoreWorkArray`(dm, 0, `MPIU_SCALAR`, &values).
8113 
8114   If idxSection is global, any constrained dofs (see `DMAddBoundary()`, for example) will get negative indices.  The value
8115   of those indices is not significant.  If idxSection is local, the constrained dofs will yield the involution -(idx+1)
8116   of their index in a local vector.  A caller who does not wish to distinguish those points may recover the nonnegative
8117   indices via involution, -(-(idx+1)+1)==idx.  Local indices are provided when idxSection == section, otherwise global
8118   indices (with the above semantics) are implied.
8119 
8120 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetClosureIndices()`, `DMPlexVecGetClosure()`, `DMPlexMatSetClosure()`, `DMGetLocalSection()`, `DMGetGlobalSection()`
8121 @*/
8122 PetscErrorCode DMPlexRestoreClosureIndices(DM dm, PetscSection section, PetscSection idxSection, PetscInt point, PetscBool useClPerm, PetscInt *numIndices, PetscInt *indices[], PetscInt outOffsets[], PetscScalar *values[])
8123 {
8124   PetscFunctionBegin;
8125   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8126   PetscAssertPointer(indices, 7);
8127   PetscCall(DMRestoreWorkArray(dm, 0, MPIU_INT, indices));
8128   PetscFunctionReturn(PETSC_SUCCESS);
8129 }
8130 
8131 PetscErrorCode DMPlexMatSetClosure_Internal(DM dm, PetscSection section, PetscSection globalSection, PetscBool useClPerm, Mat A, PetscInt point, const PetscScalar values[], InsertMode mode)
8132 {
8133   DM_Plex           *mesh = (DM_Plex *)dm->data;
8134   PetscInt          *indices;
8135   PetscInt           numIndices;
8136   const PetscScalar *valuesOrig = values;
8137   PetscErrorCode     ierr;
8138 
8139   PetscFunctionBegin;
8140   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8141   if (!section) PetscCall(DMGetLocalSection(dm, &section));
8142   PetscValidHeaderSpecific(section, PETSC_SECTION_CLASSID, 2);
8143   if (!globalSection) PetscCall(DMGetGlobalSection(dm, &globalSection));
8144   PetscValidHeaderSpecific(globalSection, PETSC_SECTION_CLASSID, 3);
8145   PetscValidHeaderSpecific(A, MAT_CLASSID, 5);
8146 
8147   PetscCall(DMPlexGetClosureIndices(dm, section, globalSection, point, useClPerm, &numIndices, &indices, NULL, (PetscScalar **)&values));
8148 
8149   if (mesh->printSetValues) PetscCall(DMPlexPrintMatSetValues(PETSC_VIEWER_STDOUT_SELF, A, point, numIndices, indices, 0, NULL, values));
8150   /* TODO: fix this code to not use error codes as handle-able exceptions! */
8151   ierr = MatSetValues(A, numIndices, indices, numIndices, indices, values, mode);
8152   if (ierr) {
8153     PetscMPIInt rank;
8154 
8155     PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank));
8156     PetscCall((*PetscErrorPrintf)("[%d]ERROR in DMPlexMatSetClosure\n", rank));
8157     PetscCall(DMPlexPrintMatSetValues(PETSC_VIEWER_STDERR_SELF, A, point, numIndices, indices, 0, NULL, values));
8158     PetscCall(DMPlexRestoreClosureIndices(dm, section, globalSection, point, PETSC_TRUE, &numIndices, &indices, NULL, (PetscScalar **)&values));
8159     if (values != valuesOrig) PetscCall(DMRestoreWorkArray(dm, 0, MPIU_SCALAR, &values));
8160     SETERRQ(PetscObjectComm((PetscObject)dm), ierr, "Not possible to set matrix values");
8161   }
8162   if (mesh->printFEM > 1) {
8163     PetscInt i;
8164     PetscCall(PetscPrintf(PETSC_COMM_SELF, "  Indices:"));
8165     for (i = 0; i < numIndices; ++i) PetscCall(PetscPrintf(PETSC_COMM_SELF, " %" PetscInt_FMT, indices[i]));
8166     PetscCall(PetscPrintf(PETSC_COMM_SELF, "\n"));
8167   }
8168 
8169   PetscCall(DMPlexRestoreClosureIndices(dm, section, globalSection, point, PETSC_TRUE, &numIndices, &indices, NULL, (PetscScalar **)&values));
8170   if (values != valuesOrig) PetscCall(DMRestoreWorkArray(dm, 0, MPIU_SCALAR, &values));
8171   PetscFunctionReturn(PETSC_SUCCESS);
8172 }
8173 
8174 /*@C
8175   DMPlexMatSetClosure - Set an array of the values on the closure of 'point'
8176 
8177   Not collective
8178 
8179   Input Parameters:
8180 + dm            - The `DM`
8181 . section       - The section describing the layout in `v`, or `NULL` to use the default section
8182 . globalSection - The section describing the layout in `v`, or `NULL` to use the default global section
8183 . A             - The matrix
8184 . point         - The point in the `DM`
8185 . values        - The array of values
8186 - mode          - The insert mode, where `INSERT_ALL_VALUES` and `ADD_ALL_VALUES` also overwrite boundary conditions
8187 
8188   Level: intermediate
8189 
8190 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexMatSetClosureGeneral()`, `DMPlexVecGetClosure()`, `DMPlexVecSetClosure()`
8191 @*/
8192 PetscErrorCode DMPlexMatSetClosure(DM dm, PetscSection section, PetscSection globalSection, Mat A, PetscInt point, const PetscScalar values[], InsertMode mode)
8193 {
8194   PetscFunctionBegin;
8195   PetscCall(DMPlexMatSetClosure_Internal(dm, section, globalSection, PETSC_TRUE, A, point, values, mode));
8196   PetscFunctionReturn(PETSC_SUCCESS);
8197 }
8198 
8199 /*@C
8200   DMPlexMatSetClosureGeneral - Set an array of the values on the closure of 'point' using a different row and column section
8201 
8202   Not collective
8203 
8204   Input Parameters:
8205 + dmRow            - The `DM` for the row fields
8206 . sectionRow       - The section describing the layout, or `NULL` to use the default section in `dmRow`
8207 . useRowPerm       - The flag to use the closure permutation of the `dmRow` if available
8208 . globalSectionRow - The section describing the layout, or `NULL` to use the default global section in `dmRow`
8209 . dmCol            - The `DM` for the column fields
8210 . sectionCol       - The section describing the layout, or `NULL` to use the default section in `dmCol`
8211 . useColPerm       - The flag to use the closure permutation of the `dmCol` if available
8212 . globalSectionCol - The section describing the layout, or `NULL` to use the default global section in `dmCol`
8213 . A                - The matrix
8214 . point            - The point in the `DM`
8215 . values           - The array of values
8216 - mode             - The insert mode, where `INSERT_ALL_VALUES` and `ADD_ALL_VALUES` also overwrite boundary conditions
8217 
8218   Level: intermediate
8219 
8220 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexMatSetClosure()`, `DMPlexVecGetClosure()`, `DMPlexVecSetClosure()`
8221 @*/
8222 PetscErrorCode DMPlexMatSetClosureGeneral(DM dmRow, PetscSection sectionRow, PetscSection globalSectionRow, PetscBool useRowPerm, DM dmCol, PetscSection sectionCol, PetscSection globalSectionCol, PetscBool useColPerm, Mat A, PetscInt point, const PetscScalar values[], InsertMode mode)
8223 {
8224   DM_Plex           *mesh = (DM_Plex *)dmRow->data;
8225   PetscInt          *indicesRow, *indicesCol;
8226   PetscInt           numIndicesRow = -1, numIndicesCol = -1;
8227   const PetscScalar *valuesV0 = values, *valuesV1, *valuesV2;
8228 
8229   PetscErrorCode ierr;
8230 
8231   PetscFunctionBegin;
8232   PetscValidHeaderSpecific(dmRow, DM_CLASSID, 1);
8233   if (!sectionRow) PetscCall(DMGetLocalSection(dmRow, &sectionRow));
8234   PetscValidHeaderSpecific(sectionRow, PETSC_SECTION_CLASSID, 2);
8235   if (!globalSectionRow) PetscCall(DMGetGlobalSection(dmRow, &globalSectionRow));
8236   PetscValidHeaderSpecific(globalSectionRow, PETSC_SECTION_CLASSID, 3);
8237   PetscValidHeaderSpecific(dmCol, DM_CLASSID, 5);
8238   if (!sectionCol) PetscCall(DMGetLocalSection(dmCol, &sectionCol));
8239   PetscValidHeaderSpecific(sectionCol, PETSC_SECTION_CLASSID, 6);
8240   if (!globalSectionCol) PetscCall(DMGetGlobalSection(dmCol, &globalSectionCol));
8241   PetscValidHeaderSpecific(globalSectionCol, PETSC_SECTION_CLASSID, 7);
8242   PetscValidHeaderSpecific(A, MAT_CLASSID, 9);
8243 
8244   PetscCall(DMPlexGetClosureIndicesSize_Internal(dmRow, sectionRow, point, &numIndicesRow));
8245   PetscCall(DMPlexGetClosureIndicesSize_Internal(dmCol, sectionCol, point, &numIndicesCol));
8246   valuesV1 = valuesV0;
8247   PetscCall(DMPlexGetClosureIndices_Internal(dmRow, sectionRow, globalSectionRow, point, useRowPerm, &numIndicesRow, &numIndicesCol, &indicesRow, NULL, (PetscScalar **)&valuesV1, PETSC_FALSE, PETSC_TRUE));
8248   valuesV2 = valuesV1;
8249   PetscCall(DMPlexGetClosureIndices_Internal(dmCol, sectionCol, globalSectionCol, point, useColPerm, &numIndicesRow, &numIndicesCol, &indicesCol, NULL, (PetscScalar **)&valuesV2, PETSC_TRUE, PETSC_FALSE));
8250 
8251   if (mesh->printSetValues) PetscCall(DMPlexPrintMatSetValues(PETSC_VIEWER_STDOUT_SELF, A, point, numIndicesRow, indicesRow, numIndicesCol, indicesCol, valuesV2));
8252   /* TODO: fix this code to not use error codes as handle-able exceptions! */
8253   ierr = MatSetValues(A, numIndicesRow, indicesRow, numIndicesCol, indicesCol, valuesV2, mode);
8254   if (ierr) {
8255     PetscMPIInt rank;
8256 
8257     PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank));
8258     PetscCall((*PetscErrorPrintf)("[%d]ERROR in DMPlexMatSetClosure\n", rank));
8259     PetscCall(DMPlexPrintMatSetValues(PETSC_VIEWER_STDERR_SELF, A, point, numIndicesRow, indicesRow, numIndicesCol, indicesCol, values));
8260     PetscCall(DMPlexRestoreClosureIndices(dmCol, sectionCol, globalSectionCol, point, PETSC_TRUE, &numIndicesCol, &indicesRow, NULL, (PetscScalar **)&valuesV2));
8261     PetscCall(DMPlexRestoreClosureIndices(dmRow, sectionRow, globalSectionRow, point, PETSC_TRUE, &numIndicesRow, &indicesRow, NULL, (PetscScalar **)&valuesV1));
8262     if (valuesV2 != valuesV1) PetscCall(DMRestoreWorkArray(dmCol, 0, MPIU_SCALAR, &valuesV2));
8263     if (valuesV1 != valuesV0) PetscCall(DMRestoreWorkArray(dmRow, 0, MPIU_SCALAR, &valuesV1));
8264   }
8265 
8266   PetscCall(DMPlexRestoreClosureIndices(dmCol, sectionCol, globalSectionCol, point, useColPerm, &numIndicesCol, &indicesCol, NULL, (PetscScalar **)&valuesV2));
8267   PetscCall(DMPlexRestoreClosureIndices(dmRow, sectionRow, globalSectionRow, point, useRowPerm, &numIndicesRow, &indicesRow, NULL, (PetscScalar **)&valuesV1));
8268   if (valuesV2 != valuesV1) PetscCall(DMRestoreWorkArray(dmCol, 0, MPIU_SCALAR, &valuesV2));
8269   if (valuesV1 != valuesV0) PetscCall(DMRestoreWorkArray(dmRow, 0, MPIU_SCALAR, &valuesV1));
8270   PetscFunctionReturn(PETSC_SUCCESS);
8271 }
8272 
8273 PetscErrorCode DMPlexMatSetClosureRefined(DM dmf, PetscSection fsection, PetscSection globalFSection, DM dmc, PetscSection csection, PetscSection globalCSection, Mat A, PetscInt point, const PetscScalar values[], InsertMode mode)
8274 {
8275   DM_Plex        *mesh    = (DM_Plex *)dmf->data;
8276   PetscInt       *fpoints = NULL, *ftotpoints = NULL;
8277   PetscInt       *cpoints = NULL;
8278   PetscInt       *findices, *cindices;
8279   const PetscInt *fclperm = NULL, *cclperm = NULL; /* Closure permutations cannot work here */
8280   PetscInt        foffsets[32], coffsets[32];
8281   DMPolytopeType  ct;
8282   PetscInt        numFields, numSubcells, maxFPoints, numFPoints, numCPoints, numFIndices, numCIndices, dof, off, globalOff, pStart, pEnd, p, q, r, s, f;
8283   PetscErrorCode  ierr;
8284 
8285   PetscFunctionBegin;
8286   PetscValidHeaderSpecific(dmf, DM_CLASSID, 1);
8287   PetscValidHeaderSpecific(dmc, DM_CLASSID, 4);
8288   if (!fsection) PetscCall(DMGetLocalSection(dmf, &fsection));
8289   PetscValidHeaderSpecific(fsection, PETSC_SECTION_CLASSID, 2);
8290   if (!csection) PetscCall(DMGetLocalSection(dmc, &csection));
8291   PetscValidHeaderSpecific(csection, PETSC_SECTION_CLASSID, 5);
8292   if (!globalFSection) PetscCall(DMGetGlobalSection(dmf, &globalFSection));
8293   PetscValidHeaderSpecific(globalFSection, PETSC_SECTION_CLASSID, 3);
8294   if (!globalCSection) PetscCall(DMGetGlobalSection(dmc, &globalCSection));
8295   PetscValidHeaderSpecific(globalCSection, PETSC_SECTION_CLASSID, 6);
8296   PetscValidHeaderSpecific(A, MAT_CLASSID, 7);
8297   PetscCall(PetscSectionGetNumFields(fsection, &numFields));
8298   PetscCheck(numFields <= 31, PetscObjectComm((PetscObject)dmf), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %" PetscInt_FMT " limited to 31", numFields);
8299   PetscCall(PetscArrayzero(foffsets, 32));
8300   PetscCall(PetscArrayzero(coffsets, 32));
8301   /* Column indices */
8302   PetscCall(DMPlexGetTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints));
8303   maxFPoints = numCPoints;
8304   /* Compress out points not in the section */
8305   /*   TODO: Squeeze out points with 0 dof as well */
8306   PetscCall(PetscSectionGetChart(csection, &pStart, &pEnd));
8307   for (p = 0, q = 0; p < numCPoints * 2; p += 2) {
8308     if ((cpoints[p] >= pStart) && (cpoints[p] < pEnd)) {
8309       cpoints[q * 2]     = cpoints[p];
8310       cpoints[q * 2 + 1] = cpoints[p + 1];
8311       ++q;
8312     }
8313   }
8314   numCPoints = q;
8315   for (p = 0, numCIndices = 0; p < numCPoints * 2; p += 2) {
8316     PetscInt fdof;
8317 
8318     PetscCall(PetscSectionGetDof(csection, cpoints[p], &dof));
8319     if (!dof) continue;
8320     for (f = 0; f < numFields; ++f) {
8321       PetscCall(PetscSectionGetFieldDof(csection, cpoints[p], f, &fdof));
8322       coffsets[f + 1] += fdof;
8323     }
8324     numCIndices += dof;
8325   }
8326   for (f = 1; f < numFields; ++f) coffsets[f + 1] += coffsets[f];
8327   /* Row indices */
8328   PetscCall(DMPlexGetCellType(dmc, point, &ct));
8329   {
8330     DMPlexTransform tr;
8331     DMPolytopeType *rct;
8332     PetscInt       *rsize, *rcone, *rornt, Nt;
8333 
8334     PetscCall(DMPlexTransformCreate(PETSC_COMM_SELF, &tr));
8335     PetscCall(DMPlexTransformSetType(tr, DMPLEXREFINEREGULAR));
8336     PetscCall(DMPlexTransformCellTransform(tr, ct, point, NULL, &Nt, &rct, &rsize, &rcone, &rornt));
8337     numSubcells = rsize[Nt - 1];
8338     PetscCall(DMPlexTransformDestroy(&tr));
8339   }
8340   PetscCall(DMGetWorkArray(dmf, maxFPoints * 2 * numSubcells, MPIU_INT, &ftotpoints));
8341   for (r = 0, q = 0; r < numSubcells; ++r) {
8342     /* TODO Map from coarse to fine cells */
8343     PetscCall(DMPlexGetTransitiveClosure(dmf, point * numSubcells + r, PETSC_TRUE, &numFPoints, &fpoints));
8344     /* Compress out points not in the section */
8345     PetscCall(PetscSectionGetChart(fsection, &pStart, &pEnd));
8346     for (p = 0; p < numFPoints * 2; p += 2) {
8347       if ((fpoints[p] >= pStart) && (fpoints[p] < pEnd)) {
8348         PetscCall(PetscSectionGetDof(fsection, fpoints[p], &dof));
8349         if (!dof) continue;
8350         for (s = 0; s < q; ++s)
8351           if (fpoints[p] == ftotpoints[s * 2]) break;
8352         if (s < q) continue;
8353         ftotpoints[q * 2]     = fpoints[p];
8354         ftotpoints[q * 2 + 1] = fpoints[p + 1];
8355         ++q;
8356       }
8357     }
8358     PetscCall(DMPlexRestoreTransitiveClosure(dmf, point, PETSC_TRUE, &numFPoints, &fpoints));
8359   }
8360   numFPoints = q;
8361   for (p = 0, numFIndices = 0; p < numFPoints * 2; p += 2) {
8362     PetscInt fdof;
8363 
8364     PetscCall(PetscSectionGetDof(fsection, ftotpoints[p], &dof));
8365     if (!dof) continue;
8366     for (f = 0; f < numFields; ++f) {
8367       PetscCall(PetscSectionGetFieldDof(fsection, ftotpoints[p], f, &fdof));
8368       foffsets[f + 1] += fdof;
8369     }
8370     numFIndices += dof;
8371   }
8372   for (f = 1; f < numFields; ++f) foffsets[f + 1] += foffsets[f];
8373 
8374   PetscCheck(!numFields || foffsets[numFields] == numFIndices, PetscObjectComm((PetscObject)dmf), PETSC_ERR_PLIB, "Invalid size for closure %" PetscInt_FMT " should be %" PetscInt_FMT, foffsets[numFields], numFIndices);
8375   PetscCheck(!numFields || coffsets[numFields] == numCIndices, PetscObjectComm((PetscObject)dmc), PETSC_ERR_PLIB, "Invalid size for closure %" PetscInt_FMT " should be %" PetscInt_FMT, coffsets[numFields], numCIndices);
8376   PetscCall(DMGetWorkArray(dmf, numFIndices, MPIU_INT, &findices));
8377   PetscCall(DMGetWorkArray(dmc, numCIndices, MPIU_INT, &cindices));
8378   if (numFields) {
8379     const PetscInt **permsF[32] = {NULL};
8380     const PetscInt **permsC[32] = {NULL};
8381 
8382     for (f = 0; f < numFields; f++) {
8383       PetscCall(PetscSectionGetFieldPointSyms(fsection, f, numFPoints, ftotpoints, &permsF[f], NULL));
8384       PetscCall(PetscSectionGetFieldPointSyms(csection, f, numCPoints, cpoints, &permsC[f], NULL));
8385     }
8386     for (p = 0; p < numFPoints; p++) {
8387       PetscCall(PetscSectionGetOffset(globalFSection, ftotpoints[2 * p], &globalOff));
8388       PetscCall(DMPlexGetIndicesPointFields_Internal(fsection, PETSC_FALSE, ftotpoints[2 * p], globalOff < 0 ? -(globalOff + 1) : globalOff, foffsets, PETSC_FALSE, permsF, p, fclperm, findices));
8389     }
8390     for (p = 0; p < numCPoints; p++) {
8391       PetscCall(PetscSectionGetOffset(globalCSection, cpoints[2 * p], &globalOff));
8392       PetscCall(DMPlexGetIndicesPointFields_Internal(csection, PETSC_FALSE, cpoints[2 * p], globalOff < 0 ? -(globalOff + 1) : globalOff, coffsets, PETSC_FALSE, permsC, p, cclperm, cindices));
8393     }
8394     for (f = 0; f < numFields; f++) {
8395       PetscCall(PetscSectionRestoreFieldPointSyms(fsection, f, numFPoints, ftotpoints, &permsF[f], NULL));
8396       PetscCall(PetscSectionRestoreFieldPointSyms(csection, f, numCPoints, cpoints, &permsC[f], NULL));
8397     }
8398   } else {
8399     const PetscInt **permsF = NULL;
8400     const PetscInt **permsC = NULL;
8401 
8402     PetscCall(PetscSectionGetPointSyms(fsection, numFPoints, ftotpoints, &permsF, NULL));
8403     PetscCall(PetscSectionGetPointSyms(csection, numCPoints, cpoints, &permsC, NULL));
8404     for (p = 0, off = 0; p < numFPoints; p++) {
8405       const PetscInt *perm = permsF ? permsF[p] : NULL;
8406 
8407       PetscCall(PetscSectionGetOffset(globalFSection, ftotpoints[2 * p], &globalOff));
8408       PetscCall(DMPlexGetIndicesPoint_Internal(fsection, PETSC_FALSE, ftotpoints[2 * p], globalOff < 0 ? -(globalOff + 1) : globalOff, &off, PETSC_FALSE, perm, fclperm, findices));
8409     }
8410     for (p = 0, off = 0; p < numCPoints; p++) {
8411       const PetscInt *perm = permsC ? permsC[p] : NULL;
8412 
8413       PetscCall(PetscSectionGetOffset(globalCSection, cpoints[2 * p], &globalOff));
8414       PetscCall(DMPlexGetIndicesPoint_Internal(csection, PETSC_FALSE, cpoints[2 * p], globalOff < 0 ? -(globalOff + 1) : globalOff, &off, PETSC_FALSE, perm, cclperm, cindices));
8415     }
8416     PetscCall(PetscSectionRestorePointSyms(fsection, numFPoints, ftotpoints, &permsF, NULL));
8417     PetscCall(PetscSectionRestorePointSyms(csection, numCPoints, cpoints, &permsC, NULL));
8418   }
8419   if (mesh->printSetValues) PetscCall(DMPlexPrintMatSetValues(PETSC_VIEWER_STDOUT_SELF, A, point, numFIndices, findices, numCIndices, cindices, values));
8420   /* TODO: flips */
8421   /* TODO: fix this code to not use error codes as handle-able exceptions! */
8422   ierr = MatSetValues(A, numFIndices, findices, numCIndices, cindices, values, mode);
8423   if (ierr) {
8424     PetscMPIInt rank;
8425 
8426     PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)A), &rank));
8427     PetscCall((*PetscErrorPrintf)("[%d]ERROR in DMPlexMatSetClosure\n", rank));
8428     PetscCall(DMPlexPrintMatSetValues(PETSC_VIEWER_STDERR_SELF, A, point, numFIndices, findices, numCIndices, cindices, values));
8429     PetscCall(DMRestoreWorkArray(dmf, numFIndices, MPIU_INT, &findices));
8430     PetscCall(DMRestoreWorkArray(dmc, numCIndices, MPIU_INT, &cindices));
8431   }
8432   PetscCall(DMRestoreWorkArray(dmf, numCPoints * 2 * 4, MPIU_INT, &ftotpoints));
8433   PetscCall(DMPlexRestoreTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints));
8434   PetscCall(DMRestoreWorkArray(dmf, numFIndices, MPIU_INT, &findices));
8435   PetscCall(DMRestoreWorkArray(dmc, numCIndices, MPIU_INT, &cindices));
8436   PetscFunctionReturn(PETSC_SUCCESS);
8437 }
8438 
8439 PetscErrorCode DMPlexMatGetClosureIndicesRefined(DM dmf, PetscSection fsection, PetscSection globalFSection, DM dmc, PetscSection csection, PetscSection globalCSection, PetscInt point, PetscInt cindices[], PetscInt findices[])
8440 {
8441   PetscInt       *fpoints = NULL, *ftotpoints = NULL;
8442   PetscInt       *cpoints      = NULL;
8443   PetscInt        foffsets[32] = {0}, coffsets[32] = {0};
8444   const PetscInt *fclperm = NULL, *cclperm = NULL; /* Closure permutations cannot work here */
8445   DMPolytopeType  ct;
8446   PetscInt        numFields, numSubcells, maxFPoints, numFPoints, numCPoints, numFIndices, numCIndices, dof, off, globalOff, pStart, pEnd, p, q, r, s, f;
8447 
8448   PetscFunctionBegin;
8449   PetscValidHeaderSpecific(dmf, DM_CLASSID, 1);
8450   PetscValidHeaderSpecific(dmc, DM_CLASSID, 4);
8451   if (!fsection) PetscCall(DMGetLocalSection(dmf, &fsection));
8452   PetscValidHeaderSpecific(fsection, PETSC_SECTION_CLASSID, 2);
8453   if (!csection) PetscCall(DMGetLocalSection(dmc, &csection));
8454   PetscValidHeaderSpecific(csection, PETSC_SECTION_CLASSID, 5);
8455   if (!globalFSection) PetscCall(DMGetGlobalSection(dmf, &globalFSection));
8456   PetscValidHeaderSpecific(globalFSection, PETSC_SECTION_CLASSID, 3);
8457   if (!globalCSection) PetscCall(DMGetGlobalSection(dmc, &globalCSection));
8458   PetscValidHeaderSpecific(globalCSection, PETSC_SECTION_CLASSID, 6);
8459   PetscCall(PetscSectionGetNumFields(fsection, &numFields));
8460   PetscCheck(numFields <= 31, PetscObjectComm((PetscObject)dmf), PETSC_ERR_ARG_OUTOFRANGE, "Number of fields %" PetscInt_FMT " limited to 31", numFields);
8461   /* Column indices */
8462   PetscCall(DMPlexGetTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints));
8463   maxFPoints = numCPoints;
8464   /* Compress out points not in the section */
8465   /*   TODO: Squeeze out points with 0 dof as well */
8466   PetscCall(PetscSectionGetChart(csection, &pStart, &pEnd));
8467   for (p = 0, q = 0; p < numCPoints * 2; p += 2) {
8468     if ((cpoints[p] >= pStart) && (cpoints[p] < pEnd)) {
8469       cpoints[q * 2]     = cpoints[p];
8470       cpoints[q * 2 + 1] = cpoints[p + 1];
8471       ++q;
8472     }
8473   }
8474   numCPoints = q;
8475   for (p = 0, numCIndices = 0; p < numCPoints * 2; p += 2) {
8476     PetscInt fdof;
8477 
8478     PetscCall(PetscSectionGetDof(csection, cpoints[p], &dof));
8479     if (!dof) continue;
8480     for (f = 0; f < numFields; ++f) {
8481       PetscCall(PetscSectionGetFieldDof(csection, cpoints[p], f, &fdof));
8482       coffsets[f + 1] += fdof;
8483     }
8484     numCIndices += dof;
8485   }
8486   for (f = 1; f < numFields; ++f) coffsets[f + 1] += coffsets[f];
8487   /* Row indices */
8488   PetscCall(DMPlexGetCellType(dmc, point, &ct));
8489   {
8490     DMPlexTransform tr;
8491     DMPolytopeType *rct;
8492     PetscInt       *rsize, *rcone, *rornt, Nt;
8493 
8494     PetscCall(DMPlexTransformCreate(PETSC_COMM_SELF, &tr));
8495     PetscCall(DMPlexTransformSetType(tr, DMPLEXREFINEREGULAR));
8496     PetscCall(DMPlexTransformCellTransform(tr, ct, point, NULL, &Nt, &rct, &rsize, &rcone, &rornt));
8497     numSubcells = rsize[Nt - 1];
8498     PetscCall(DMPlexTransformDestroy(&tr));
8499   }
8500   PetscCall(DMGetWorkArray(dmf, maxFPoints * 2 * numSubcells, MPIU_INT, &ftotpoints));
8501   for (r = 0, q = 0; r < numSubcells; ++r) {
8502     /* TODO Map from coarse to fine cells */
8503     PetscCall(DMPlexGetTransitiveClosure(dmf, point * numSubcells + r, PETSC_TRUE, &numFPoints, &fpoints));
8504     /* Compress out points not in the section */
8505     PetscCall(PetscSectionGetChart(fsection, &pStart, &pEnd));
8506     for (p = 0; p < numFPoints * 2; p += 2) {
8507       if ((fpoints[p] >= pStart) && (fpoints[p] < pEnd)) {
8508         PetscCall(PetscSectionGetDof(fsection, fpoints[p], &dof));
8509         if (!dof) continue;
8510         for (s = 0; s < q; ++s)
8511           if (fpoints[p] == ftotpoints[s * 2]) break;
8512         if (s < q) continue;
8513         ftotpoints[q * 2]     = fpoints[p];
8514         ftotpoints[q * 2 + 1] = fpoints[p + 1];
8515         ++q;
8516       }
8517     }
8518     PetscCall(DMPlexRestoreTransitiveClosure(dmf, point, PETSC_TRUE, &numFPoints, &fpoints));
8519   }
8520   numFPoints = q;
8521   for (p = 0, numFIndices = 0; p < numFPoints * 2; p += 2) {
8522     PetscInt fdof;
8523 
8524     PetscCall(PetscSectionGetDof(fsection, ftotpoints[p], &dof));
8525     if (!dof) continue;
8526     for (f = 0; f < numFields; ++f) {
8527       PetscCall(PetscSectionGetFieldDof(fsection, ftotpoints[p], f, &fdof));
8528       foffsets[f + 1] += fdof;
8529     }
8530     numFIndices += dof;
8531   }
8532   for (f = 1; f < numFields; ++f) foffsets[f + 1] += foffsets[f];
8533 
8534   PetscCheck(!numFields || foffsets[numFields] == numFIndices, PetscObjectComm((PetscObject)dmf), PETSC_ERR_PLIB, "Invalid size for closure %" PetscInt_FMT " should be %" PetscInt_FMT, foffsets[numFields], numFIndices);
8535   PetscCheck(!numFields || coffsets[numFields] == numCIndices, PetscObjectComm((PetscObject)dmc), PETSC_ERR_PLIB, "Invalid size for closure %" PetscInt_FMT " should be %" PetscInt_FMT, coffsets[numFields], numCIndices);
8536   if (numFields) {
8537     const PetscInt **permsF[32] = {NULL};
8538     const PetscInt **permsC[32] = {NULL};
8539 
8540     for (f = 0; f < numFields; f++) {
8541       PetscCall(PetscSectionGetFieldPointSyms(fsection, f, numFPoints, ftotpoints, &permsF[f], NULL));
8542       PetscCall(PetscSectionGetFieldPointSyms(csection, f, numCPoints, cpoints, &permsC[f], NULL));
8543     }
8544     for (p = 0; p < numFPoints; p++) {
8545       PetscCall(PetscSectionGetOffset(globalFSection, ftotpoints[2 * p], &globalOff));
8546       PetscCall(DMPlexGetIndicesPointFields_Internal(fsection, PETSC_FALSE, ftotpoints[2 * p], globalOff < 0 ? -(globalOff + 1) : globalOff, foffsets, PETSC_FALSE, permsF, p, fclperm, findices));
8547     }
8548     for (p = 0; p < numCPoints; p++) {
8549       PetscCall(PetscSectionGetOffset(globalCSection, cpoints[2 * p], &globalOff));
8550       PetscCall(DMPlexGetIndicesPointFields_Internal(csection, PETSC_FALSE, cpoints[2 * p], globalOff < 0 ? -(globalOff + 1) : globalOff, coffsets, PETSC_FALSE, permsC, p, cclperm, cindices));
8551     }
8552     for (f = 0; f < numFields; f++) {
8553       PetscCall(PetscSectionRestoreFieldPointSyms(fsection, f, numFPoints, ftotpoints, &permsF[f], NULL));
8554       PetscCall(PetscSectionRestoreFieldPointSyms(csection, f, numCPoints, cpoints, &permsC[f], NULL));
8555     }
8556   } else {
8557     const PetscInt **permsF = NULL;
8558     const PetscInt **permsC = NULL;
8559 
8560     PetscCall(PetscSectionGetPointSyms(fsection, numFPoints, ftotpoints, &permsF, NULL));
8561     PetscCall(PetscSectionGetPointSyms(csection, numCPoints, cpoints, &permsC, NULL));
8562     for (p = 0, off = 0; p < numFPoints; p++) {
8563       const PetscInt *perm = permsF ? permsF[p] : NULL;
8564 
8565       PetscCall(PetscSectionGetOffset(globalFSection, ftotpoints[2 * p], &globalOff));
8566       PetscCall(DMPlexGetIndicesPoint_Internal(fsection, PETSC_FALSE, ftotpoints[2 * p], globalOff < 0 ? -(globalOff + 1) : globalOff, &off, PETSC_FALSE, perm, fclperm, findices));
8567     }
8568     for (p = 0, off = 0; p < numCPoints; p++) {
8569       const PetscInt *perm = permsC ? permsC[p] : NULL;
8570 
8571       PetscCall(PetscSectionGetOffset(globalCSection, cpoints[2 * p], &globalOff));
8572       PetscCall(DMPlexGetIndicesPoint_Internal(csection, PETSC_FALSE, cpoints[2 * p], globalOff < 0 ? -(globalOff + 1) : globalOff, &off, PETSC_FALSE, perm, cclperm, cindices));
8573     }
8574     PetscCall(PetscSectionRestorePointSyms(fsection, numFPoints, ftotpoints, &permsF, NULL));
8575     PetscCall(PetscSectionRestorePointSyms(csection, numCPoints, cpoints, &permsC, NULL));
8576   }
8577   PetscCall(DMRestoreWorkArray(dmf, numCPoints * 2 * 4, MPIU_INT, &ftotpoints));
8578   PetscCall(DMPlexRestoreTransitiveClosure(dmc, point, PETSC_TRUE, &numCPoints, &cpoints));
8579   PetscFunctionReturn(PETSC_SUCCESS);
8580 }
8581 
8582 /*@
8583   DMPlexGetVTKCellHeight - Returns the height in the DAG used to determine which points are cells (normally 0)
8584 
8585   Input Parameter:
8586 . dm - The `DMPLEX` object
8587 
8588   Output Parameter:
8589 . cellHeight - The height of a cell
8590 
8591   Level: developer
8592 
8593 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexSetVTKCellHeight()`
8594 @*/
8595 PetscErrorCode DMPlexGetVTKCellHeight(DM dm, PetscInt *cellHeight)
8596 {
8597   DM_Plex *mesh = (DM_Plex *)dm->data;
8598 
8599   PetscFunctionBegin;
8600   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8601   PetscAssertPointer(cellHeight, 2);
8602   *cellHeight = mesh->vtkCellHeight;
8603   PetscFunctionReturn(PETSC_SUCCESS);
8604 }
8605 
8606 /*@
8607   DMPlexSetVTKCellHeight - Sets the height in the DAG used to determine which points are cells (normally 0)
8608 
8609   Input Parameters:
8610 + dm         - The `DMPLEX` object
8611 - cellHeight - The height of a cell
8612 
8613   Level: developer
8614 
8615 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetVTKCellHeight()`
8616 @*/
8617 PetscErrorCode DMPlexSetVTKCellHeight(DM dm, PetscInt cellHeight)
8618 {
8619   DM_Plex *mesh = (DM_Plex *)dm->data;
8620 
8621   PetscFunctionBegin;
8622   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8623   mesh->vtkCellHeight = cellHeight;
8624   PetscFunctionReturn(PETSC_SUCCESS);
8625 }
8626 
8627 /*@
8628   DMPlexGetCellTypeStratum - Get the range of cells of a given celltype
8629 
8630   Input Parameters:
8631 + dm - The `DMPLEX` object
8632 - ct - The `DMPolytopeType` of the cell
8633 
8634   Output Parameters:
8635 + start - The first cell of this type, or `NULL`
8636 - end   - The upper bound on this celltype, or `NULL`
8637 
8638   Level: advanced
8639 
8640 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexConstructGhostCells()`, `DMPlexGetDepthStratum()`, `DMPlexGetHeightStratum()`
8641 @*/
8642 PetscErrorCode DMPlexGetCellTypeStratum(DM dm, DMPolytopeType ct, PetscInt *start, PetscInt *end)
8643 {
8644   DM_Plex *mesh = (DM_Plex *)dm->data;
8645   DMLabel  label;
8646   PetscInt pStart, pEnd;
8647 
8648   PetscFunctionBegin;
8649   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8650   if (start) {
8651     PetscAssertPointer(start, 3);
8652     *start = 0;
8653   }
8654   if (end) {
8655     PetscAssertPointer(end, 4);
8656     *end = 0;
8657   }
8658   PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
8659   if (pStart == pEnd) PetscFunctionReturn(PETSC_SUCCESS);
8660   if (mesh->tr) {
8661     PetscCall(DMPlexTransformGetCellTypeStratum(mesh->tr, ct, start, end));
8662   } else {
8663     PetscCall(DMPlexGetCellTypeLabel(dm, &label));
8664     PetscCheck(label, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "No label named celltype was found");
8665     PetscCall(DMLabelGetStratumBounds(label, ct, start, end));
8666   }
8667   PetscFunctionReturn(PETSC_SUCCESS);
8668 }
8669 
8670 /*@
8671   DMPlexGetDepthStratumGlobalSize - Get the global size for a given depth stratum
8672 
8673   Input Parameters:
8674 + dm    - The `DMPLEX` object
8675 - depth - The depth for the given point stratum
8676 
8677   Output Parameter:
8678 . gsize - The global number of points in the stratum
8679 
8680   Level: advanced
8681 
8682 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetCellNumbering()`, `DMPlexGetVertexNumbering()`, `DMPlexGetDepthStratum()`, `DMPlexGetHeightStratum()`
8683 @*/
8684 PetscErrorCode DMPlexGetDepthStratumGlobalSize(DM dm, PetscInt depth, PetscInt *gsize)
8685 {
8686   PetscSF         sf;
8687   const PetscInt *leaves;
8688   PetscInt        Nl, loc, start, end, lsize = 0;
8689 
8690   PetscFunctionBegin;
8691   PetscCall(DMGetPointSF(dm, &sf));
8692   PetscCall(PetscSFGetGraph(sf, NULL, &Nl, &leaves, NULL));
8693   PetscCall(DMPlexGetDepthStratum(dm, depth, &start, &end));
8694   for (PetscInt p = start; p < end; ++p) {
8695     PetscCall(PetscFindInt(p, Nl, leaves, &loc));
8696     if (loc < 0) ++lsize;
8697   }
8698   PetscCallMPI(MPI_Allreduce(&lsize, gsize, 1, MPIU_INT, MPI_SUM, PetscObjectComm((PetscObject)dm)));
8699   PetscFunctionReturn(PETSC_SUCCESS);
8700 }
8701 
8702 PetscErrorCode DMPlexCreateNumbering_Plex(DM dm, PetscInt pStart, PetscInt pEnd, PetscInt shift, PetscInt *globalSize, PetscSF sf, IS *numbering)
8703 {
8704   PetscSection section, globalSection;
8705   PetscInt    *numbers, p;
8706 
8707   PetscFunctionBegin;
8708   if (PetscDefined(USE_DEBUG)) PetscCall(DMPlexCheckPointSF(dm, sf, PETSC_TRUE));
8709   PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dm), &section));
8710   PetscCall(PetscSectionSetChart(section, pStart, pEnd));
8711   for (p = pStart; p < pEnd; ++p) PetscCall(PetscSectionSetDof(section, p, 1));
8712   PetscCall(PetscSectionSetUp(section));
8713   PetscCall(PetscSectionCreateGlobalSection(section, sf, PETSC_TRUE, PETSC_FALSE, PETSC_FALSE, &globalSection));
8714   PetscCall(PetscMalloc1(pEnd - pStart, &numbers));
8715   for (p = pStart; p < pEnd; ++p) {
8716     PetscCall(PetscSectionGetOffset(globalSection, p, &numbers[p - pStart]));
8717     if (numbers[p - pStart] < 0) numbers[p - pStart] -= shift;
8718     else numbers[p - pStart] += shift;
8719   }
8720   PetscCall(ISCreateGeneral(PetscObjectComm((PetscObject)dm), pEnd - pStart, numbers, PETSC_OWN_POINTER, numbering));
8721   if (globalSize) {
8722     PetscLayout layout;
8723     PetscCall(PetscSectionGetPointLayout(PetscObjectComm((PetscObject)dm), globalSection, &layout));
8724     PetscCall(PetscLayoutGetSize(layout, globalSize));
8725     PetscCall(PetscLayoutDestroy(&layout));
8726   }
8727   PetscCall(PetscSectionDestroy(&section));
8728   PetscCall(PetscSectionDestroy(&globalSection));
8729   PetscFunctionReturn(PETSC_SUCCESS);
8730 }
8731 
8732 /*@
8733   DMPlexCreateCellNumbering - Get a global cell numbering for all cells on this process
8734 
8735   Input Parameters:
8736 + dm         - The `DMPLEX` object
8737 - includeAll - Whether to include all cells, or just the simplex and box cells
8738 
8739   Output Parameter:
8740 . globalCellNumbers - Global cell numbers for all cells on this process
8741 
8742   Level: developer
8743 
8744 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetCellNumbering()`, `DMPlexGetVertexNumbering()`
8745 @*/
8746 PetscErrorCode DMPlexCreateCellNumbering(DM dm, PetscBool includeAll, IS *globalCellNumbers)
8747 {
8748   PetscInt cellHeight, cStart, cEnd;
8749 
8750   PetscFunctionBegin;
8751   PetscCall(DMPlexGetVTKCellHeight(dm, &cellHeight));
8752   if (includeAll) PetscCall(DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd));
8753   else PetscCall(DMPlexGetSimplexOrBoxCells(dm, cellHeight, &cStart, &cEnd));
8754   PetscCall(DMPlexCreateNumbering_Plex(dm, cStart, cEnd, 0, NULL, dm->sf, globalCellNumbers));
8755   PetscFunctionReturn(PETSC_SUCCESS);
8756 }
8757 
8758 /*@
8759   DMPlexGetCellNumbering - Get a global cell numbering for all cells on this process
8760 
8761   Input Parameter:
8762 . dm - The `DMPLEX` object
8763 
8764   Output Parameter:
8765 . globalCellNumbers - Global cell numbers for all cells on this process
8766 
8767   Level: developer
8768 
8769 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateCellNumbering()`, `DMPlexGetVertexNumbering()`
8770 @*/
8771 PetscErrorCode DMPlexGetCellNumbering(DM dm, IS *globalCellNumbers)
8772 {
8773   DM_Plex *mesh = (DM_Plex *)dm->data;
8774 
8775   PetscFunctionBegin;
8776   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8777   if (!mesh->globalCellNumbers) PetscCall(DMPlexCreateCellNumbering(dm, PETSC_FALSE, &mesh->globalCellNumbers));
8778   *globalCellNumbers = mesh->globalCellNumbers;
8779   PetscFunctionReturn(PETSC_SUCCESS);
8780 }
8781 
8782 PetscErrorCode DMPlexCreateVertexNumbering_Internal(DM dm, PetscBool includeHybrid, IS *globalVertexNumbers)
8783 {
8784   PetscInt vStart, vEnd;
8785 
8786   PetscFunctionBegin;
8787   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8788   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
8789   PetscCall(DMPlexCreateNumbering_Plex(dm, vStart, vEnd, 0, NULL, dm->sf, globalVertexNumbers));
8790   PetscFunctionReturn(PETSC_SUCCESS);
8791 }
8792 
8793 /*@
8794   DMPlexGetVertexNumbering - Get a global vertex numbering for all vertices on this process
8795 
8796   Input Parameter:
8797 . dm - The `DMPLEX` object
8798 
8799   Output Parameter:
8800 . globalVertexNumbers - Global vertex numbers for all vertices on this process
8801 
8802   Level: developer
8803 
8804 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetCellNumbering()`
8805 @*/
8806 PetscErrorCode DMPlexGetVertexNumbering(DM dm, IS *globalVertexNumbers)
8807 {
8808   DM_Plex *mesh = (DM_Plex *)dm->data;
8809 
8810   PetscFunctionBegin;
8811   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8812   if (!mesh->globalVertexNumbers) PetscCall(DMPlexCreateVertexNumbering_Internal(dm, PETSC_FALSE, &mesh->globalVertexNumbers));
8813   *globalVertexNumbers = mesh->globalVertexNumbers;
8814   PetscFunctionReturn(PETSC_SUCCESS);
8815 }
8816 
8817 /*@
8818   DMPlexCreatePointNumbering - Create a global numbering for all points.
8819 
8820   Collective
8821 
8822   Input Parameter:
8823 . dm - The `DMPLEX` object
8824 
8825   Output Parameter:
8826 . globalPointNumbers - Global numbers for all points on this process
8827 
8828   Level: developer
8829 
8830   Notes:
8831   The point numbering `IS` is parallel, with local portion indexed by local points (see `DMGetLocalSection()`). The global
8832   points are taken as stratified, with each MPI rank owning a contiguous subset of each stratum. In the IS, owned points
8833   will have their non-negative value while points owned by different ranks will be involuted -(idx+1). As an example,
8834   consider a parallel mesh in which the first two elements and first two vertices are owned by rank 0.
8835 
8836   The partitioned mesh is
8837   ```
8838   (2)--0--(3)--1--(4)    (1)--0--(2)
8839   ```
8840   and its global numbering is
8841   ```
8842   (3)--0--(4)--1--(5)--2--(6)
8843   ```
8844   Then the global numbering is provided as
8845   ```
8846   [0] Number of indices in set 5
8847   [0] 0 0
8848   [0] 1 1
8849   [0] 2 3
8850   [0] 3 4
8851   [0] 4 -6
8852   [1] Number of indices in set 3
8853   [1] 0 2
8854   [1] 1 5
8855   [1] 2 6
8856   ```
8857 
8858 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetCellNumbering()`
8859 @*/
8860 PetscErrorCode DMPlexCreatePointNumbering(DM dm, IS *globalPointNumbers)
8861 {
8862   IS        nums[4];
8863   PetscInt  depths[4], gdepths[4], starts[4];
8864   PetscInt  depth, d, shift = 0;
8865   PetscBool empty = PETSC_FALSE;
8866 
8867   PetscFunctionBegin;
8868   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8869   PetscCall(DMPlexGetDepth(dm, &depth));
8870   // For unstratified meshes use dim instead of depth
8871   if (depth < 0) PetscCall(DMGetDimension(dm, &depth));
8872   // If any stratum is empty, we must mark all empty
8873   for (d = 0; d <= depth; ++d) {
8874     PetscInt end;
8875 
8876     depths[d] = depth - d;
8877     PetscCall(DMPlexGetDepthStratum(dm, depths[d], &starts[d], &end));
8878     if (!(starts[d] - end)) empty = PETSC_TRUE;
8879   }
8880   if (empty)
8881     for (d = 0; d <= depth; ++d) {
8882       depths[d] = -1;
8883       starts[d] = -1;
8884     }
8885   else PetscCall(PetscSortIntWithArray(depth + 1, starts, depths));
8886   PetscCallMPI(MPIU_Allreduce(depths, gdepths, (PetscMPIInt)(depth + 1), MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject)dm)));
8887   for (d = 0; d <= depth; ++d) PetscCheck(starts[d] < 0 || depths[d] == gdepths[d], PETSC_COMM_SELF, PETSC_ERR_PLIB, "Expected depth %" PetscInt_FMT ", found %" PetscInt_FMT, depths[d], gdepths[d]);
8888   // Note here that 'shift' is collective, so that the numbering is stratified by depth
8889   for (d = 0; d <= depth; ++d) {
8890     PetscInt pStart, pEnd, gsize;
8891 
8892     PetscCall(DMPlexGetDepthStratum(dm, gdepths[d], &pStart, &pEnd));
8893     PetscCall(DMPlexCreateNumbering_Plex(dm, pStart, pEnd, shift, &gsize, dm->sf, &nums[d]));
8894     shift += gsize;
8895   }
8896   PetscCall(ISConcatenate(PETSC_COMM_SELF, depth + 1, nums, globalPointNumbers));
8897   for (d = 0; d <= depth; ++d) PetscCall(ISDestroy(&nums[d]));
8898   PetscFunctionReturn(PETSC_SUCCESS);
8899 }
8900 
8901 /*@
8902   DMPlexCreateEdgeNumbering - Create a global numbering for edges.
8903 
8904   Collective
8905 
8906   Input Parameter:
8907 . dm - The `DMPLEX` object
8908 
8909   Output Parameter:
8910 . globalEdgeNumbers - Global numbers for all edges on this process
8911 
8912   Level: developer
8913 
8914   Notes:
8915   The point numbering `IS` is parallel, with local portion indexed by local points (see `DMGetLocalSection()`). In the IS, owned edges will have their non-negative value while edges owned by different ranks will be involuted -(idx+1).
8916 
8917 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetCellNumbering()`, `DMPlexGetVertexNumbering()`, `DMPlexCreatePointNumbering()`
8918 @*/
8919 PetscErrorCode DMPlexCreateEdgeNumbering(DM dm, IS *globalEdgeNumbers)
8920 {
8921   PetscSF  sf;
8922   PetscInt eStart, eEnd;
8923 
8924   PetscFunctionBegin;
8925   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8926   PetscCall(DMGetPointSF(dm, &sf));
8927   PetscCall(DMPlexGetDepthStratum(dm, 1, &eStart, &eEnd));
8928   PetscCall(DMPlexCreateNumbering_Plex(dm, eStart, eEnd, 0, NULL, sf, globalEdgeNumbers));
8929   PetscFunctionReturn(PETSC_SUCCESS);
8930 }
8931 
8932 /*@
8933   DMPlexCreateRankField - Create a cell field whose value is the rank of the owner
8934 
8935   Input Parameter:
8936 . dm - The `DMPLEX` object
8937 
8938   Output Parameter:
8939 . ranks - The rank field
8940 
8941   Options Database Key:
8942 . -dm_partition_view - Adds the rank field into the `DM` output from `-dm_view` using the same viewer
8943 
8944   Level: intermediate
8945 
8946 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMView()`
8947 @*/
8948 PetscErrorCode DMPlexCreateRankField(DM dm, Vec *ranks)
8949 {
8950   DM             rdm;
8951   PetscFE        fe;
8952   PetscScalar   *r;
8953   PetscMPIInt    rank;
8954   DMPolytopeType ct;
8955   PetscInt       dim, cStart, cEnd, c;
8956   PetscBool      simplex;
8957 
8958   PetscFunctionBeginUser;
8959   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
8960   PetscAssertPointer(ranks, 2);
8961   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
8962   PetscCall(DMClone(dm, &rdm));
8963   PetscCall(DMGetDimension(rdm, &dim));
8964   PetscCall(DMPlexGetHeightStratum(rdm, 0, &cStart, &cEnd));
8965   PetscCall(DMPlexGetCellType(dm, cStart, &ct));
8966   simplex = DMPolytopeTypeGetNumVertices(ct) == DMPolytopeTypeGetDim(ct) + 1 ? PETSC_TRUE : PETSC_FALSE;
8967   PetscCall(PetscFECreateDefault(PETSC_COMM_SELF, dim, 1, simplex, "PETSc___rank_", -1, &fe));
8968   PetscCall(PetscObjectSetName((PetscObject)fe, "rank"));
8969   PetscCall(DMSetField(rdm, 0, NULL, (PetscObject)fe));
8970   PetscCall(PetscFEDestroy(&fe));
8971   PetscCall(DMCreateDS(rdm));
8972   PetscCall(DMCreateGlobalVector(rdm, ranks));
8973   PetscCall(PetscObjectSetName((PetscObject)*ranks, "partition"));
8974   PetscCall(VecGetArray(*ranks, &r));
8975   for (c = cStart; c < cEnd; ++c) {
8976     PetscScalar *lr;
8977 
8978     PetscCall(DMPlexPointGlobalRef(rdm, c, r, &lr));
8979     if (lr) *lr = rank;
8980   }
8981   PetscCall(VecRestoreArray(*ranks, &r));
8982   PetscCall(DMDestroy(&rdm));
8983   PetscFunctionReturn(PETSC_SUCCESS);
8984 }
8985 
8986 /*@
8987   DMPlexCreateLabelField - Create a field whose value is the label value for that point
8988 
8989   Input Parameters:
8990 + dm    - The `DMPLEX`
8991 - label - The `DMLabel`
8992 
8993   Output Parameter:
8994 . val - The label value field
8995 
8996   Options Database Key:
8997 . -dm_label_view - Adds the label value field into the `DM` output from `-dm_view` using the same viewer
8998 
8999   Level: intermediate
9000 
9001 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMView()`
9002 @*/
9003 PetscErrorCode DMPlexCreateLabelField(DM dm, DMLabel label, Vec *val)
9004 {
9005   DM             rdm, plex;
9006   Vec            lval;
9007   PetscSection   section;
9008   PetscFE        fe;
9009   PetscScalar   *v;
9010   PetscInt       dim, pStart, pEnd, p, cStart;
9011   DMPolytopeType ct;
9012   char           name[PETSC_MAX_PATH_LEN];
9013   const char    *lname, *prefix;
9014 
9015   PetscFunctionBeginUser;
9016   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9017   PetscAssertPointer(label, 2);
9018   PetscAssertPointer(val, 3);
9019   PetscCall(DMClone(dm, &rdm));
9020   PetscCall(DMConvert(rdm, DMPLEX, &plex));
9021   PetscCall(DMPlexGetHeightStratum(plex, 0, &cStart, NULL));
9022   PetscCall(DMPlexGetCellType(plex, cStart, &ct));
9023   PetscCall(DMDestroy(&plex));
9024   PetscCall(DMGetDimension(rdm, &dim));
9025   PetscCall(DMGetOptionsPrefix(dm, &prefix));
9026   PetscCall(PetscObjectGetName((PetscObject)label, &lname));
9027   PetscCall(PetscSNPrintf(name, sizeof(name), "%s%s_", prefix ? prefix : "", lname));
9028   PetscCall(PetscFECreateByCell(PETSC_COMM_SELF, dim, 1, ct, name, -1, &fe));
9029   PetscCall(PetscObjectSetName((PetscObject)fe, ""));
9030   PetscCall(DMSetField(rdm, 0, NULL, (PetscObject)fe));
9031   PetscCall(PetscFEDestroy(&fe));
9032   PetscCall(DMCreateDS(rdm));
9033   PetscCall(DMCreateGlobalVector(rdm, val));
9034   PetscCall(DMCreateLocalVector(rdm, &lval));
9035   PetscCall(PetscObjectSetName((PetscObject)*val, lname));
9036   PetscCall(DMGetLocalSection(rdm, &section));
9037   PetscCall(PetscSectionGetChart(section, &pStart, &pEnd));
9038   PetscCall(VecGetArray(lval, &v));
9039   for (p = pStart; p < pEnd; ++p) {
9040     PetscInt cval, dof, off;
9041 
9042     PetscCall(PetscSectionGetDof(section, p, &dof));
9043     if (!dof) continue;
9044     PetscCall(DMLabelGetValue(label, p, &cval));
9045     PetscCall(PetscSectionGetOffset(section, p, &off));
9046     for (PetscInt d = 0; d < dof; d++) v[off + d] = cval;
9047   }
9048   PetscCall(VecRestoreArray(lval, &v));
9049   PetscCall(DMLocalToGlobal(rdm, lval, INSERT_VALUES, *val));
9050   PetscCall(VecDestroy(&lval));
9051   PetscCall(DMDestroy(&rdm));
9052   PetscFunctionReturn(PETSC_SUCCESS);
9053 }
9054 
9055 /*@
9056   DMPlexCheckSymmetry - Check that the adjacency information in the mesh is symmetric.
9057 
9058   Input Parameter:
9059 . dm - The `DMPLEX` object
9060 
9061   Level: developer
9062 
9063   Notes:
9064   This is a useful diagnostic when creating meshes programmatically.
9065 
9066   For the complete list of DMPlexCheck* functions, see `DMSetFromOptions()`.
9067 
9068 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMCreate()`, `DMSetFromOptions()`
9069 @*/
9070 PetscErrorCode DMPlexCheckSymmetry(DM dm)
9071 {
9072   PetscSection    coneSection, supportSection;
9073   const PetscInt *cone, *support;
9074   PetscInt        coneSize, c, supportSize, s;
9075   PetscInt        pStart, pEnd, p, pp, csize, ssize;
9076   PetscBool       storagecheck = PETSC_TRUE;
9077 
9078   PetscFunctionBegin;
9079   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9080   PetscCall(DMViewFromOptions(dm, NULL, "-sym_dm_view"));
9081   PetscCall(DMPlexGetConeSection(dm, &coneSection));
9082   PetscCall(DMPlexGetSupportSection(dm, &supportSection));
9083   /* Check that point p is found in the support of its cone points, and vice versa */
9084   PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
9085   for (p = pStart; p < pEnd; ++p) {
9086     PetscCall(DMPlexGetConeSize(dm, p, &coneSize));
9087     PetscCall(DMPlexGetCone(dm, p, &cone));
9088     for (c = 0; c < coneSize; ++c) {
9089       PetscBool dup = PETSC_FALSE;
9090       PetscInt  d;
9091       for (d = c - 1; d >= 0; --d) {
9092         if (cone[c] == cone[d]) {
9093           dup = PETSC_TRUE;
9094           break;
9095         }
9096       }
9097       PetscCall(DMPlexGetSupportSize(dm, cone[c], &supportSize));
9098       PetscCall(DMPlexGetSupport(dm, cone[c], &support));
9099       for (s = 0; s < supportSize; ++s) {
9100         if (support[s] == p) break;
9101       }
9102       if ((s >= supportSize) || (dup && (support[s + 1] != p))) {
9103         PetscCall(PetscPrintf(PETSC_COMM_SELF, "p: %" PetscInt_FMT " cone: ", p));
9104         for (s = 0; s < coneSize; ++s) PetscCall(PetscPrintf(PETSC_COMM_SELF, "%" PetscInt_FMT ", ", cone[s]));
9105         PetscCall(PetscPrintf(PETSC_COMM_SELF, "\n"));
9106         PetscCall(PetscPrintf(PETSC_COMM_SELF, "p: %" PetscInt_FMT " support: ", cone[c]));
9107         for (s = 0; s < supportSize; ++s) PetscCall(PetscPrintf(PETSC_COMM_SELF, "%" PetscInt_FMT ", ", support[s]));
9108         PetscCall(PetscPrintf(PETSC_COMM_SELF, "\n"));
9109         PetscCheck(!dup, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %" PetscInt_FMT " not repeatedly found in support of repeated cone point %" PetscInt_FMT, p, cone[c]);
9110         SETERRQ(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %" PetscInt_FMT " not found in support of cone point %" PetscInt_FMT, p, cone[c]);
9111       }
9112     }
9113     PetscCall(DMPlexGetTreeParent(dm, p, &pp, NULL));
9114     if (p != pp) {
9115       storagecheck = PETSC_FALSE;
9116       continue;
9117     }
9118     PetscCall(DMPlexGetSupportSize(dm, p, &supportSize));
9119     PetscCall(DMPlexGetSupport(dm, p, &support));
9120     for (s = 0; s < supportSize; ++s) {
9121       PetscCall(DMPlexGetConeSize(dm, support[s], &coneSize));
9122       PetscCall(DMPlexGetCone(dm, support[s], &cone));
9123       for (c = 0; c < coneSize; ++c) {
9124         PetscCall(DMPlexGetTreeParent(dm, cone[c], &pp, NULL));
9125         if (cone[c] != pp) {
9126           c = 0;
9127           break;
9128         }
9129         if (cone[c] == p) break;
9130       }
9131       if (c >= coneSize) {
9132         PetscCall(PetscPrintf(PETSC_COMM_SELF, "p: %" PetscInt_FMT " support: ", p));
9133         for (c = 0; c < supportSize; ++c) PetscCall(PetscPrintf(PETSC_COMM_SELF, "%" PetscInt_FMT ", ", support[c]));
9134         PetscCall(PetscPrintf(PETSC_COMM_SELF, "\n"));
9135         PetscCall(PetscPrintf(PETSC_COMM_SELF, "p: %" PetscInt_FMT " cone: ", support[s]));
9136         for (c = 0; c < coneSize; ++c) PetscCall(PetscPrintf(PETSC_COMM_SELF, "%" PetscInt_FMT ", ", cone[c]));
9137         PetscCall(PetscPrintf(PETSC_COMM_SELF, "\n"));
9138         SETERRQ(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %" PetscInt_FMT " not found in cone of support point %" PetscInt_FMT, p, support[s]);
9139       }
9140     }
9141   }
9142   if (storagecheck) {
9143     PetscCall(PetscSectionGetStorageSize(coneSection, &csize));
9144     PetscCall(PetscSectionGetStorageSize(supportSection, &ssize));
9145     PetscCheck(csize == ssize, PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "Total cone size %" PetscInt_FMT " != Total support size %" PetscInt_FMT, csize, ssize);
9146   }
9147   PetscFunctionReturn(PETSC_SUCCESS);
9148 }
9149 
9150 /*
9151   For submeshes with cohesive cells (see DMPlexConstructCohesiveCells()), we allow a special case where some of the boundary of a face (edges and vertices) are not duplicated. We call these special boundary points "unsplit", since the same edge or vertex appears in both copies of the face. These unsplit points throw off our counting, so we have to explicitly account for them here.
9152 */
9153 static PetscErrorCode DMPlexCellUnsplitVertices_Private(DM dm, PetscInt c, DMPolytopeType ct, PetscInt *unsplit)
9154 {
9155   DMPolytopeType  cct;
9156   PetscInt        ptpoints[4];
9157   const PetscInt *cone, *ccone, *ptcone;
9158   PetscInt        coneSize, cp, cconeSize, ccp, npt = 0, pt;
9159 
9160   PetscFunctionBegin;
9161   *unsplit = 0;
9162   switch (ct) {
9163   case DM_POLYTOPE_POINT_PRISM_TENSOR:
9164     ptpoints[npt++] = c;
9165     break;
9166   case DM_POLYTOPE_SEG_PRISM_TENSOR:
9167     PetscCall(DMPlexGetCone(dm, c, &cone));
9168     PetscCall(DMPlexGetConeSize(dm, c, &coneSize));
9169     for (cp = 0; cp < coneSize; ++cp) {
9170       PetscCall(DMPlexGetCellType(dm, cone[cp], &cct));
9171       if (cct == DM_POLYTOPE_POINT_PRISM_TENSOR) ptpoints[npt++] = cone[cp];
9172     }
9173     break;
9174   case DM_POLYTOPE_TRI_PRISM_TENSOR:
9175   case DM_POLYTOPE_QUAD_PRISM_TENSOR:
9176     PetscCall(DMPlexGetCone(dm, c, &cone));
9177     PetscCall(DMPlexGetConeSize(dm, c, &coneSize));
9178     for (cp = 0; cp < coneSize; ++cp) {
9179       PetscCall(DMPlexGetCone(dm, cone[cp], &ccone));
9180       PetscCall(DMPlexGetConeSize(dm, cone[cp], &cconeSize));
9181       for (ccp = 0; ccp < cconeSize; ++ccp) {
9182         PetscCall(DMPlexGetCellType(dm, ccone[ccp], &cct));
9183         if (cct == DM_POLYTOPE_POINT_PRISM_TENSOR) {
9184           PetscInt p;
9185           for (p = 0; p < npt; ++p)
9186             if (ptpoints[p] == ccone[ccp]) break;
9187           if (p == npt) ptpoints[npt++] = ccone[ccp];
9188         }
9189       }
9190     }
9191     break;
9192   default:
9193     break;
9194   }
9195   for (pt = 0; pt < npt; ++pt) {
9196     PetscCall(DMPlexGetCone(dm, ptpoints[pt], &ptcone));
9197     if (ptcone[0] == ptcone[1]) ++(*unsplit);
9198   }
9199   PetscFunctionReturn(PETSC_SUCCESS);
9200 }
9201 
9202 /*@
9203   DMPlexCheckSkeleton - Check that each cell has the correct number of vertices
9204 
9205   Input Parameters:
9206 + dm         - The `DMPLEX` object
9207 - cellHeight - Normally 0
9208 
9209   Level: developer
9210 
9211   Notes:
9212   This is a useful diagnostic when creating meshes programmatically.
9213   Currently applicable only to homogeneous simplex or tensor meshes.
9214 
9215   For the complete list of DMPlexCheck* functions, see `DMSetFromOptions()`.
9216 
9217 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMCreate()`, `DMSetFromOptions()`
9218 @*/
9219 PetscErrorCode DMPlexCheckSkeleton(DM dm, PetscInt cellHeight)
9220 {
9221   DMPlexInterpolatedFlag interp;
9222   DMPolytopeType         ct;
9223   PetscInt               vStart, vEnd, cStart, cEnd, c;
9224 
9225   PetscFunctionBegin;
9226   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9227   PetscCall(DMPlexIsInterpolated(dm, &interp));
9228   PetscCall(DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd));
9229   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
9230   for (c = cStart; c < cEnd; ++c) {
9231     PetscInt *closure = NULL;
9232     PetscInt  coneSize, closureSize, cl, Nv = 0;
9233 
9234     PetscCall(DMPlexGetCellType(dm, c, &ct));
9235     if (ct == DM_POLYTOPE_UNKNOWN) continue;
9236     if (interp == DMPLEX_INTERPOLATED_FULL) {
9237       PetscCall(DMPlexGetConeSize(dm, c, &coneSize));
9238       PetscCheck(coneSize == DMPolytopeTypeGetConeSize(ct), PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %" PetscInt_FMT " of type %s has cone size %" PetscInt_FMT " != %" PetscInt_FMT, c, DMPolytopeTypes[ct], coneSize, DMPolytopeTypeGetConeSize(ct));
9239     }
9240     PetscCall(DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure));
9241     for (cl = 0; cl < closureSize * 2; cl += 2) {
9242       const PetscInt p = closure[cl];
9243       if ((p >= vStart) && (p < vEnd)) ++Nv;
9244     }
9245     PetscCall(DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure));
9246     /* Special Case: Tensor faces with identified vertices */
9247     if (Nv < DMPolytopeTypeGetNumVertices(ct)) {
9248       PetscInt unsplit;
9249 
9250       PetscCall(DMPlexCellUnsplitVertices_Private(dm, c, ct, &unsplit));
9251       if (Nv + unsplit == DMPolytopeTypeGetNumVertices(ct)) continue;
9252     }
9253     PetscCheck(Nv == DMPolytopeTypeGetNumVertices(ct), PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %" PetscInt_FMT " of type %s has %" PetscInt_FMT " vertices != %" PetscInt_FMT, c, DMPolytopeTypes[ct], Nv, DMPolytopeTypeGetNumVertices(ct));
9254   }
9255   PetscFunctionReturn(PETSC_SUCCESS);
9256 }
9257 
9258 /*@
9259   DMPlexCheckFaces - Check that the faces of each cell give a vertex order this is consistent with what we expect from the cell type
9260 
9261   Collective
9262 
9263   Input Parameters:
9264 + dm         - The `DMPLEX` object
9265 - cellHeight - Normally 0
9266 
9267   Level: developer
9268 
9269   Notes:
9270   This is a useful diagnostic when creating meshes programmatically.
9271   This routine is only relevant for meshes that are fully interpolated across all ranks.
9272   It will error out if a partially interpolated mesh is given on some rank.
9273   It will do nothing for locally uninterpolated mesh (as there is nothing to check).
9274 
9275   For the complete list of DMPlexCheck* functions, see `DMSetFromOptions()`.
9276 
9277 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMCreate()`, `DMPlexGetVTKCellHeight()`, `DMSetFromOptions()`
9278 @*/
9279 PetscErrorCode DMPlexCheckFaces(DM dm, PetscInt cellHeight)
9280 {
9281   PetscInt               dim, depth, vStart, vEnd, cStart, cEnd, c, h;
9282   DMPlexInterpolatedFlag interpEnum;
9283 
9284   PetscFunctionBegin;
9285   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9286   PetscCall(DMPlexIsInterpolatedCollective(dm, &interpEnum));
9287   if (interpEnum == DMPLEX_INTERPOLATED_NONE) PetscFunctionReturn(PETSC_SUCCESS);
9288   if (interpEnum != DMPLEX_INTERPOLATED_FULL) {
9289     PetscCall(PetscPrintf(PetscObjectComm((PetscObject)dm), "DMPlexCheckFaces() warning: Mesh is only partially interpolated, this is currently not supported"));
9290     PetscFunctionReturn(PETSC_SUCCESS);
9291   }
9292 
9293   PetscCall(DMGetDimension(dm, &dim));
9294   PetscCall(DMPlexGetDepth(dm, &depth));
9295   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
9296   for (h = cellHeight; h < PetscMin(depth, dim); ++h) {
9297     PetscCall(DMPlexGetHeightStratum(dm, h, &cStart, &cEnd));
9298     for (c = cStart; c < cEnd; ++c) {
9299       const PetscInt       *cone, *ornt, *faceSizes, *faces;
9300       const DMPolytopeType *faceTypes;
9301       DMPolytopeType        ct;
9302       PetscInt              numFaces, coneSize, f;
9303       PetscInt             *closure = NULL, closureSize, cl, numCorners = 0, fOff = 0, unsplit;
9304 
9305       PetscCall(DMPlexGetCellType(dm, c, &ct));
9306       PetscCall(DMPlexCellUnsplitVertices_Private(dm, c, ct, &unsplit));
9307       if (unsplit) continue;
9308       PetscCall(DMPlexGetConeSize(dm, c, &coneSize));
9309       PetscCall(DMPlexGetCone(dm, c, &cone));
9310       PetscCall(DMPlexGetConeOrientation(dm, c, &ornt));
9311       PetscCall(DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure));
9312       for (cl = 0; cl < closureSize * 2; cl += 2) {
9313         const PetscInt p = closure[cl];
9314         if ((p >= vStart) && (p < vEnd)) closure[numCorners++] = p;
9315       }
9316       PetscCall(DMPlexGetRawFaces_Internal(dm, ct, closure, &numFaces, &faceTypes, &faceSizes, &faces));
9317       PetscCheck(coneSize == numFaces, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cell %" PetscInt_FMT " of type %s has %" PetscInt_FMT " faces but should have %" PetscInt_FMT, c, DMPolytopeTypes[ct], coneSize, numFaces);
9318       for (f = 0; f < numFaces; ++f) {
9319         DMPolytopeType fct;
9320         PetscInt      *fclosure = NULL, fclosureSize, cl, fnumCorners = 0, v;
9321 
9322         PetscCall(DMPlexGetCellType(dm, cone[f], &fct));
9323         PetscCall(DMPlexGetTransitiveClosure_Internal(dm, cone[f], ornt[f], PETSC_TRUE, &fclosureSize, &fclosure));
9324         for (cl = 0; cl < fclosureSize * 2; cl += 2) {
9325           const PetscInt p = fclosure[cl];
9326           if ((p >= vStart) && (p < vEnd)) fclosure[fnumCorners++] = p;
9327         }
9328         PetscCheck(fnumCorners == faceSizes[f], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " of type %s (cone idx %" PetscInt_FMT ") of cell %" PetscInt_FMT " of type %s has %" PetscInt_FMT " vertices but should have %" PetscInt_FMT, cone[f], DMPolytopeTypes[fct], f, c, DMPolytopeTypes[ct], fnumCorners, faceSizes[f]);
9329         for (v = 0; v < fnumCorners; ++v) {
9330           if (fclosure[v] != faces[fOff + v]) {
9331             PetscInt v1;
9332 
9333             PetscCall(PetscPrintf(PETSC_COMM_SELF, "face closure:"));
9334             for (v1 = 0; v1 < fnumCorners; ++v1) PetscCall(PetscPrintf(PETSC_COMM_SELF, " %" PetscInt_FMT, fclosure[v1]));
9335             PetscCall(PetscPrintf(PETSC_COMM_SELF, "\ncell face:"));
9336             for (v1 = 0; v1 < fnumCorners; ++v1) PetscCall(PetscPrintf(PETSC_COMM_SELF, " %" PetscInt_FMT, faces[fOff + v1]));
9337             PetscCall(PetscPrintf(PETSC_COMM_SELF, "\n"));
9338             SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " of type %s (cone idx %" PetscInt_FMT ", ornt %" PetscInt_FMT ") of cell %" PetscInt_FMT " of type %s vertex %" PetscInt_FMT ", %" PetscInt_FMT " != %" PetscInt_FMT, cone[f], DMPolytopeTypes[fct], f, ornt[f], c, DMPolytopeTypes[ct], v, fclosure[v], faces[fOff + v]);
9339           }
9340         }
9341         PetscCall(DMPlexRestoreTransitiveClosure(dm, cone[f], PETSC_TRUE, &fclosureSize, &fclosure));
9342         fOff += faceSizes[f];
9343       }
9344       PetscCall(DMPlexRestoreRawFaces_Internal(dm, ct, closure, &numFaces, &faceTypes, &faceSizes, &faces));
9345       PetscCall(DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &closureSize, &closure));
9346     }
9347   }
9348   PetscFunctionReturn(PETSC_SUCCESS);
9349 }
9350 
9351 /*@
9352   DMPlexCheckGeometry - Check the geometry of mesh cells
9353 
9354   Input Parameter:
9355 . dm - The `DMPLEX` object
9356 
9357   Level: developer
9358 
9359   Notes:
9360   This is a useful diagnostic when creating meshes programmatically.
9361 
9362   For the complete list of DMPlexCheck* functions, see `DMSetFromOptions()`.
9363 
9364 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMCreate()`, `DMSetFromOptions()`
9365 @*/
9366 PetscErrorCode DMPlexCheckGeometry(DM dm)
9367 {
9368   Vec       coordinates;
9369   PetscReal detJ, J[9], refVol = 1.0;
9370   PetscReal vol;
9371   PetscInt  dim, depth, dE, d, cStart, cEnd, c;
9372 
9373   PetscFunctionBegin;
9374   PetscCall(DMGetDimension(dm, &dim));
9375   PetscCall(DMGetCoordinateDim(dm, &dE));
9376   if (dim != dE) PetscFunctionReturn(PETSC_SUCCESS);
9377   PetscCall(DMPlexGetDepth(dm, &depth));
9378   for (d = 0; d < dim; ++d) refVol *= 2.0;
9379   PetscCall(DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd));
9380   /* Make sure local coordinates are created, because that step is collective */
9381   PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
9382   if (!coordinates) PetscFunctionReturn(PETSC_SUCCESS);
9383   for (c = cStart; c < cEnd; ++c) {
9384     DMPolytopeType ct;
9385     PetscInt       unsplit;
9386     PetscBool      ignoreZeroVol = PETSC_FALSE;
9387 
9388     PetscCall(DMPlexGetCellType(dm, c, &ct));
9389     switch (ct) {
9390     case DM_POLYTOPE_SEG_PRISM_TENSOR:
9391     case DM_POLYTOPE_TRI_PRISM_TENSOR:
9392     case DM_POLYTOPE_QUAD_PRISM_TENSOR:
9393       ignoreZeroVol = PETSC_TRUE;
9394       break;
9395     default:
9396       break;
9397     }
9398     switch (ct) {
9399     case DM_POLYTOPE_TRI_PRISM:
9400     case DM_POLYTOPE_TRI_PRISM_TENSOR:
9401     case DM_POLYTOPE_QUAD_PRISM_TENSOR:
9402     case DM_POLYTOPE_PYRAMID:
9403       continue;
9404     default:
9405       break;
9406     }
9407     PetscCall(DMPlexCellUnsplitVertices_Private(dm, c, ct, &unsplit));
9408     if (unsplit) continue;
9409     PetscCall(DMPlexComputeCellGeometryFEM(dm, c, NULL, NULL, J, NULL, &detJ));
9410     PetscCheck(detJ >= -PETSC_SMALL && (detJ > 0.0 || ignoreZeroVol), PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mesh cell %" PetscInt_FMT " of type %s is inverted, |J| = %g", c, DMPolytopeTypes[ct], (double)detJ);
9411     PetscCall(PetscInfo(dm, "Cell %" PetscInt_FMT " FEM Volume %g\n", c, (double)(detJ * refVol)));
9412     /* This should work with periodicity since DG coordinates should be used */
9413     if (depth > 1) {
9414       PetscCall(DMPlexComputeCellGeometryFVM(dm, c, &vol, NULL, NULL));
9415       PetscCheck(vol >= -PETSC_SMALL && (vol > 0.0 || ignoreZeroVol), PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mesh cell %" PetscInt_FMT " of type %s is inverted, vol = %g", c, DMPolytopeTypes[ct], (double)vol);
9416       PetscCall(PetscInfo(dm, "Cell %" PetscInt_FMT " FVM Volume %g\n", c, (double)vol));
9417     }
9418   }
9419   PetscFunctionReturn(PETSC_SUCCESS);
9420 }
9421 
9422 /*@
9423   DMPlexCheckPointSF - Check that several necessary conditions are met for the point `PetscSF` of this plex.
9424 
9425   Collective
9426 
9427   Input Parameters:
9428 + dm              - The `DMPLEX` object
9429 . pointSF         - The `PetscSF`, or `NULL` for `PointSF` attached to `DM`
9430 - allowExtraRoots - Flag to allow extra points not present in the `DM`
9431 
9432   Level: developer
9433 
9434   Notes:
9435   This is mainly intended for debugging/testing purposes.
9436 
9437   For the complete list of DMPlexCheck* functions, see `DMSetFromOptions()`.
9438 
9439   Extra roots can come from periodic cuts, where additional points appear on the boundary
9440 
9441 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMGetPointSF()`, `DMSetFromOptions()`
9442 @*/
9443 PetscErrorCode DMPlexCheckPointSF(DM dm, PetscSF pointSF, PetscBool allowExtraRoots)
9444 {
9445   PetscInt           l, nleaves, nroots, overlap;
9446   const PetscInt    *locals;
9447   const PetscSFNode *remotes;
9448   PetscBool          distributed;
9449   MPI_Comm           comm;
9450   PetscMPIInt        rank;
9451 
9452   PetscFunctionBegin;
9453   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9454   if (pointSF) PetscValidHeaderSpecific(pointSF, PETSCSF_CLASSID, 2);
9455   else pointSF = dm->sf;
9456   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
9457   PetscCheck(pointSF, comm, PETSC_ERR_ARG_WRONGSTATE, "DMPlex must have Point SF attached");
9458   PetscCallMPI(MPI_Comm_rank(comm, &rank));
9459   {
9460     PetscMPIInt mpiFlag;
9461 
9462     PetscCallMPI(MPI_Comm_compare(comm, PetscObjectComm((PetscObject)pointSF), &mpiFlag));
9463     PetscCheck(mpiFlag == MPI_CONGRUENT || mpiFlag == MPI_IDENT, PETSC_COMM_SELF, PETSC_ERR_ARG_NOTSAMECOMM, "DM and Point SF have different communicators (flag %d)", mpiFlag);
9464   }
9465   PetscCall(PetscSFGetGraph(pointSF, &nroots, &nleaves, &locals, &remotes));
9466   PetscCall(DMPlexIsDistributed(dm, &distributed));
9467   if (!distributed) {
9468     PetscCheck(nroots < 0 || nleaves == 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Undistributed DMPlex cannot have non-empty PointSF (has %" PetscInt_FMT " roots, %" PetscInt_FMT " leaves)", nroots, nleaves);
9469     PetscFunctionReturn(PETSC_SUCCESS);
9470   }
9471   PetscCheck(nroots >= 0, comm, PETSC_ERR_ARG_WRONGSTATE, "This DMPlex is distributed but its PointSF has no graph set (has %" PetscInt_FMT " roots, %" PetscInt_FMT " leaves)", nroots, nleaves);
9472   PetscCall(DMPlexGetOverlap(dm, &overlap));
9473 
9474   /* Check SF graph is compatible with DMPlex chart */
9475   {
9476     PetscInt pStart, pEnd, maxLeaf;
9477 
9478     PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
9479     PetscCall(PetscSFGetLeafRange(pointSF, NULL, &maxLeaf));
9480     PetscCheck(allowExtraRoots || pEnd - pStart == nroots, PETSC_COMM_SELF, PETSC_ERR_PLIB, "pEnd - pStart = %" PetscInt_FMT " != nroots = %" PetscInt_FMT, pEnd - pStart, nroots);
9481     PetscCheck(maxLeaf < pEnd, PETSC_COMM_SELF, PETSC_ERR_PLIB, "maxLeaf = %" PetscInt_FMT " >= pEnd = %" PetscInt_FMT, maxLeaf, pEnd);
9482   }
9483 
9484   /* Check Point SF has no local points referenced */
9485   for (l = 0; l < nleaves; l++) {
9486     PetscAssert(remotes[l].rank != (PetscInt)rank, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point SF contains local point %" PetscInt_FMT " <- (%d,%" PetscInt_FMT ")", locals ? locals[l] : l, (PetscMPIInt)remotes[l].rank, remotes[l].index);
9487   }
9488 
9489   /* Check there are no cells in interface */
9490   if (!overlap) {
9491     PetscInt cellHeight, cStart, cEnd;
9492 
9493     PetscCall(DMPlexGetVTKCellHeight(dm, &cellHeight));
9494     PetscCall(DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd));
9495     for (l = 0; l < nleaves; ++l) {
9496       const PetscInt point = locals ? locals[l] : l;
9497 
9498       PetscCheck(point < cStart || point >= cEnd, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point SF contains %" PetscInt_FMT " which is a cell", point);
9499     }
9500   }
9501 
9502   /* If some point is in interface, then all its cone points must be also in interface (either as leaves or roots) */
9503   {
9504     const PetscInt *rootdegree;
9505 
9506     PetscCall(PetscSFComputeDegreeBegin(pointSF, &rootdegree));
9507     PetscCall(PetscSFComputeDegreeEnd(pointSF, &rootdegree));
9508     for (l = 0; l < nleaves; ++l) {
9509       const PetscInt  point = locals ? locals[l] : l;
9510       const PetscInt *cone;
9511       PetscInt        coneSize, c, idx;
9512 
9513       PetscCall(DMPlexGetConeSize(dm, point, &coneSize));
9514       PetscCall(DMPlexGetCone(dm, point, &cone));
9515       for (c = 0; c < coneSize; ++c) {
9516         if (!rootdegree[cone[c]]) {
9517           if (locals) {
9518             PetscCall(PetscFindInt(cone[c], nleaves, locals, &idx));
9519           } else {
9520             idx = (cone[c] < nleaves) ? cone[c] : -1;
9521           }
9522           PetscCheck(idx >= 0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point SF contains %" PetscInt_FMT " but not %" PetscInt_FMT " from its cone", point, cone[c]);
9523         }
9524       }
9525     }
9526   }
9527   PetscFunctionReturn(PETSC_SUCCESS);
9528 }
9529 
9530 /*@
9531   DMPlexCheckOrphanVertices - Check that no vertices are disconnected from the mesh, unless the mesh only consists of disconnected vertices.
9532 
9533   Collective
9534 
9535   Input Parameter:
9536 . dm - The `DMPLEX` object
9537 
9538   Level: developer
9539 
9540   Notes:
9541   This is mainly intended for debugging/testing purposes.
9542 
9543   Other cell types which are disconnected would be caught by the symmetry and face checks.
9544 
9545   For the complete list of DMPlexCheck* functions, see `DMSetFromOptions()`.
9546 
9547 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCheck()`, `DMSetFromOptions()`
9548 @*/
9549 PetscErrorCode DMPlexCheckOrphanVertices(DM dm)
9550 {
9551   PetscInt pStart, pEnd, vStart, vEnd;
9552 
9553   PetscFunctionBegin;
9554   PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
9555   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
9556   if (pStart == vStart && pEnd == vEnd) PetscFunctionReturn(PETSC_SUCCESS);
9557   for (PetscInt v = vStart; v < vEnd; ++v) {
9558     PetscInt suppSize;
9559 
9560     PetscCall(DMPlexGetSupportSize(dm, v, &suppSize));
9561     PetscCheck(suppSize, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Vertex %" PetscInt_FMT " is disconnected from the mesh", v);
9562   }
9563   PetscFunctionReturn(PETSC_SUCCESS);
9564 }
9565 
9566 /*@
9567   DMPlexCheck - Perform various checks of `DMPLEX` sanity
9568 
9569   Input Parameter:
9570 . dm - The `DMPLEX` object
9571 
9572   Level: developer
9573 
9574   Notes:
9575   This is a useful diagnostic when creating meshes programmatically.
9576 
9577   For the complete list of DMPlexCheck* functions, see `DMSetFromOptions()`.
9578 
9579   Currently does not include `DMPlexCheckCellShape()`.
9580 
9581 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMCreate()`, `DMSetFromOptions()`
9582 @*/
9583 PetscErrorCode DMPlexCheck(DM dm)
9584 {
9585   PetscInt cellHeight;
9586 
9587   PetscFunctionBegin;
9588   PetscCall(DMPlexGetVTKCellHeight(dm, &cellHeight));
9589   PetscCall(DMPlexCheckSymmetry(dm));
9590   PetscCall(DMPlexCheckSkeleton(dm, cellHeight));
9591   PetscCall(DMPlexCheckFaces(dm, cellHeight));
9592   PetscCall(DMPlexCheckGeometry(dm));
9593   PetscCall(DMPlexCheckPointSF(dm, NULL, PETSC_FALSE));
9594   PetscCall(DMPlexCheckInterfaceCones(dm));
9595   PetscCall(DMPlexCheckOrphanVertices(dm));
9596   PetscFunctionReturn(PETSC_SUCCESS);
9597 }
9598 
9599 typedef struct cell_stats {
9600   PetscReal min, max, sum, squaresum;
9601   PetscInt  count;
9602 } cell_stats_t;
9603 
9604 static void MPIAPI cell_stats_reduce(void *a, void *b, int *len, MPI_Datatype *datatype)
9605 {
9606   PetscInt i, N = *len;
9607 
9608   for (i = 0; i < N; i++) {
9609     cell_stats_t *A = (cell_stats_t *)a;
9610     cell_stats_t *B = (cell_stats_t *)b;
9611 
9612     B->min = PetscMin(A->min, B->min);
9613     B->max = PetscMax(A->max, B->max);
9614     B->sum += A->sum;
9615     B->squaresum += A->squaresum;
9616     B->count += A->count;
9617   }
9618 }
9619 
9620 /*@
9621   DMPlexCheckCellShape - Checks the Jacobian of the mapping from reference to real cells and computes some minimal statistics.
9622 
9623   Collective
9624 
9625   Input Parameters:
9626 + dm        - The `DMPLEX` object
9627 . output    - If true, statistics will be displayed on `stdout`
9628 - condLimit - Display all cells above this condition number, or `PETSC_DETERMINE` for no cell output
9629 
9630   Level: developer
9631 
9632   Notes:
9633   This is mainly intended for debugging/testing purposes.
9634 
9635   For the complete list of DMPlexCheck* functions, see `DMSetFromOptions()`.
9636 
9637 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMSetFromOptions()`, `DMPlexComputeOrthogonalQuality()`
9638 @*/
9639 PetscErrorCode DMPlexCheckCellShape(DM dm, PetscBool output, PetscReal condLimit)
9640 {
9641   DM           dmCoarse;
9642   cell_stats_t stats, globalStats;
9643   MPI_Comm     comm = PetscObjectComm((PetscObject)dm);
9644   PetscReal   *J, *invJ, min = 0, max = 0, mean = 0, stdev = 0;
9645   PetscReal    limit = condLimit > 0 ? condLimit : PETSC_MAX_REAL;
9646   PetscInt     cdim, cStart, cEnd, c, eStart, eEnd, count = 0;
9647   PetscMPIInt  rank, size;
9648 
9649   PetscFunctionBegin;
9650   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9651   stats.min = PETSC_MAX_REAL;
9652   stats.max = PETSC_MIN_REAL;
9653   stats.sum = stats.squaresum = 0.;
9654   stats.count                 = 0;
9655 
9656   PetscCallMPI(MPI_Comm_size(comm, &size));
9657   PetscCallMPI(MPI_Comm_rank(comm, &rank));
9658   PetscCall(DMGetCoordinateDim(dm, &cdim));
9659   PetscCall(PetscMalloc2(PetscSqr(cdim), &J, PetscSqr(cdim), &invJ));
9660   PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd));
9661   PetscCall(DMPlexGetDepthStratum(dm, 1, &eStart, &eEnd));
9662   for (c = cStart; c < cEnd; c++) {
9663     PetscInt  i;
9664     PetscReal frobJ = 0., frobInvJ = 0., cond2, cond, detJ;
9665 
9666     PetscCall(DMPlexComputeCellGeometryAffineFEM(dm, c, NULL, J, invJ, &detJ));
9667     PetscCheck(detJ >= 0.0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mesh cell %" PetscInt_FMT " is inverted", c);
9668     for (i = 0; i < PetscSqr(cdim); ++i) {
9669       frobJ += J[i] * J[i];
9670       frobInvJ += invJ[i] * invJ[i];
9671     }
9672     cond2 = frobJ * frobInvJ;
9673     cond  = PetscSqrtReal(cond2);
9674 
9675     stats.min = PetscMin(stats.min, cond);
9676     stats.max = PetscMax(stats.max, cond);
9677     stats.sum += cond;
9678     stats.squaresum += cond2;
9679     stats.count++;
9680     if (output && cond > limit) {
9681       PetscSection coordSection;
9682       Vec          coordsLocal;
9683       PetscScalar *coords = NULL;
9684       PetscInt     Nv, d, clSize, cl, *closure = NULL;
9685 
9686       PetscCall(DMGetCoordinatesLocal(dm, &coordsLocal));
9687       PetscCall(DMGetCoordinateSection(dm, &coordSection));
9688       PetscCall(DMPlexVecGetClosure(dm, coordSection, coordsLocal, c, &Nv, &coords));
9689       PetscCall(PetscSynchronizedPrintf(comm, "[%d] Cell %" PetscInt_FMT " cond %g\n", rank, c, (double)cond));
9690       for (i = 0; i < Nv / cdim; ++i) {
9691         PetscCall(PetscSynchronizedPrintf(comm, "  Vertex %" PetscInt_FMT ": (", i));
9692         for (d = 0; d < cdim; ++d) {
9693           if (d > 0) PetscCall(PetscSynchronizedPrintf(comm, ", "));
9694           PetscCall(PetscSynchronizedPrintf(comm, "%g", (double)PetscRealPart(coords[i * cdim + d])));
9695         }
9696         PetscCall(PetscSynchronizedPrintf(comm, ")\n"));
9697       }
9698       PetscCall(DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &clSize, &closure));
9699       for (cl = 0; cl < clSize * 2; cl += 2) {
9700         const PetscInt edge = closure[cl];
9701 
9702         if ((edge >= eStart) && (edge < eEnd)) {
9703           PetscReal len;
9704 
9705           PetscCall(DMPlexComputeCellGeometryFVM(dm, edge, &len, NULL, NULL));
9706           PetscCall(PetscSynchronizedPrintf(comm, "  Edge %" PetscInt_FMT ": length %g\n", edge, (double)len));
9707         }
9708       }
9709       PetscCall(DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &clSize, &closure));
9710       PetscCall(DMPlexVecRestoreClosure(dm, coordSection, coordsLocal, c, &Nv, &coords));
9711     }
9712   }
9713   if (output) PetscCall(PetscSynchronizedFlush(comm, NULL));
9714 
9715   if (size > 1) {
9716     PetscMPIInt  blockLengths[2] = {4, 1};
9717     MPI_Aint     blockOffsets[2] = {offsetof(cell_stats_t, min), offsetof(cell_stats_t, count)};
9718     MPI_Datatype blockTypes[2]   = {MPIU_REAL, MPIU_INT}, statType;
9719     MPI_Op       statReduce;
9720 
9721     PetscCallMPI(MPI_Type_create_struct(2, blockLengths, blockOffsets, blockTypes, &statType));
9722     PetscCallMPI(MPI_Type_commit(&statType));
9723     PetscCallMPI(MPI_Op_create(cell_stats_reduce, PETSC_TRUE, &statReduce));
9724     PetscCallMPI(MPI_Reduce(&stats, &globalStats, 1, statType, statReduce, 0, comm));
9725     PetscCallMPI(MPI_Op_free(&statReduce));
9726     PetscCallMPI(MPI_Type_free(&statType));
9727   } else {
9728     PetscCall(PetscArraycpy(&globalStats, &stats, 1));
9729   }
9730   if (rank == 0) {
9731     count = globalStats.count;
9732     min   = globalStats.min;
9733     max   = globalStats.max;
9734     mean  = globalStats.sum / globalStats.count;
9735     stdev = globalStats.count > 1 ? PetscSqrtReal(PetscMax((globalStats.squaresum - globalStats.count * mean * mean) / (globalStats.count - 1), 0)) : 0.0;
9736   }
9737 
9738   if (output) PetscCall(PetscPrintf(comm, "Mesh with %" PetscInt_FMT " cells, shape condition numbers: min = %g, max = %g, mean = %g, stddev = %g\n", count, (double)min, (double)max, (double)mean, (double)stdev));
9739   PetscCall(PetscFree2(J, invJ));
9740 
9741   PetscCall(DMGetCoarseDM(dm, &dmCoarse));
9742   if (dmCoarse) {
9743     PetscBool isplex;
9744 
9745     PetscCall(PetscObjectTypeCompare((PetscObject)dmCoarse, DMPLEX, &isplex));
9746     if (isplex) PetscCall(DMPlexCheckCellShape(dmCoarse, output, condLimit));
9747   }
9748   PetscFunctionReturn(PETSC_SUCCESS);
9749 }
9750 
9751 /*@
9752   DMPlexComputeOrthogonalQuality - Compute cell-wise orthogonal quality mesh statistic. Optionally tags all cells with
9753   orthogonal quality below given tolerance.
9754 
9755   Collective
9756 
9757   Input Parameters:
9758 + dm   - The `DMPLEX` object
9759 . fv   - Optional `PetscFV` object for pre-computed cell/face centroid information
9760 - atol - [0, 1] Absolute tolerance for tagging cells.
9761 
9762   Output Parameters:
9763 + OrthQual      - `Vec` containing orthogonal quality per cell
9764 - OrthQualLabel - `DMLabel` tagging cells below atol with `DM_ADAPT_REFINE`
9765 
9766   Options Database Keys:
9767 + -dm_plex_orthogonal_quality_label_view - view OrthQualLabel if label is requested. Currently only `PETSCVIEWERASCII` is supported.
9768 - -dm_plex_orthogonal_quality_vec_view   - view OrthQual vector.
9769 
9770   Level: intermediate
9771 
9772   Notes:
9773   Orthogonal quality is given by the following formula\:
9774 
9775   $ \min \left[ \frac{A_i \cdot f_i}{\|A_i\| \|f_i\|} , \frac{A_i \cdot c_i}{\|A_i\| \|c_i\|} \right]$
9776 
9777   Where A_i is the i'th face-normal vector, f_i is the vector from the cell centroid to the i'th face centroid, and c_i
9778   is the vector from the current cells centroid to the centroid of its i'th neighbor (which shares a face with the
9779   current cell). This computes the vector similarity between each cell face and its corresponding neighbor centroid by
9780   calculating the cosine of the angle between these vectors.
9781 
9782   Orthogonal quality ranges from 1 (best) to 0 (worst).
9783 
9784   This routine is mainly useful for FVM, however is not restricted to only FVM. The `PetscFV` object is optionally used to check for
9785   pre-computed FVM cell data, but if it is not passed in then this data will be computed.
9786 
9787   Cells are tagged if they have an orthogonal quality less than or equal to the absolute tolerance.
9788 
9789 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCheckCellShape()`, `DMCreateLabel()`, `PetscFV`, `DMLabel`, `Vec`
9790 @*/
9791 PetscErrorCode DMPlexComputeOrthogonalQuality(DM dm, PetscFV fv, PetscReal atol, Vec *OrthQual, DMLabel *OrthQualLabel)
9792 {
9793   PetscInt               nc, cellHeight, cStart, cEnd, cell, cellIter = 0;
9794   PetscInt              *idx;
9795   PetscScalar           *oqVals;
9796   const PetscScalar     *cellGeomArr, *faceGeomArr;
9797   PetscReal             *ci, *fi, *Ai;
9798   MPI_Comm               comm;
9799   Vec                    cellgeom, facegeom;
9800   DM                     dmFace, dmCell;
9801   IS                     glob;
9802   ISLocalToGlobalMapping ltog;
9803   PetscViewer            vwr;
9804 
9805   PetscFunctionBegin;
9806   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9807   if (fv) PetscValidHeaderSpecific(fv, PETSCFV_CLASSID, 2);
9808   PetscAssertPointer(OrthQual, 4);
9809   PetscCheck(atol >= 0.0 && atol <= 1.0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Absolute tolerance %g not in [0,1]", (double)atol);
9810   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
9811   PetscCall(DMGetDimension(dm, &nc));
9812   PetscCheck(nc >= 2, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONGSTATE, "DM must have dimension >= 2 (current %" PetscInt_FMT ")", nc);
9813   {
9814     DMPlexInterpolatedFlag interpFlag;
9815 
9816     PetscCall(DMPlexIsInterpolated(dm, &interpFlag));
9817     if (interpFlag != DMPLEX_INTERPOLATED_FULL) {
9818       PetscMPIInt rank;
9819 
9820       PetscCallMPI(MPI_Comm_rank(comm, &rank));
9821       SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONGSTATE, "DM must be fully interpolated, DM on rank %d is not fully interpolated", rank);
9822     }
9823   }
9824   if (OrthQualLabel) {
9825     PetscAssertPointer(OrthQualLabel, 5);
9826     PetscCall(DMCreateLabel(dm, "Orthogonal_Quality"));
9827     PetscCall(DMGetLabel(dm, "Orthogonal_Quality", OrthQualLabel));
9828   } else {
9829     *OrthQualLabel = NULL;
9830   }
9831   PetscCall(DMPlexGetVTKCellHeight(dm, &cellHeight));
9832   PetscCall(DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd));
9833   PetscCall(DMPlexCreateCellNumbering(dm, PETSC_TRUE, &glob));
9834   PetscCall(ISLocalToGlobalMappingCreateIS(glob, &ltog));
9835   PetscCall(ISLocalToGlobalMappingSetType(ltog, ISLOCALTOGLOBALMAPPINGHASH));
9836   PetscCall(VecCreate(comm, OrthQual));
9837   PetscCall(VecSetType(*OrthQual, VECSTANDARD));
9838   PetscCall(VecSetSizes(*OrthQual, cEnd - cStart, PETSC_DETERMINE));
9839   PetscCall(VecSetLocalToGlobalMapping(*OrthQual, ltog));
9840   PetscCall(VecSetUp(*OrthQual));
9841   PetscCall(ISDestroy(&glob));
9842   PetscCall(ISLocalToGlobalMappingDestroy(&ltog));
9843   PetscCall(DMPlexGetDataFVM(dm, fv, &cellgeom, &facegeom, NULL));
9844   PetscCall(VecGetArrayRead(cellgeom, &cellGeomArr));
9845   PetscCall(VecGetArrayRead(facegeom, &faceGeomArr));
9846   PetscCall(VecGetDM(cellgeom, &dmCell));
9847   PetscCall(VecGetDM(facegeom, &dmFace));
9848   PetscCall(PetscMalloc5(cEnd - cStart, &idx, cEnd - cStart, &oqVals, nc, &ci, nc, &fi, nc, &Ai));
9849   for (cell = cStart; cell < cEnd; cellIter++, cell++) {
9850     PetscInt         cellneigh, cellneighiter = 0, adjSize = PETSC_DETERMINE;
9851     PetscInt         cellarr[2], *adj = NULL;
9852     PetscScalar     *cArr, *fArr;
9853     PetscReal        minvalc = 1.0, minvalf = 1.0;
9854     PetscFVCellGeom *cg;
9855 
9856     idx[cellIter] = cell - cStart;
9857     cellarr[0]    = cell;
9858     /* Make indexing into cellGeom easier */
9859     PetscCall(DMPlexPointLocalRead(dmCell, cell, cellGeomArr, &cg));
9860     PetscCall(DMPlexGetAdjacency_Internal(dm, cell, PETSC_TRUE, PETSC_FALSE, PETSC_FALSE, &adjSize, &adj));
9861     /* Technically 1 too big, but easier than fiddling with empty adjacency array */
9862     PetscCall(PetscCalloc2(adjSize, &cArr, adjSize, &fArr));
9863     for (cellneigh = 0; cellneigh < adjSize; cellneighiter++, cellneigh++) {
9864       PetscInt         i;
9865       const PetscInt   neigh  = adj[cellneigh];
9866       PetscReal        normci = 0, normfi = 0, normai = 0;
9867       PetscFVCellGeom *cgneigh;
9868       PetscFVFaceGeom *fg;
9869 
9870       /* Don't count ourselves in the neighbor list */
9871       if (neigh == cell) continue;
9872       PetscCall(DMPlexPointLocalRead(dmCell, neigh, cellGeomArr, &cgneigh));
9873       cellarr[1] = neigh;
9874       {
9875         PetscInt        numcovpts;
9876         const PetscInt *covpts;
9877 
9878         PetscCall(DMPlexGetMeet(dm, 2, cellarr, &numcovpts, &covpts));
9879         PetscCall(DMPlexPointLocalRead(dmFace, covpts[0], faceGeomArr, &fg));
9880         PetscCall(DMPlexRestoreMeet(dm, 2, cellarr, &numcovpts, &covpts));
9881       }
9882 
9883       /* Compute c_i, f_i and their norms */
9884       for (i = 0; i < nc; i++) {
9885         ci[i] = cgneigh->centroid[i] - cg->centroid[i];
9886         fi[i] = fg->centroid[i] - cg->centroid[i];
9887         Ai[i] = fg->normal[i];
9888         normci += PetscPowReal(ci[i], 2);
9889         normfi += PetscPowReal(fi[i], 2);
9890         normai += PetscPowReal(Ai[i], 2);
9891       }
9892       normci = PetscSqrtReal(normci);
9893       normfi = PetscSqrtReal(normfi);
9894       normai = PetscSqrtReal(normai);
9895 
9896       /* Normalize and compute for each face-cell-normal pair */
9897       for (i = 0; i < nc; i++) {
9898         ci[i] = ci[i] / normci;
9899         fi[i] = fi[i] / normfi;
9900         Ai[i] = Ai[i] / normai;
9901         /* PetscAbs because I don't know if normals are guaranteed to point out */
9902         cArr[cellneighiter] += PetscAbs(Ai[i] * ci[i]);
9903         fArr[cellneighiter] += PetscAbs(Ai[i] * fi[i]);
9904       }
9905       if (PetscRealPart(cArr[cellneighiter]) < minvalc) minvalc = PetscRealPart(cArr[cellneighiter]);
9906       if (PetscRealPart(fArr[cellneighiter]) < minvalf) minvalf = PetscRealPart(fArr[cellneighiter]);
9907     }
9908     PetscCall(PetscFree(adj));
9909     PetscCall(PetscFree2(cArr, fArr));
9910     /* Defer to cell if they're equal */
9911     oqVals[cellIter] = PetscMin(minvalf, minvalc);
9912     if (OrthQualLabel) {
9913       if (PetscRealPart(oqVals[cellIter]) <= atol) PetscCall(DMLabelSetValue(*OrthQualLabel, cell, DM_ADAPT_REFINE));
9914     }
9915   }
9916   PetscCall(VecSetValuesLocal(*OrthQual, cEnd - cStart, idx, oqVals, INSERT_VALUES));
9917   PetscCall(VecAssemblyBegin(*OrthQual));
9918   PetscCall(VecAssemblyEnd(*OrthQual));
9919   PetscCall(VecRestoreArrayRead(cellgeom, &cellGeomArr));
9920   PetscCall(VecRestoreArrayRead(facegeom, &faceGeomArr));
9921   PetscCall(PetscOptionsCreateViewer(comm, NULL, NULL, "-dm_plex_orthogonal_quality_label_view", &vwr, NULL, NULL));
9922   if (OrthQualLabel) {
9923     if (vwr) PetscCall(DMLabelView(*OrthQualLabel, vwr));
9924   }
9925   PetscCall(PetscFree5(idx, oqVals, ci, fi, Ai));
9926   PetscCall(PetscViewerDestroy(&vwr));
9927   PetscCall(VecViewFromOptions(*OrthQual, NULL, "-dm_plex_orthogonal_quality_vec_view"));
9928   PetscFunctionReturn(PETSC_SUCCESS);
9929 }
9930 
9931 /* this is here instead of DMGetOutputDM because output DM still has constraints in the local indices that affect
9932  * interpolator construction */
9933 static PetscErrorCode DMGetFullDM(DM dm, DM *odm)
9934 {
9935   PetscSection section, newSection, gsection;
9936   PetscSF      sf;
9937   PetscBool    hasConstraints, ghasConstraints;
9938 
9939   PetscFunctionBegin;
9940   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
9941   PetscAssertPointer(odm, 2);
9942   PetscCall(DMGetLocalSection(dm, &section));
9943   PetscCall(PetscSectionHasConstraints(section, &hasConstraints));
9944   PetscCallMPI(MPIU_Allreduce(&hasConstraints, &ghasConstraints, 1, MPIU_BOOL, MPI_LOR, PetscObjectComm((PetscObject)dm)));
9945   if (!ghasConstraints) {
9946     PetscCall(PetscObjectReference((PetscObject)dm));
9947     *odm = dm;
9948     PetscFunctionReturn(PETSC_SUCCESS);
9949   }
9950   PetscCall(DMClone(dm, odm));
9951   PetscCall(DMCopyFields(dm, PETSC_DETERMINE, PETSC_DETERMINE, *odm));
9952   PetscCall(DMGetLocalSection(*odm, &newSection));
9953   PetscCall(DMGetPointSF(*odm, &sf));
9954   PetscCall(PetscSectionCreateGlobalSection(newSection, sf, PETSC_TRUE, PETSC_TRUE, PETSC_FALSE, &gsection));
9955   PetscCall(DMSetGlobalSection(*odm, gsection));
9956   PetscCall(PetscSectionDestroy(&gsection));
9957   PetscFunctionReturn(PETSC_SUCCESS);
9958 }
9959 
9960 static PetscErrorCode DMCreateAffineInterpolationCorrection_Plex(DM dmc, DM dmf, Vec *shift)
9961 {
9962   DM        dmco, dmfo;
9963   Mat       interpo;
9964   Vec       rscale;
9965   Vec       cglobalo, clocal;
9966   Vec       fglobal, fglobalo, flocal;
9967   PetscBool regular;
9968 
9969   PetscFunctionBegin;
9970   PetscCall(DMGetFullDM(dmc, &dmco));
9971   PetscCall(DMGetFullDM(dmf, &dmfo));
9972   PetscCall(DMSetCoarseDM(dmfo, dmco));
9973   PetscCall(DMPlexGetRegularRefinement(dmf, &regular));
9974   PetscCall(DMPlexSetRegularRefinement(dmfo, regular));
9975   PetscCall(DMCreateInterpolation(dmco, dmfo, &interpo, &rscale));
9976   PetscCall(DMCreateGlobalVector(dmco, &cglobalo));
9977   PetscCall(DMCreateLocalVector(dmc, &clocal));
9978   PetscCall(VecSet(cglobalo, 0.));
9979   PetscCall(VecSet(clocal, 0.));
9980   PetscCall(DMCreateGlobalVector(dmf, &fglobal));
9981   PetscCall(DMCreateGlobalVector(dmfo, &fglobalo));
9982   PetscCall(DMCreateLocalVector(dmf, &flocal));
9983   PetscCall(VecSet(fglobal, 0.));
9984   PetscCall(VecSet(fglobalo, 0.));
9985   PetscCall(VecSet(flocal, 0.));
9986   PetscCall(DMPlexInsertBoundaryValues(dmc, PETSC_TRUE, clocal, 0., NULL, NULL, NULL));
9987   PetscCall(DMLocalToGlobalBegin(dmco, clocal, INSERT_VALUES, cglobalo));
9988   PetscCall(DMLocalToGlobalEnd(dmco, clocal, INSERT_VALUES, cglobalo));
9989   PetscCall(MatMult(interpo, cglobalo, fglobalo));
9990   PetscCall(DMGlobalToLocalBegin(dmfo, fglobalo, INSERT_VALUES, flocal));
9991   PetscCall(DMGlobalToLocalEnd(dmfo, fglobalo, INSERT_VALUES, flocal));
9992   PetscCall(DMLocalToGlobalBegin(dmf, flocal, INSERT_VALUES, fglobal));
9993   PetscCall(DMLocalToGlobalEnd(dmf, flocal, INSERT_VALUES, fglobal));
9994   *shift = fglobal;
9995   PetscCall(VecDestroy(&flocal));
9996   PetscCall(VecDestroy(&fglobalo));
9997   PetscCall(VecDestroy(&clocal));
9998   PetscCall(VecDestroy(&cglobalo));
9999   PetscCall(VecDestroy(&rscale));
10000   PetscCall(MatDestroy(&interpo));
10001   PetscCall(DMDestroy(&dmfo));
10002   PetscCall(DMDestroy(&dmco));
10003   PetscFunctionReturn(PETSC_SUCCESS);
10004 }
10005 
10006 PETSC_INTERN PetscErrorCode DMInterpolateSolution_Plex(DM coarse, DM fine, Mat interp, Vec coarseSol, Vec fineSol)
10007 {
10008   PetscObject shifto;
10009   Vec         shift;
10010 
10011   PetscFunctionBegin;
10012   if (!interp) {
10013     Vec rscale;
10014 
10015     PetscCall(DMCreateInterpolation(coarse, fine, &interp, &rscale));
10016     PetscCall(VecDestroy(&rscale));
10017   } else {
10018     PetscCall(PetscObjectReference((PetscObject)interp));
10019   }
10020   PetscCall(PetscObjectQuery((PetscObject)interp, "_DMInterpolateSolution_Plex_Vec", &shifto));
10021   if (!shifto) {
10022     PetscCall(DMCreateAffineInterpolationCorrection_Plex(coarse, fine, &shift));
10023     PetscCall(PetscObjectCompose((PetscObject)interp, "_DMInterpolateSolution_Plex_Vec", (PetscObject)shift));
10024     shifto = (PetscObject)shift;
10025     PetscCall(VecDestroy(&shift));
10026   }
10027   shift = (Vec)shifto;
10028   PetscCall(MatInterpolate(interp, coarseSol, fineSol));
10029   PetscCall(VecAXPY(fineSol, 1.0, shift));
10030   PetscCall(MatDestroy(&interp));
10031   PetscFunctionReturn(PETSC_SUCCESS);
10032 }
10033 
10034 /* Pointwise interpolation
10035      Just code FEM for now
10036      u^f = I u^c
10037      sum_k u^f_k phi^f_k = I sum_j u^c_j phi^c_j
10038      u^f_i = sum_j psi^f_i I phi^c_j u^c_j
10039      I_{ij} = psi^f_i phi^c_j
10040 */
10041 PetscErrorCode DMCreateInterpolation_Plex(DM dmCoarse, DM dmFine, Mat *interpolation, Vec *scaling)
10042 {
10043   PetscSection gsc, gsf;
10044   PetscInt     m, n;
10045   void        *ctx;
10046   DM           cdm;
10047   PetscBool    regular, ismatis, isRefined = dmCoarse->data == dmFine->data ? PETSC_FALSE : PETSC_TRUE;
10048 
10049   PetscFunctionBegin;
10050   PetscCall(DMGetGlobalSection(dmFine, &gsf));
10051   PetscCall(PetscSectionGetConstrainedStorageSize(gsf, &m));
10052   PetscCall(DMGetGlobalSection(dmCoarse, &gsc));
10053   PetscCall(PetscSectionGetConstrainedStorageSize(gsc, &n));
10054 
10055   PetscCall(PetscStrcmp(dmCoarse->mattype, MATIS, &ismatis));
10056   PetscCall(MatCreate(PetscObjectComm((PetscObject)dmCoarse), interpolation));
10057   PetscCall(MatSetSizes(*interpolation, m, n, PETSC_DETERMINE, PETSC_DETERMINE));
10058   PetscCall(MatSetType(*interpolation, ismatis ? MATAIJ : dmCoarse->mattype));
10059   PetscCall(DMGetApplicationContext(dmFine, &ctx));
10060 
10061   PetscCall(DMGetCoarseDM(dmFine, &cdm));
10062   PetscCall(DMPlexGetRegularRefinement(dmFine, &regular));
10063   if (!isRefined || (regular && cdm == dmCoarse)) PetscCall(DMPlexComputeInterpolatorNested(dmCoarse, dmFine, isRefined, *interpolation, ctx));
10064   else PetscCall(DMPlexComputeInterpolatorGeneral(dmCoarse, dmFine, *interpolation, ctx));
10065   PetscCall(MatViewFromOptions(*interpolation, NULL, "-interp_mat_view"));
10066   if (scaling) {
10067     /* Use naive scaling */
10068     PetscCall(DMCreateInterpolationScale(dmCoarse, dmFine, *interpolation, scaling));
10069   }
10070   PetscFunctionReturn(PETSC_SUCCESS);
10071 }
10072 
10073 PetscErrorCode DMCreateInjection_Plex(DM dmCoarse, DM dmFine, Mat *mat)
10074 {
10075   VecScatter ctx;
10076 
10077   PetscFunctionBegin;
10078   PetscCall(DMPlexComputeInjectorFEM(dmCoarse, dmFine, &ctx, NULL));
10079   PetscCall(MatCreateScatter(PetscObjectComm((PetscObject)ctx), ctx, mat));
10080   PetscCall(VecScatterDestroy(&ctx));
10081   PetscFunctionReturn(PETSC_SUCCESS);
10082 }
10083 
10084 static void g0_identity_private(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, PetscReal u_tShift, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar g0[])
10085 {
10086   const PetscInt f  = (PetscInt)PetscRealPart(constants[numConstants]);
10087   const PetscInt Nc = uOff[f + 1] - uOff[f];
10088   for (PetscInt c = 0; c < Nc; ++c) g0[c * Nc + c] = 1.0;
10089 }
10090 
10091 PetscErrorCode DMCreateMassMatrixLumped_Plex(DM dm, Vec *lmass, Vec *mass)
10092 {
10093   DM           dmc;
10094   PetscDS      ds;
10095   Vec          ones, locmass;
10096   IS           cellIS;
10097   PetscFormKey key;
10098   PetscInt     depth;
10099 
10100   PetscFunctionBegin;
10101   PetscCall(DMClone(dm, &dmc));
10102   PetscCall(DMCopyDisc(dm, dmc));
10103   PetscCall(DMGetDS(dmc, &ds));
10104   for (PetscInt f = 0; f < dmc->Nf; ++f) PetscCall(PetscDSSetJacobian(ds, f, f, g0_identity_private, NULL, NULL, NULL));
10105   if (mass) PetscCall(DMCreateGlobalVector(dm, mass));
10106   if (lmass) PetscCall(DMCreateLocalVector(dm, &locmass));
10107   else PetscCall(DMGetLocalVector(dm, &locmass));
10108   PetscCall(DMGetLocalVector(dm, &ones));
10109   PetscCall(DMPlexGetDepth(dm, &depth));
10110   PetscCall(DMGetStratumIS(dm, "depth", depth, &cellIS));
10111   PetscCall(VecSet(locmass, 0.0));
10112   PetscCall(VecSet(ones, 1.0));
10113   key.label = NULL;
10114   key.value = 0;
10115   key.field = 0;
10116   key.part  = 0;
10117   PetscCall(DMPlexComputeJacobian_Action_Internal(dmc, key, cellIS, 0.0, 0.0, ones, NULL, ones, locmass, NULL));
10118   PetscCall(ISDestroy(&cellIS));
10119   if (mass) {
10120     PetscCall(DMLocalToGlobalBegin(dm, locmass, ADD_VALUES, *mass));
10121     PetscCall(DMLocalToGlobalEnd(dm, locmass, ADD_VALUES, *mass));
10122   }
10123   PetscCall(DMRestoreLocalVector(dm, &ones));
10124   if (lmass) *lmass = locmass;
10125   else PetscCall(DMRestoreLocalVector(dm, &locmass));
10126   PetscCall(DMDestroy(&dmc));
10127   PetscFunctionReturn(PETSC_SUCCESS);
10128 }
10129 
10130 PetscErrorCode DMCreateMassMatrix_Plex(DM dmCoarse, DM dmFine, Mat *mass)
10131 {
10132   PetscSection gsc, gsf;
10133   PetscInt     m, n;
10134   void        *ctx;
10135   DM           cdm;
10136   PetscBool    regular;
10137 
10138   PetscFunctionBegin;
10139   if (dmFine == dmCoarse) {
10140     DM            dmc;
10141     PetscDS       ds;
10142     PetscWeakForm wf;
10143     Vec           u;
10144     IS            cellIS;
10145     PetscFormKey  key;
10146     PetscInt      depth;
10147 
10148     PetscCall(DMClone(dmFine, &dmc));
10149     PetscCall(DMCopyDisc(dmFine, dmc));
10150     PetscCall(DMGetDS(dmc, &ds));
10151     PetscCall(PetscDSGetWeakForm(ds, &wf));
10152     PetscCall(PetscWeakFormClear(wf));
10153     for (PetscInt f = 0; f < dmc->Nf; ++f) PetscCall(PetscDSSetJacobian(ds, f, f, g0_identity_private, NULL, NULL, NULL));
10154     PetscCall(DMCreateMatrix(dmc, mass));
10155     PetscCall(DMGetLocalVector(dmc, &u));
10156     PetscCall(DMPlexGetDepth(dmc, &depth));
10157     PetscCall(DMGetStratumIS(dmc, "depth", depth, &cellIS));
10158     PetscCall(MatZeroEntries(*mass));
10159     key.label = NULL;
10160     key.value = 0;
10161     key.field = 0;
10162     key.part  = 0;
10163     PetscCall(DMPlexComputeJacobian_Internal(dmc, key, cellIS, 0.0, 0.0, u, NULL, *mass, *mass, NULL));
10164     PetscCall(ISDestroy(&cellIS));
10165     PetscCall(DMRestoreLocalVector(dmc, &u));
10166     PetscCall(DMDestroy(&dmc));
10167   } else {
10168     PetscCall(DMGetGlobalSection(dmFine, &gsf));
10169     PetscCall(PetscSectionGetConstrainedStorageSize(gsf, &m));
10170     PetscCall(DMGetGlobalSection(dmCoarse, &gsc));
10171     PetscCall(PetscSectionGetConstrainedStorageSize(gsc, &n));
10172 
10173     PetscCall(MatCreate(PetscObjectComm((PetscObject)dmCoarse), mass));
10174     PetscCall(MatSetSizes(*mass, m, n, PETSC_DETERMINE, PETSC_DETERMINE));
10175     PetscCall(MatSetType(*mass, dmCoarse->mattype));
10176     PetscCall(DMGetApplicationContext(dmFine, &ctx));
10177 
10178     PetscCall(DMGetCoarseDM(dmFine, &cdm));
10179     PetscCall(DMPlexGetRegularRefinement(dmFine, &regular));
10180     if (regular && cdm == dmCoarse) PetscCall(DMPlexComputeMassMatrixNested(dmCoarse, dmFine, *mass, ctx));
10181     else PetscCall(DMPlexComputeMassMatrixGeneral(dmCoarse, dmFine, *mass, ctx));
10182   }
10183   PetscCall(MatViewFromOptions(*mass, NULL, "-mass_mat_view"));
10184   PetscFunctionReturn(PETSC_SUCCESS);
10185 }
10186 
10187 /*@
10188   DMPlexGetRegularRefinement - Get the flag indicating that this mesh was obtained by regular refinement from its coarse mesh
10189 
10190   Input Parameter:
10191 . dm - The `DMPLEX` object
10192 
10193   Output Parameter:
10194 . regular - The flag
10195 
10196   Level: intermediate
10197 
10198 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexSetRegularRefinement()`
10199 @*/
10200 PetscErrorCode DMPlexGetRegularRefinement(DM dm, PetscBool *regular)
10201 {
10202   PetscFunctionBegin;
10203   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
10204   PetscAssertPointer(regular, 2);
10205   *regular = ((DM_Plex *)dm->data)->regularRefinement;
10206   PetscFunctionReturn(PETSC_SUCCESS);
10207 }
10208 
10209 /*@
10210   DMPlexSetRegularRefinement - Set the flag indicating that this mesh was obtained by regular refinement from its coarse mesh
10211 
10212   Input Parameters:
10213 + dm      - The `DMPLEX` object
10214 - regular - The flag
10215 
10216   Level: intermediate
10217 
10218 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetRegularRefinement()`
10219 @*/
10220 PetscErrorCode DMPlexSetRegularRefinement(DM dm, PetscBool regular)
10221 {
10222   PetscFunctionBegin;
10223   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
10224   ((DM_Plex *)dm->data)->regularRefinement = regular;
10225   PetscFunctionReturn(PETSC_SUCCESS);
10226 }
10227 
10228 /*@
10229   DMPlexGetAnchors - Get the layout of the anchor (point-to-point) constraints.  Typically, the user will not have to
10230   call DMPlexGetAnchors() directly: if there are anchors, then `DMPlexGetAnchors()` is called during `DMGetDefaultConstraints()`.
10231 
10232   Not Collective
10233 
10234   Input Parameter:
10235 . dm - The `DMPLEX` object
10236 
10237   Output Parameters:
10238 + anchorSection - If not `NULL`, set to the section describing which points anchor the constrained points.
10239 - anchorIS      - If not `NULL`, set to the list of anchors indexed by `anchorSection`
10240 
10241   Level: intermediate
10242 
10243 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexSetAnchors()`, `DMGetDefaultConstraints()`, `DMSetDefaultConstraints()`, `IS`, `PetscSection`
10244 @*/
10245 PetscErrorCode DMPlexGetAnchors(DM dm, PetscSection *anchorSection, IS *anchorIS)
10246 {
10247   DM_Plex *plex = (DM_Plex *)dm->data;
10248 
10249   PetscFunctionBegin;
10250   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
10251   if (!plex->anchorSection && !plex->anchorIS && plex->createanchors) PetscCall((*plex->createanchors)(dm));
10252   if (anchorSection) *anchorSection = plex->anchorSection;
10253   if (anchorIS) *anchorIS = plex->anchorIS;
10254   PetscFunctionReturn(PETSC_SUCCESS);
10255 }
10256 
10257 /*@
10258   DMPlexSetAnchors - Set the layout of the local anchor (point-to-point) constraints.
10259 
10260   Collective
10261 
10262   Input Parameters:
10263 + dm            - The `DMPLEX` object
10264 . anchorSection - The section that describes the mapping from constrained points to the anchor points listed in anchorIS.
10265                   Must have a local communicator (`PETSC_COMM_SELF` or derivative).
10266 - anchorIS      - The list of all anchor points.  Must have a local communicator (`PETSC_COMM_SELF` or derivative).
10267 
10268   Level: intermediate
10269 
10270   Notes:
10271   Unlike boundary conditions, when a point's degrees of freedom in a section are constrained to
10272   an outside value, the anchor constraints set a point's degrees of freedom to be a linear
10273   combination of other points' degrees of freedom.
10274 
10275   After specifying the layout of constraints with `DMPlexSetAnchors()`, one specifies the constraints by calling
10276   `DMGetDefaultConstraints()` and filling in the entries in the constraint matrix.
10277 
10278   The reference counts of `anchorSection` and `anchorIS` are incremented.
10279 
10280 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetAnchors()`, `DMGetDefaultConstraints()`, `DMSetDefaultConstraints()`
10281 @*/
10282 PetscErrorCode DMPlexSetAnchors(DM dm, PetscSection anchorSection, IS anchorIS)
10283 {
10284   DM_Plex    *plex = (DM_Plex *)dm->data;
10285   PetscMPIInt result;
10286 
10287   PetscFunctionBegin;
10288   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
10289   if (anchorSection) {
10290     PetscValidHeaderSpecific(anchorSection, PETSC_SECTION_CLASSID, 2);
10291     PetscCallMPI(MPI_Comm_compare(PETSC_COMM_SELF, PetscObjectComm((PetscObject)anchorSection), &result));
10292     PetscCheck(result == MPI_CONGRUENT || result == MPI_IDENT, PETSC_COMM_SELF, PETSC_ERR_ARG_NOTSAMECOMM, "anchor section must have local communicator");
10293   }
10294   if (anchorIS) {
10295     PetscValidHeaderSpecific(anchorIS, IS_CLASSID, 3);
10296     PetscCallMPI(MPI_Comm_compare(PETSC_COMM_SELF, PetscObjectComm((PetscObject)anchorIS), &result));
10297     PetscCheck(result == MPI_CONGRUENT || result == MPI_IDENT, PETSC_COMM_SELF, PETSC_ERR_ARG_NOTSAMECOMM, "anchor IS must have local communicator");
10298   }
10299 
10300   PetscCall(PetscObjectReference((PetscObject)anchorSection));
10301   PetscCall(PetscSectionDestroy(&plex->anchorSection));
10302   plex->anchorSection = anchorSection;
10303 
10304   PetscCall(PetscObjectReference((PetscObject)anchorIS));
10305   PetscCall(ISDestroy(&plex->anchorIS));
10306   plex->anchorIS = anchorIS;
10307 
10308   if (PetscUnlikelyDebug(anchorIS && anchorSection)) {
10309     PetscInt        size, a, pStart, pEnd;
10310     const PetscInt *anchors;
10311 
10312     PetscCall(PetscSectionGetChart(anchorSection, &pStart, &pEnd));
10313     PetscCall(ISGetLocalSize(anchorIS, &size));
10314     PetscCall(ISGetIndices(anchorIS, &anchors));
10315     for (a = 0; a < size; a++) {
10316       PetscInt p;
10317 
10318       p = anchors[a];
10319       if (p >= pStart && p < pEnd) {
10320         PetscInt dof;
10321 
10322         PetscCall(PetscSectionGetDof(anchorSection, p, &dof));
10323         if (dof) {
10324           PetscCall(ISRestoreIndices(anchorIS, &anchors));
10325           SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "Point %" PetscInt_FMT " cannot be constrained and an anchor", p);
10326         }
10327       }
10328     }
10329     PetscCall(ISRestoreIndices(anchorIS, &anchors));
10330   }
10331   /* reset the generic constraints */
10332   PetscCall(DMSetDefaultConstraints(dm, NULL, NULL, NULL));
10333   PetscFunctionReturn(PETSC_SUCCESS);
10334 }
10335 
10336 static PetscErrorCode DMPlexCreateConstraintSection_Anchors(DM dm, PetscSection section, PetscSection *cSec)
10337 {
10338   PetscSection anchorSection;
10339   PetscInt     pStart, pEnd, sStart, sEnd, p, dof, numFields, f;
10340 
10341   PetscFunctionBegin;
10342   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
10343   PetscCall(DMPlexGetAnchors(dm, &anchorSection, NULL));
10344   PetscCall(PetscSectionCreate(PETSC_COMM_SELF, cSec));
10345   PetscCall(PetscSectionGetNumFields(section, &numFields));
10346   if (numFields) {
10347     PetscInt f;
10348     PetscCall(PetscSectionSetNumFields(*cSec, numFields));
10349 
10350     for (f = 0; f < numFields; f++) {
10351       PetscInt numComp;
10352 
10353       PetscCall(PetscSectionGetFieldComponents(section, f, &numComp));
10354       PetscCall(PetscSectionSetFieldComponents(*cSec, f, numComp));
10355     }
10356   }
10357   PetscCall(PetscSectionGetChart(anchorSection, &pStart, &pEnd));
10358   PetscCall(PetscSectionGetChart(section, &sStart, &sEnd));
10359   pStart = PetscMax(pStart, sStart);
10360   pEnd   = PetscMin(pEnd, sEnd);
10361   pEnd   = PetscMax(pStart, pEnd);
10362   PetscCall(PetscSectionSetChart(*cSec, pStart, pEnd));
10363   for (p = pStart; p < pEnd; p++) {
10364     PetscCall(PetscSectionGetDof(anchorSection, p, &dof));
10365     if (dof) {
10366       PetscCall(PetscSectionGetDof(section, p, &dof));
10367       PetscCall(PetscSectionSetDof(*cSec, p, dof));
10368       for (f = 0; f < numFields; f++) {
10369         PetscCall(PetscSectionGetFieldDof(section, p, f, &dof));
10370         PetscCall(PetscSectionSetFieldDof(*cSec, p, f, dof));
10371       }
10372     }
10373   }
10374   PetscCall(PetscSectionSetUp(*cSec));
10375   PetscCall(PetscObjectSetName((PetscObject)*cSec, "Constraint Section"));
10376   PetscFunctionReturn(PETSC_SUCCESS);
10377 }
10378 
10379 static PetscErrorCode DMPlexCreateConstraintMatrix_Anchors(DM dm, PetscSection section, PetscSection cSec, Mat *cMat)
10380 {
10381   PetscSection    aSec;
10382   PetscInt        pStart, pEnd, p, sStart, sEnd, dof, aDof, aOff, off, nnz, annz, m, n, q, a, offset, *i, *j;
10383   const PetscInt *anchors;
10384   PetscInt        numFields, f;
10385   IS              aIS;
10386   MatType         mtype;
10387   PetscBool       iscuda, iskokkos;
10388 
10389   PetscFunctionBegin;
10390   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
10391   PetscCall(PetscSectionGetStorageSize(cSec, &m));
10392   PetscCall(PetscSectionGetStorageSize(section, &n));
10393   PetscCall(MatCreate(PETSC_COMM_SELF, cMat));
10394   PetscCall(MatSetSizes(*cMat, m, n, m, n));
10395   PetscCall(PetscStrcmp(dm->mattype, MATSEQAIJCUSPARSE, &iscuda));
10396   if (!iscuda) PetscCall(PetscStrcmp(dm->mattype, MATMPIAIJCUSPARSE, &iscuda));
10397   PetscCall(PetscStrcmp(dm->mattype, MATSEQAIJKOKKOS, &iskokkos));
10398   if (!iskokkos) PetscCall(PetscStrcmp(dm->mattype, MATMPIAIJKOKKOS, &iskokkos));
10399   if (iscuda) mtype = MATSEQAIJCUSPARSE;
10400   else if (iskokkos) mtype = MATSEQAIJKOKKOS;
10401   else mtype = MATSEQAIJ;
10402   PetscCall(MatSetType(*cMat, mtype));
10403   PetscCall(DMPlexGetAnchors(dm, &aSec, &aIS));
10404   PetscCall(ISGetIndices(aIS, &anchors));
10405   /* cSec will be a subset of aSec and section */
10406   PetscCall(PetscSectionGetChart(cSec, &pStart, &pEnd));
10407   PetscCall(PetscSectionGetChart(section, &sStart, &sEnd));
10408   PetscCall(PetscMalloc1(m + 1, &i));
10409   i[0] = 0;
10410   PetscCall(PetscSectionGetNumFields(section, &numFields));
10411   for (p = pStart; p < pEnd; p++) {
10412     PetscInt rDof, rOff, r;
10413 
10414     PetscCall(PetscSectionGetDof(aSec, p, &rDof));
10415     if (!rDof) continue;
10416     PetscCall(PetscSectionGetOffset(aSec, p, &rOff));
10417     if (numFields) {
10418       for (f = 0; f < numFields; f++) {
10419         annz = 0;
10420         for (r = 0; r < rDof; r++) {
10421           a = anchors[rOff + r];
10422           if (a < sStart || a >= sEnd) continue;
10423           PetscCall(PetscSectionGetFieldDof(section, a, f, &aDof));
10424           annz += aDof;
10425         }
10426         PetscCall(PetscSectionGetFieldDof(cSec, p, f, &dof));
10427         PetscCall(PetscSectionGetFieldOffset(cSec, p, f, &off));
10428         for (q = 0; q < dof; q++) i[off + q + 1] = i[off + q] + annz;
10429       }
10430     } else {
10431       annz = 0;
10432       PetscCall(PetscSectionGetDof(cSec, p, &dof));
10433       for (q = 0; q < dof; q++) {
10434         a = anchors[rOff + q];
10435         if (a < sStart || a >= sEnd) continue;
10436         PetscCall(PetscSectionGetDof(section, a, &aDof));
10437         annz += aDof;
10438       }
10439       PetscCall(PetscSectionGetDof(cSec, p, &dof));
10440       PetscCall(PetscSectionGetOffset(cSec, p, &off));
10441       for (q = 0; q < dof; q++) i[off + q + 1] = i[off + q] + annz;
10442     }
10443   }
10444   nnz = i[m];
10445   PetscCall(PetscMalloc1(nnz, &j));
10446   offset = 0;
10447   for (p = pStart; p < pEnd; p++) {
10448     if (numFields) {
10449       for (f = 0; f < numFields; f++) {
10450         PetscCall(PetscSectionGetFieldDof(cSec, p, f, &dof));
10451         for (q = 0; q < dof; q++) {
10452           PetscInt rDof, rOff, r;
10453           PetscCall(PetscSectionGetDof(aSec, p, &rDof));
10454           PetscCall(PetscSectionGetOffset(aSec, p, &rOff));
10455           for (r = 0; r < rDof; r++) {
10456             PetscInt s;
10457 
10458             a = anchors[rOff + r];
10459             if (a < sStart || a >= sEnd) continue;
10460             PetscCall(PetscSectionGetFieldDof(section, a, f, &aDof));
10461             PetscCall(PetscSectionGetFieldOffset(section, a, f, &aOff));
10462             for (s = 0; s < aDof; s++) j[offset++] = aOff + s;
10463           }
10464         }
10465       }
10466     } else {
10467       PetscCall(PetscSectionGetDof(cSec, p, &dof));
10468       for (q = 0; q < dof; q++) {
10469         PetscInt rDof, rOff, r;
10470         PetscCall(PetscSectionGetDof(aSec, p, &rDof));
10471         PetscCall(PetscSectionGetOffset(aSec, p, &rOff));
10472         for (r = 0; r < rDof; r++) {
10473           PetscInt s;
10474 
10475           a = anchors[rOff + r];
10476           if (a < sStart || a >= sEnd) continue;
10477           PetscCall(PetscSectionGetDof(section, a, &aDof));
10478           PetscCall(PetscSectionGetOffset(section, a, &aOff));
10479           for (s = 0; s < aDof; s++) j[offset++] = aOff + s;
10480         }
10481       }
10482     }
10483   }
10484   PetscCall(MatSeqAIJSetPreallocationCSR(*cMat, i, j, NULL));
10485   PetscCall(PetscFree(i));
10486   PetscCall(PetscFree(j));
10487   PetscCall(ISRestoreIndices(aIS, &anchors));
10488   PetscFunctionReturn(PETSC_SUCCESS);
10489 }
10490 
10491 PetscErrorCode DMCreateDefaultConstraints_Plex(DM dm)
10492 {
10493   DM_Plex     *plex = (DM_Plex *)dm->data;
10494   PetscSection anchorSection, section, cSec;
10495   Mat          cMat;
10496 
10497   PetscFunctionBegin;
10498   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
10499   PetscCall(DMPlexGetAnchors(dm, &anchorSection, NULL));
10500   if (anchorSection) {
10501     PetscInt Nf;
10502 
10503     PetscCall(DMGetLocalSection(dm, &section));
10504     PetscCall(DMPlexCreateConstraintSection_Anchors(dm, section, &cSec));
10505     PetscCall(DMPlexCreateConstraintMatrix_Anchors(dm, section, cSec, &cMat));
10506     PetscCall(DMGetNumFields(dm, &Nf));
10507     if (Nf && plex->computeanchormatrix) PetscCall((*plex->computeanchormatrix)(dm, section, cSec, cMat));
10508     PetscCall(DMSetDefaultConstraints(dm, cSec, cMat, NULL));
10509     PetscCall(PetscSectionDestroy(&cSec));
10510     PetscCall(MatDestroy(&cMat));
10511   }
10512   PetscFunctionReturn(PETSC_SUCCESS);
10513 }
10514 
10515 PetscErrorCode DMCreateSubDomainDM_Plex(DM dm, DMLabel label, PetscInt value, IS *is, DM *subdm)
10516 {
10517   IS           subis;
10518   PetscSection section, subsection;
10519 
10520   PetscFunctionBegin;
10521   PetscCall(DMGetLocalSection(dm, &section));
10522   PetscCheck(section, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Must set default section for DM before splitting subdomain");
10523   PetscCheck(subdm, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Must set output subDM for splitting subdomain");
10524   /* Create subdomain */
10525   PetscCall(DMPlexFilter(dm, label, value, PETSC_FALSE, PETSC_FALSE, NULL, subdm));
10526   /* Create submodel */
10527   PetscCall(DMPlexGetSubpointIS(*subdm, &subis));
10528   PetscCall(PetscSectionCreateSubmeshSection(section, subis, &subsection));
10529   PetscCall(DMSetLocalSection(*subdm, subsection));
10530   PetscCall(PetscSectionDestroy(&subsection));
10531   PetscCall(DMCopyDisc(dm, *subdm));
10532   /* Create map from submodel to global model */
10533   if (is) {
10534     PetscSection    sectionGlobal, subsectionGlobal;
10535     IS              spIS;
10536     const PetscInt *spmap;
10537     PetscInt       *subIndices;
10538     PetscInt        subSize = 0, subOff = 0, pStart, pEnd, p;
10539     PetscInt        Nf, f, bs = -1, bsLocal[2], bsMinMax[2];
10540 
10541     PetscCall(DMPlexGetSubpointIS(*subdm, &spIS));
10542     PetscCall(ISGetIndices(spIS, &spmap));
10543     PetscCall(PetscSectionGetNumFields(section, &Nf));
10544     PetscCall(DMGetGlobalSection(dm, &sectionGlobal));
10545     PetscCall(DMGetGlobalSection(*subdm, &subsectionGlobal));
10546     PetscCall(PetscSectionGetChart(subsection, &pStart, &pEnd));
10547     for (p = pStart; p < pEnd; ++p) {
10548       PetscInt gdof, pSubSize = 0;
10549 
10550       PetscCall(PetscSectionGetDof(sectionGlobal, p, &gdof));
10551       if (gdof > 0) {
10552         for (f = 0; f < Nf; ++f) {
10553           PetscInt fdof, fcdof;
10554 
10555           PetscCall(PetscSectionGetFieldDof(subsection, p, f, &fdof));
10556           PetscCall(PetscSectionGetFieldConstraintDof(subsection, p, f, &fcdof));
10557           pSubSize += fdof - fcdof;
10558         }
10559         subSize += pSubSize;
10560         if (pSubSize) {
10561           if (bs < 0) {
10562             bs = pSubSize;
10563           } else if (bs != pSubSize) {
10564             /* Layout does not admit a pointwise block size */
10565             bs = 1;
10566           }
10567         }
10568       }
10569     }
10570     /* Must have same blocksize on all procs (some might have no points) */
10571     bsLocal[0] = bs < 0 ? PETSC_INT_MAX : bs;
10572     bsLocal[1] = bs;
10573     PetscCall(PetscGlobalMinMaxInt(PetscObjectComm((PetscObject)dm), bsLocal, bsMinMax));
10574     if (bsMinMax[0] != bsMinMax[1]) {
10575       bs = 1;
10576     } else {
10577       bs = bsMinMax[0];
10578     }
10579     PetscCall(PetscMalloc1(subSize, &subIndices));
10580     for (p = pStart; p < pEnd; ++p) {
10581       PetscInt gdof, goff;
10582 
10583       PetscCall(PetscSectionGetDof(subsectionGlobal, p, &gdof));
10584       if (gdof > 0) {
10585         const PetscInt point = spmap[p];
10586 
10587         PetscCall(PetscSectionGetOffset(sectionGlobal, point, &goff));
10588         for (f = 0; f < Nf; ++f) {
10589           PetscInt fdof, fcdof, fc, f2, poff = 0;
10590 
10591           /* Can get rid of this loop by storing field information in the global section */
10592           for (f2 = 0; f2 < f; ++f2) {
10593             PetscCall(PetscSectionGetFieldDof(section, p, f2, &fdof));
10594             PetscCall(PetscSectionGetFieldConstraintDof(section, p, f2, &fcdof));
10595             poff += fdof - fcdof;
10596           }
10597           PetscCall(PetscSectionGetFieldDof(section, p, f, &fdof));
10598           PetscCall(PetscSectionGetFieldConstraintDof(section, p, f, &fcdof));
10599           for (fc = 0; fc < fdof - fcdof; ++fc, ++subOff) subIndices[subOff] = goff + poff + fc;
10600         }
10601       }
10602     }
10603     PetscCall(ISRestoreIndices(spIS, &spmap));
10604     PetscCall(ISCreateGeneral(PetscObjectComm((PetscObject)dm), subSize, subIndices, PETSC_OWN_POINTER, is));
10605     if (bs > 1) {
10606       /* We need to check that the block size does not come from non-contiguous fields */
10607       PetscInt i, j, set = 1;
10608       for (i = 0; i < subSize; i += bs) {
10609         for (j = 0; j < bs; ++j) {
10610           if (subIndices[i + j] != subIndices[i] + j) {
10611             set = 0;
10612             break;
10613           }
10614         }
10615       }
10616       if (set) PetscCall(ISSetBlockSize(*is, bs));
10617     }
10618     /* Attach nullspace */
10619     for (f = 0; f < Nf; ++f) {
10620       (*subdm)->nullspaceConstructors[f] = dm->nullspaceConstructors[f];
10621       if ((*subdm)->nullspaceConstructors[f]) break;
10622     }
10623     if (f < Nf) {
10624       MatNullSpace nullSpace;
10625       PetscCall((*(*subdm)->nullspaceConstructors[f])(*subdm, f, f, &nullSpace));
10626 
10627       PetscCall(PetscObjectCompose((PetscObject)*is, "nullspace", (PetscObject)nullSpace));
10628       PetscCall(MatNullSpaceDestroy(&nullSpace));
10629     }
10630   }
10631   PetscFunctionReturn(PETSC_SUCCESS);
10632 }
10633 
10634 /*@
10635   DMPlexMonitorThroughput - Report the cell throughput of FE integration
10636 
10637   Input Parameters:
10638 + dm    - The `DM`
10639 - dummy - unused argument
10640 
10641   Options Database Key:
10642 . -dm_plex_monitor_throughput - Activate the monitor
10643 
10644   Level: developer
10645 
10646 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMSetFromOptions()`, `DMPlexCreate()`
10647 @*/
10648 PetscErrorCode DMPlexMonitorThroughput(DM dm, void *dummy)
10649 {
10650   PetscLogHandler default_handler;
10651 
10652   PetscFunctionBegin;
10653   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
10654   PetscCall(PetscLogGetDefaultHandler(&default_handler));
10655   if (default_handler) {
10656     PetscLogEvent      event;
10657     PetscEventPerfInfo eventInfo;
10658     PetscReal          cellRate, flopRate;
10659     PetscInt           cStart, cEnd, Nf, N;
10660     const char        *name;
10661 
10662     PetscCall(PetscObjectGetName((PetscObject)dm, &name));
10663     PetscCall(DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd));
10664     PetscCall(DMGetNumFields(dm, &Nf));
10665     PetscCall(PetscLogEventGetId("DMPlexResidualFE", &event));
10666     PetscCall(PetscLogEventGetPerfInfo(PETSC_DEFAULT, event, &eventInfo));
10667     N        = (cEnd - cStart) * Nf * eventInfo.count;
10668     flopRate = eventInfo.flops / eventInfo.time;
10669     cellRate = N / eventInfo.time;
10670     PetscCall(PetscPrintf(PetscObjectComm((PetscObject)dm), "DM (%s) FE Residual Integration: %" PetscInt_FMT " integrals %d reps\n  Cell rate: %.2g/s flop rate: %.2g MF/s\n", name ? name : "unknown", N, eventInfo.count, (double)cellRate, (double)(flopRate / 1.e6)));
10671   } else {
10672     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Plex Throughput Monitor is not supported if logging is turned off or the default log handler is not running. Reconfigure using --with-log and run with -log_view.");
10673   }
10674   PetscFunctionReturn(PETSC_SUCCESS);
10675 }
10676