xref: /petsc/src/dm/impls/plex/plexsubmesh.c (revision 375d23e45bc7f27f90f326da17aaea364b3bd8d3)
1 #include <petsc/private/dmpleximpl.h>  /*I      "petscdmplex.h"    I*/
2 #include <petsc/private/dmlabelimpl.h> /*I      "petscdmlabel.h"   I*/
3 #include <petscsf.h>
4 
5 static PetscErrorCode DMPlexCellIsHybrid_Internal(DM dm, PetscInt p, PetscBool *isHybrid)
6 {
7   DMPolytopeType ct;
8 
9   PetscFunctionBegin;
10   PetscCall(DMPlexGetCellType(dm, p, &ct));
11   switch (ct) {
12   case DM_POLYTOPE_POINT_PRISM_TENSOR:
13   case DM_POLYTOPE_SEG_PRISM_TENSOR:
14   case DM_POLYTOPE_TRI_PRISM_TENSOR:
15   case DM_POLYTOPE_QUAD_PRISM_TENSOR:
16     *isHybrid = PETSC_TRUE;
17     break;
18   default:
19     *isHybrid = PETSC_FALSE;
20     break;
21   }
22   PetscFunctionReturn(PETSC_SUCCESS);
23 }
24 
25 static PetscErrorCode DMPlexGetTensorPrismBounds_Internal(DM dm, PetscInt dim, PetscInt *cStart, PetscInt *cEnd)
26 {
27   DMLabel ctLabel;
28 
29   PetscFunctionBegin;
30   if (cStart) *cStart = -1;
31   if (cEnd) *cEnd = -1;
32   PetscCall(DMPlexGetCellTypeLabel(dm, &ctLabel));
33   switch (dim) {
34   case 1:
35     PetscCall(DMLabelGetStratumBounds(ctLabel, DM_POLYTOPE_POINT_PRISM_TENSOR, cStart, cEnd));
36     break;
37   case 2:
38     PetscCall(DMLabelGetStratumBounds(ctLabel, DM_POLYTOPE_SEG_PRISM_TENSOR, cStart, cEnd));
39     break;
40   case 3:
41     PetscCall(DMLabelGetStratumBounds(ctLabel, DM_POLYTOPE_TRI_PRISM_TENSOR, cStart, cEnd));
42     if (*cStart < 0) PetscCall(DMLabelGetStratumBounds(ctLabel, DM_POLYTOPE_QUAD_PRISM_TENSOR, cStart, cEnd));
43     break;
44   default:
45     PetscFunctionReturn(PETSC_SUCCESS);
46   }
47   PetscFunctionReturn(PETSC_SUCCESS);
48 }
49 
50 PetscErrorCode DMPlexMarkBoundaryFaces_Internal(DM dm, PetscInt val, PetscInt cellHeight, DMLabel label, PetscBool missing_only)
51 {
52   PetscInt           depth, pStart, pEnd, fStart, fEnd, f, supportSize, nroots = -1, nleaves = -1, defval;
53   PetscSF            sf;
54   const PetscSFNode *iremote = NULL;
55   const PetscInt    *ilocal  = NULL;
56   PetscInt          *leafData;
57 
58   PetscFunctionBegin;
59   PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
60   PetscCall(DMPlexGetDepth(dm, &depth));
61   if (depth >= cellHeight + 1) {
62     PetscCall(DMPlexGetHeightStratum(dm, cellHeight + 1, &fStart, &fEnd));
63   } else {
64     /* Note DMPlexGetHeightStratum() returns fStart, fEnd = pStart, pEnd */
65     /* if height > depth, which is not what we want here.                */
66     fStart = 0;
67     fEnd   = 0;
68   }
69   PetscCall(DMLabelGetDefaultValue(label, &defval));
70   PetscCall(PetscCalloc1(pEnd - pStart, &leafData));
71   leafData -= pStart;
72   PetscCall(DMGetPointSF(dm, &sf));
73   if (sf) PetscCall(PetscSFGetGraph(sf, &nroots, &nleaves, &ilocal, &iremote));
74   if (sf && nroots >= 0) {
75     PetscInt        cStart, cEnd, c, i;
76     PetscInt       *rootData, *rootData1, *cellOwners, hasTwoSupportCells = -2;
77     const PetscInt *support;
78     PetscMPIInt     rank;
79 
80     PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
81     PetscCall(DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd));
82     PetscCall(PetscCalloc3(pEnd - pStart, &rootData, pEnd - pStart, &rootData1, cEnd - cStart, &cellOwners));
83     rootData -= pStart;
84     rootData1 -= pStart;
85     for (c = cStart; c < cEnd; ++c) cellOwners[c - cStart] = (PetscInt)rank;
86     for (i = 0; i < nleaves; ++i) {
87       c = ilocal ? ilocal[i] : i;
88       if (c >= cStart && c < cEnd) cellOwners[c - cStart] = iremote[i].rank;
89     }
90     for (f = fStart; f < fEnd; ++f) {
91       PetscCall(DMPlexGetSupportSize(dm, f, &supportSize));
92       if (supportSize == 1) {
93         PetscCall(DMPlexGetSupport(dm, f, &support));
94         leafData[f] = cellOwners[support[0] - cStart];
95       } else {
96         /* TODO: When using DMForest, we could have a parent facet (a coarse facet)     */
97         /*       supportSize of which alone does not tell us if it is an interior       */
98         /*       facet or an exterior facet. Those facets can be identified by checking */
99         /*       if they are in the parent tree. We should probably skip those parent   */
100         /*       facets here, which will allow for including the following check, and   */
101         /*       see if they are exterior facets or not afterwards by checking if the   */
102         /*       children are exterior or not.                                          */
103         /* PetscCheck(supportSize == 2, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid facet support size (%" PetscInt_FMT ") on facet (%" PetscInt_FMT ")", supportSize, f); */
104         leafData[f] = hasTwoSupportCells; /* some negative PetscInt */
105       }
106       rootData[f]  = leafData[f];
107       rootData1[f] = leafData[f];
108     }
109     PetscCall(PetscSFReduceBegin(sf, MPIU_INT, leafData, rootData, MPI_MIN));
110     PetscCall(PetscSFReduceBegin(sf, MPIU_INT, leafData, rootData1, MPI_MAX));
111     PetscCall(PetscSFReduceEnd(sf, MPIU_INT, leafData, rootData, MPI_MIN));
112     PetscCall(PetscSFReduceEnd(sf, MPIU_INT, leafData, rootData1, MPI_MAX));
113     for (f = fStart; f < fEnd; ++f) {
114       /* Store global support size of f.                                                    */
115       /* Facet f is an interior facet if and only if one of the following two is satisfied: */
116       /* 1. supportSize is 2 on some rank.                                                  */
117       /* 2. supportSize is 1 on any rank that can see f, but f is on a partition boundary;  */
118       /*    i.e., rootData[f] < rootData1[f].                                               */
119       rootData[f] = (rootData[f] == hasTwoSupportCells || (rootData[f] < rootData1[f])) ? 2 : 1;
120       leafData[f] = rootData[f];
121     }
122     PetscCall(PetscSFBcastBegin(sf, MPIU_INT, rootData, leafData, MPI_REPLACE));
123     PetscCall(PetscSFBcastEnd(sf, MPIU_INT, rootData, leafData, MPI_REPLACE));
124     rootData += pStart;
125     rootData1 += pStart;
126     PetscCall(PetscFree3(rootData, rootData1, cellOwners));
127   } else {
128     for (f = fStart; f < fEnd; ++f) {
129       PetscCall(DMPlexGetSupportSize(dm, f, &supportSize));
130       leafData[f] = supportSize;
131     }
132   }
133   for (f = fStart; f < fEnd; ++f) {
134     if (leafData[f] == 1) {
135       if (val < 0) {
136         PetscInt *closure = NULL;
137         PetscInt  clSize, cl, cval;
138 
139         PetscAssert(!missing_only, PETSC_COMM_SELF, PETSC_ERR_SUP, "Not implemented");
140         PetscCall(DMPlexGetTransitiveClosure(dm, f, PETSC_TRUE, &clSize, &closure));
141         for (cl = 0; cl < clSize * 2; cl += 2) {
142           PetscCall(DMLabelGetValue(label, closure[cl], &cval));
143           if (cval < 0) continue;
144           PetscCall(DMLabelSetValue(label, f, cval));
145           break;
146         }
147         if (cl == clSize * 2) PetscCall(DMLabelSetValue(label, f, 1));
148         PetscCall(DMPlexRestoreTransitiveClosure(dm, f, PETSC_TRUE, &clSize, &closure));
149       } else {
150         if (missing_only) {
151           PetscInt fval;
152           PetscCall(DMLabelGetValue(label, f, &fval));
153           if (fval != defval) PetscCall(DMLabelClearValue(label, f, fval));
154           else PetscCall(DMLabelSetValue(label, f, val));
155         } else {
156           PetscCall(DMLabelSetValue(label, f, val));
157         }
158       }
159     } else {
160       /* TODO: See the above comment on DMForest */
161       /* PetscCheck(leafData[f] == 2, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid facet support size (%" PetscInt_FMT ") on facet (%" PetscInt_FMT ")", leafData[f], f); */
162     }
163   }
164   leafData += pStart;
165   PetscCall(PetscFree(leafData));
166   PetscFunctionReturn(PETSC_SUCCESS);
167 }
168 
169 /*@
170   DMPlexMarkBoundaryFaces - Mark all faces on the boundary
171 
172   Collective
173 
174   Input Parameters:
175 + dm  - The original `DM`
176 - val - The marker value, or `PETSC_DETERMINE` to use some value in the closure (or 1 if none are found)
177 
178   Output Parameter:
179 . label - The `DMLabel` marking boundary faces with the given value
180 
181   Level: developer
182 
183   Note:
184   This function will use the point `PetscSF` from the input `DM` and the ownership of the support cells to exclude points on the partition boundary from being marked. If you also wish to mark the partition boundary, you can use `DMSetPointSF()` to temporarily set it to `NULL`, and then reset it to the original object after the call.
185 
186   In DMForest there can be facets support sizes of which alone can not determine whether they are on the boundary. Currently, this function is not guaranteed to produce the correct result in such case.
187 
188 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMLabelCreate()`, `DMCreateLabel()`
189 @*/
190 PetscErrorCode DMPlexMarkBoundaryFaces(DM dm, PetscInt val, DMLabel label)
191 {
192   DMPlexInterpolatedFlag flg;
193 
194   PetscFunctionBegin;
195   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
196   PetscCall(DMPlexIsInterpolated(dm, &flg));
197   PetscCheck(flg == DMPLEX_INTERPOLATED_FULL, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM is not fully interpolated on this rank");
198   PetscCall(DMPlexMarkBoundaryFaces_Internal(dm, val, 0, label, PETSC_FALSE));
199   PetscFunctionReturn(PETSC_SUCCESS);
200 }
201 
202 static PetscErrorCode DMPlexLabelComplete_Internal(DM dm, DMLabel label, PetscBool completeCells)
203 {
204   IS              valueIS;
205   PetscSF         sfPoint;
206   const PetscInt *values;
207   PetscInt        numValues, v, cStart, cEnd, nroots;
208 
209   PetscFunctionBegin;
210   PetscCall(DMLabelGetNumValues(label, &numValues));
211   PetscCall(DMLabelGetValueIS(label, &valueIS));
212   PetscCall(DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd));
213   PetscCall(ISGetIndices(valueIS, &values));
214   for (v = 0; v < numValues; ++v) {
215     IS              pointIS;
216     const PetscInt *points;
217     PetscInt        numPoints, p;
218 
219     PetscCall(DMLabelGetStratumSize(label, values[v], &numPoints));
220     PetscCall(DMLabelGetStratumIS(label, values[v], &pointIS));
221     PetscCall(ISGetIndices(pointIS, &points));
222     for (p = 0; p < numPoints; ++p) {
223       PetscInt  q       = points[p];
224       PetscInt *closure = NULL;
225       PetscInt  closureSize, c;
226 
227       if (cStart <= q && q < cEnd && !completeCells) { /* skip cells */
228         continue;
229       }
230       PetscCall(DMPlexGetTransitiveClosure(dm, q, PETSC_TRUE, &closureSize, &closure));
231       for (c = 0; c < closureSize * 2; c += 2) PetscCall(DMLabelSetValue(label, closure[c], values[v]));
232       PetscCall(DMPlexRestoreTransitiveClosure(dm, q, PETSC_TRUE, &closureSize, &closure));
233     }
234     PetscCall(ISRestoreIndices(pointIS, &points));
235     PetscCall(ISDestroy(&pointIS));
236   }
237   PetscCall(ISRestoreIndices(valueIS, &values));
238   PetscCall(ISDestroy(&valueIS));
239   PetscCall(DMGetPointSF(dm, &sfPoint));
240   PetscCall(PetscSFGetGraph(sfPoint, &nroots, NULL, NULL, NULL));
241   if (nroots >= 0) {
242     DMLabel         lblRoots, lblLeaves;
243     IS              valueIS, pointIS;
244     const PetscInt *values;
245     PetscInt        numValues, v;
246 
247     /* Pull point contributions from remote leaves into local roots */
248     PetscCall(DMLabelGather(label, sfPoint, &lblLeaves));
249     PetscCall(DMLabelGetValueIS(lblLeaves, &valueIS));
250     PetscCall(ISGetLocalSize(valueIS, &numValues));
251     PetscCall(ISGetIndices(valueIS, &values));
252     for (v = 0; v < numValues; ++v) {
253       const PetscInt value = values[v];
254 
255       PetscCall(DMLabelGetStratumIS(lblLeaves, value, &pointIS));
256       PetscCall(DMLabelInsertIS(label, pointIS, value));
257       PetscCall(ISDestroy(&pointIS));
258     }
259     PetscCall(ISRestoreIndices(valueIS, &values));
260     PetscCall(ISDestroy(&valueIS));
261     PetscCall(DMLabelDestroy(&lblLeaves));
262     /* Push point contributions from roots into remote leaves */
263     PetscCall(DMLabelDistribute(label, sfPoint, &lblRoots));
264     PetscCall(DMLabelGetValueIS(lblRoots, &valueIS));
265     PetscCall(ISGetLocalSize(valueIS, &numValues));
266     PetscCall(ISGetIndices(valueIS, &values));
267     for (v = 0; v < numValues; ++v) {
268       const PetscInt value = values[v];
269 
270       PetscCall(DMLabelGetStratumIS(lblRoots, value, &pointIS));
271       PetscCall(DMLabelInsertIS(label, pointIS, value));
272       PetscCall(ISDestroy(&pointIS));
273     }
274     PetscCall(ISRestoreIndices(valueIS, &values));
275     PetscCall(ISDestroy(&valueIS));
276     PetscCall(DMLabelDestroy(&lblRoots));
277   }
278   PetscFunctionReturn(PETSC_SUCCESS);
279 }
280 
281 /*@
282   DMPlexLabelComplete - Starting with a label marking points on a surface, we add the transitive closure to the surface
283 
284   Input Parameters:
285 + dm    - The `DM`
286 - label - A `DMLabel` marking the surface points
287 
288   Output Parameter:
289 . label - A `DMLabel` marking all surface points in the transitive closure
290 
291   Level: developer
292 
293 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexLabelCohesiveComplete()`
294 @*/
295 PetscErrorCode DMPlexLabelComplete(DM dm, DMLabel label)
296 {
297   PetscFunctionBegin;
298   PetscCall(DMPlexLabelComplete_Internal(dm, label, PETSC_TRUE));
299   PetscFunctionReturn(PETSC_SUCCESS);
300 }
301 
302 /*@
303   DMPlexLabelAddCells - Starting with a label marking points on a surface, we add a cell for each point
304 
305   Input Parameters:
306 + dm    - The `DM`
307 - label - A `DMLabel` marking the surface points
308 
309   Output Parameter:
310 . label - A `DMLabel` incorporating cells
311 
312   Level: developer
313 
314   Note:
315   The cells allow FEM boundary conditions to be applied using the cell geometry
316 
317 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexLabelAddFaceCells()`, `DMPlexLabelComplete()`, `DMPlexLabelCohesiveComplete()`
318 @*/
319 PetscErrorCode DMPlexLabelAddCells(DM dm, DMLabel label)
320 {
321   IS              valueIS;
322   const PetscInt *values;
323   PetscInt        numValues, v, csStart, csEnd, chStart, chEnd;
324 
325   PetscFunctionBegin;
326   PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &csStart, &csEnd));
327   PetscCall(DMPlexGetHeightStratum(dm, 0, &chStart, &chEnd));
328   PetscCall(DMLabelGetNumValues(label, &numValues));
329   PetscCall(DMLabelGetValueIS(label, &valueIS));
330   PetscCall(ISGetIndices(valueIS, &values));
331   for (v = 0; v < numValues; ++v) {
332     IS              pointIS;
333     const PetscInt *points;
334     PetscInt        numPoints, p;
335 
336     PetscCall(DMLabelGetStratumSize(label, values[v], &numPoints));
337     PetscCall(DMLabelGetStratumIS(label, values[v], &pointIS));
338     PetscCall(ISGetIndices(pointIS, &points));
339     for (p = 0; p < numPoints; ++p) {
340       const PetscInt point   = points[p];
341       PetscInt      *closure = NULL;
342       PetscInt       closureSize, cl, h, cStart, cEnd;
343       DMPolytopeType ct;
344 
345       // If the point is a hybrid, allow hybrid cells
346       PetscCall(DMPlexGetCellType(dm, point, &ct));
347       PetscCall(DMPlexGetPointHeight(dm, point, &h));
348       if (DMPolytopeTypeIsHybrid(ct)) {
349         cStart = chStart;
350         cEnd   = chEnd;
351       } else {
352         cStart = csStart;
353         cEnd   = csEnd;
354       }
355 
356       PetscCall(DMPlexGetTransitiveClosure(dm, points[p], PETSC_FALSE, &closureSize, &closure));
357       for (cl = closureSize - 1; cl > 0; --cl) {
358         const PetscInt cell = closure[cl * 2];
359         if ((cell >= cStart) && (cell < cEnd)) {
360           PetscCall(DMLabelSetValue(label, cell, values[v]));
361           break;
362         }
363       }
364       PetscCall(DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_FALSE, &closureSize, &closure));
365     }
366     PetscCall(ISRestoreIndices(pointIS, &points));
367     PetscCall(ISDestroy(&pointIS));
368   }
369   PetscCall(ISRestoreIndices(valueIS, &values));
370   PetscCall(ISDestroy(&valueIS));
371   PetscFunctionReturn(PETSC_SUCCESS);
372 }
373 
374 /*@
375   DMPlexLabelAddFaceCells - Starting with a label marking faces on a surface, we add a cell for each face
376 
377   Input Parameters:
378 + dm    - The `DM`
379 - label - A `DMLabel` marking the surface points
380 
381   Output Parameter:
382 . label - A `DMLabel` incorporating cells
383 
384   Level: developer
385 
386   Note:
387   The cells allow FEM boundary conditions to be applied using the cell geometry
388 
389 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexLabelAddCells()`, `DMPlexLabelComplete()`, `DMPlexLabelCohesiveComplete()`
390 @*/
391 PetscErrorCode DMPlexLabelAddFaceCells(DM dm, DMLabel label)
392 {
393   IS              valueIS;
394   const PetscInt *values;
395   PetscInt        numValues, v, cStart, cEnd, fStart, fEnd;
396 
397   PetscFunctionBegin;
398   PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd));
399   PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
400   PetscCall(DMLabelGetNumValues(label, &numValues));
401   PetscCall(DMLabelGetValueIS(label, &valueIS));
402   PetscCall(ISGetIndices(valueIS, &values));
403   for (v = 0; v < numValues; ++v) {
404     IS              pointIS;
405     const PetscInt *points;
406     PetscInt        numPoints, p;
407 
408     PetscCall(DMLabelGetStratumSize(label, values[v], &numPoints));
409     PetscCall(DMLabelGetStratumIS(label, values[v], &pointIS));
410     PetscCall(ISGetIndices(pointIS, &points));
411     for (p = 0; p < numPoints; ++p) {
412       const PetscInt face    = points[p];
413       PetscInt      *closure = NULL;
414       PetscInt       closureSize, cl;
415 
416       if ((face < fStart) || (face >= fEnd)) continue;
417       PetscCall(DMPlexGetTransitiveClosure(dm, face, PETSC_FALSE, &closureSize, &closure));
418       for (cl = closureSize - 1; cl > 0; --cl) {
419         const PetscInt cell = closure[cl * 2];
420         if ((cell >= cStart) && (cell < cEnd)) {
421           PetscCall(DMLabelSetValue(label, cell, values[v]));
422           break;
423         }
424       }
425       PetscCall(DMPlexRestoreTransitiveClosure(dm, face, PETSC_FALSE, &closureSize, &closure));
426     }
427     PetscCall(ISRestoreIndices(pointIS, &points));
428     PetscCall(ISDestroy(&pointIS));
429   }
430   PetscCall(ISRestoreIndices(valueIS, &values));
431   PetscCall(ISDestroy(&valueIS));
432   PetscFunctionReturn(PETSC_SUCCESS);
433 }
434 
435 /*@
436   DMPlexLabelClearCells - Remove cells from a label
437 
438   Input Parameters:
439 + dm    - The `DM`
440 - label - A `DMLabel` marking surface points and their adjacent cells
441 
442   Output Parameter:
443 . label - A `DMLabel` without cells
444 
445   Level: developer
446 
447   Note:
448   This undoes `DMPlexLabelAddCells()` or `DMPlexLabelAddFaceCells()`
449 
450 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexLabelComplete()`, `DMPlexLabelCohesiveComplete()`, `DMPlexLabelAddCells()`
451 @*/
452 PetscErrorCode DMPlexLabelClearCells(DM dm, DMLabel label)
453 {
454   IS              valueIS;
455   const PetscInt *values;
456   PetscInt        numValues, v, cStart, cEnd;
457 
458   PetscFunctionBegin;
459   PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd));
460   PetscCall(DMLabelGetNumValues(label, &numValues));
461   PetscCall(DMLabelGetValueIS(label, &valueIS));
462   PetscCall(ISGetIndices(valueIS, &values));
463   for (v = 0; v < numValues; ++v) {
464     IS              pointIS;
465     const PetscInt *points;
466     PetscInt        numPoints, p;
467 
468     PetscCall(DMLabelGetStratumSize(label, values[v], &numPoints));
469     PetscCall(DMLabelGetStratumIS(label, values[v], &pointIS));
470     PetscCall(ISGetIndices(pointIS, &points));
471     for (p = 0; p < numPoints; ++p) {
472       PetscInt point = points[p];
473 
474       if (point >= cStart && point < cEnd) PetscCall(DMLabelClearValue(label, point, values[v]));
475     }
476     PetscCall(ISRestoreIndices(pointIS, &points));
477     PetscCall(ISDestroy(&pointIS));
478   }
479   PetscCall(ISRestoreIndices(valueIS, &values));
480   PetscCall(ISDestroy(&valueIS));
481   PetscFunctionReturn(PETSC_SUCCESS);
482 }
483 
484 /* take (oldEnd, added) pairs, ordered by height and convert them to (oldstart, newstart) pairs, ordered by ascending
485  * index (skipping first, which is (0,0)) */
486 static inline PetscErrorCode DMPlexShiftPointSetUp_Internal(PetscInt depth, PetscInt depthShift[])
487 {
488   PetscInt d, off = 0;
489 
490   PetscFunctionBegin;
491   /* sort by (oldend): yes this is an O(n^2) sort, we expect depth <= 3 */
492   for (d = 0; d < depth; d++) {
493     PetscInt firstd     = d;
494     PetscInt firstStart = depthShift[2 * d];
495     PetscInt e;
496 
497     for (e = d + 1; e <= depth; e++) {
498       if (depthShift[2 * e] < firstStart) {
499         firstd     = e;
500         firstStart = depthShift[2 * d];
501       }
502     }
503     if (firstd != d) {
504       PetscInt swap[2];
505 
506       e                     = firstd;
507       swap[0]               = depthShift[2 * d];
508       swap[1]               = depthShift[2 * d + 1];
509       depthShift[2 * d]     = depthShift[2 * e];
510       depthShift[2 * d + 1] = depthShift[2 * e + 1];
511       depthShift[2 * e]     = swap[0];
512       depthShift[2 * e + 1] = swap[1];
513     }
514   }
515   /* convert (oldstart, added) to (oldstart, newstart) */
516   for (d = 0; d <= depth; d++) {
517     off += depthShift[2 * d + 1];
518     depthShift[2 * d + 1] = depthShift[2 * d] + off;
519   }
520   PetscFunctionReturn(PETSC_SUCCESS);
521 }
522 
523 /* depthShift is a list of (old, new) pairs */
524 static inline PetscInt DMPlexShiftPoint_Internal(PetscInt p, PetscInt depth, PetscInt depthShift[])
525 {
526   PetscInt d;
527   PetscInt newOff = 0;
528 
529   for (d = 0; d <= depth; d++) {
530     if (p < depthShift[2 * d]) return p + newOff;
531     else newOff = depthShift[2 * d + 1] - depthShift[2 * d];
532   }
533   return p + newOff;
534 }
535 
536 /* depthShift is a list of (old, new) pairs */
537 static inline PetscInt DMPlexShiftPointInverse_Internal(PetscInt p, PetscInt depth, PetscInt depthShift[])
538 {
539   PetscInt d;
540   PetscInt newOff = 0;
541 
542   for (d = 0; d <= depth; d++) {
543     if (p < depthShift[2 * d + 1]) return p + newOff;
544     else newOff = depthShift[2 * d] - depthShift[2 * d + 1];
545   }
546   return p + newOff;
547 }
548 
549 static PetscErrorCode DMPlexShiftSizes_Internal(DM dm, PetscInt depthShift[], DM dmNew)
550 {
551   PetscInt depth = 0, d, pStart, pEnd, p;
552   DMLabel  depthLabel;
553 
554   PetscFunctionBegin;
555   PetscCall(DMPlexGetDepth(dm, &depth));
556   if (depth < 0) PetscFunctionReturn(PETSC_SUCCESS);
557   /* Step 1: Expand chart */
558   PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
559   pEnd = DMPlexShiftPoint_Internal(pEnd, depth, depthShift);
560   PetscCall(DMPlexSetChart(dmNew, pStart, pEnd));
561   PetscCall(DMCreateLabel(dmNew, "depth"));
562   PetscCall(DMPlexGetDepthLabel(dmNew, &depthLabel));
563   PetscCall(DMCreateLabel(dmNew, "celltype"));
564   /* Step 2: Set cone and support sizes */
565   for (d = 0; d <= depth; ++d) {
566     PetscInt pStartNew, pEndNew;
567     IS       pIS;
568 
569     PetscCall(DMPlexGetDepthStratum(dm, d, &pStart, &pEnd));
570     pStartNew = DMPlexShiftPoint_Internal(pStart, depth, depthShift);
571     pEndNew   = DMPlexShiftPoint_Internal(pEnd, depth, depthShift);
572     PetscCall(ISCreateStride(PETSC_COMM_SELF, pEndNew - pStartNew, pStartNew, 1, &pIS));
573     PetscCall(DMLabelSetStratumIS(depthLabel, d, pIS));
574     PetscCall(ISDestroy(&pIS));
575     for (p = pStart; p < pEnd; ++p) {
576       PetscInt       newp = DMPlexShiftPoint_Internal(p, depth, depthShift);
577       PetscInt       size;
578       DMPolytopeType ct;
579 
580       PetscCall(DMPlexGetConeSize(dm, p, &size));
581       PetscCall(DMPlexSetConeSize(dmNew, newp, size));
582       PetscCall(DMPlexGetSupportSize(dm, p, &size));
583       PetscCall(DMPlexSetSupportSize(dmNew, newp, size));
584       PetscCall(DMPlexGetCellType(dm, p, &ct));
585       PetscCall(DMPlexSetCellType(dmNew, newp, ct));
586     }
587   }
588   PetscFunctionReturn(PETSC_SUCCESS);
589 }
590 
591 static PetscErrorCode DMPlexShiftPoints_Internal(DM dm, PetscInt depthShift[], DM dmNew)
592 {
593   PetscInt *newpoints;
594   PetscInt  depth = 0, maxConeSize, maxSupportSize, maxConeSizeNew, maxSupportSizeNew, pStart, pEnd, p;
595 
596   PetscFunctionBegin;
597   PetscCall(DMPlexGetDepth(dm, &depth));
598   if (depth < 0) PetscFunctionReturn(PETSC_SUCCESS);
599   PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize));
600   PetscCall(DMPlexGetMaxSizes(dmNew, &maxConeSizeNew, &maxSupportSizeNew));
601   PetscCall(PetscMalloc1(PetscMax(PetscMax(maxConeSize, maxSupportSize), PetscMax(maxConeSizeNew, maxSupportSizeNew)), &newpoints));
602   /* Step 5: Set cones and supports */
603   PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
604   for (p = pStart; p < pEnd; ++p) {
605     const PetscInt *points = NULL, *orientations = NULL;
606     PetscInt        size, sizeNew, i, newp = DMPlexShiftPoint_Internal(p, depth, depthShift);
607 
608     PetscCall(DMPlexGetConeSize(dm, p, &size));
609     PetscCall(DMPlexGetCone(dm, p, &points));
610     PetscCall(DMPlexGetConeOrientation(dm, p, &orientations));
611     for (i = 0; i < size; ++i) newpoints[i] = DMPlexShiftPoint_Internal(points[i], depth, depthShift);
612     PetscCall(DMPlexSetCone(dmNew, newp, newpoints));
613     PetscCall(DMPlexSetConeOrientation(dmNew, newp, orientations));
614     PetscCall(DMPlexGetSupportSize(dm, p, &size));
615     PetscCall(DMPlexGetSupportSize(dmNew, newp, &sizeNew));
616     PetscCall(DMPlexGetSupport(dm, p, &points));
617     for (i = 0; i < size; ++i) newpoints[i] = DMPlexShiftPoint_Internal(points[i], depth, depthShift);
618     for (i = size; i < sizeNew; ++i) newpoints[i] = 0;
619     PetscCall(DMPlexSetSupport(dmNew, newp, newpoints));
620   }
621   PetscCall(PetscFree(newpoints));
622   PetscFunctionReturn(PETSC_SUCCESS);
623 }
624 
625 static PetscErrorCode DMPlexShiftCoordinates_Internal(DM dm, PetscInt depthShift[], DM dmNew)
626 {
627   PetscSection coordSection, newCoordSection;
628   Vec          coordinates, newCoordinates;
629   PetscScalar *coords, *newCoords;
630   PetscInt     coordSize, sStart, sEnd;
631   PetscInt     dim, depth = 0, cStart, cEnd, cStartNew, cEndNew, c, vStart, vEnd, vStartNew, vEndNew, v;
632   PetscBool    hasCells;
633 
634   PetscFunctionBegin;
635   PetscCall(DMGetCoordinateDim(dm, &dim));
636   PetscCall(DMSetCoordinateDim(dmNew, dim));
637   PetscCall(DMPlexGetDepth(dm, &depth));
638   /* Step 8: Convert coordinates */
639   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
640   PetscCall(DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd));
641   PetscCall(DMPlexGetDepthStratum(dmNew, 0, &vStartNew, &vEndNew));
642   PetscCall(DMPlexGetHeightStratum(dmNew, 0, &cStartNew, &cEndNew));
643   PetscCall(DMGetCoordinateSection(dm, &coordSection));
644   PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dm), &newCoordSection));
645   PetscCall(PetscSectionSetNumFields(newCoordSection, 1));
646   PetscCall(PetscSectionSetFieldComponents(newCoordSection, 0, dim));
647   PetscCall(PetscSectionGetChart(coordSection, &sStart, &sEnd));
648   hasCells = sStart == cStart ? PETSC_TRUE : PETSC_FALSE;
649   PetscCall(PetscSectionSetChart(newCoordSection, hasCells ? cStartNew : vStartNew, vEndNew));
650   if (hasCells) {
651     for (c = cStart; c < cEnd; ++c) {
652       PetscInt cNew = DMPlexShiftPoint_Internal(c, depth, depthShift), dof;
653 
654       PetscCall(PetscSectionGetDof(coordSection, c, &dof));
655       PetscCall(PetscSectionSetDof(newCoordSection, cNew, dof));
656       PetscCall(PetscSectionSetFieldDof(newCoordSection, cNew, 0, dof));
657     }
658   }
659   for (v = vStartNew; v < vEndNew; ++v) {
660     PetscCall(PetscSectionSetDof(newCoordSection, v, dim));
661     PetscCall(PetscSectionSetFieldDof(newCoordSection, v, 0, dim));
662   }
663   PetscCall(PetscSectionSetUp(newCoordSection));
664   PetscCall(DMSetCoordinateSection(dmNew, PETSC_DETERMINE, newCoordSection));
665   PetscCall(PetscSectionGetStorageSize(newCoordSection, &coordSize));
666   PetscCall(VecCreate(PETSC_COMM_SELF, &newCoordinates));
667   PetscCall(PetscObjectSetName((PetscObject)newCoordinates, "coordinates"));
668   PetscCall(VecSetSizes(newCoordinates, coordSize, PETSC_DETERMINE));
669   PetscCall(VecSetBlockSize(newCoordinates, dim));
670   PetscCall(VecSetType(newCoordinates, VECSTANDARD));
671   PetscCall(DMSetCoordinatesLocal(dmNew, newCoordinates));
672   PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
673   PetscCall(VecGetArray(coordinates, &coords));
674   PetscCall(VecGetArray(newCoordinates, &newCoords));
675   if (hasCells) {
676     for (c = cStart; c < cEnd; ++c) {
677       PetscInt cNew = DMPlexShiftPoint_Internal(c, depth, depthShift), dof, off, noff, d;
678 
679       PetscCall(PetscSectionGetDof(coordSection, c, &dof));
680       PetscCall(PetscSectionGetOffset(coordSection, c, &off));
681       PetscCall(PetscSectionGetOffset(newCoordSection, cNew, &noff));
682       for (d = 0; d < dof; ++d) newCoords[noff + d] = coords[off + d];
683     }
684   }
685   for (v = vStart; v < vEnd; ++v) {
686     PetscInt dof, off, noff, d;
687 
688     PetscCall(PetscSectionGetDof(coordSection, v, &dof));
689     PetscCall(PetscSectionGetOffset(coordSection, v, &off));
690     PetscCall(PetscSectionGetOffset(newCoordSection, DMPlexShiftPoint_Internal(v, depth, depthShift), &noff));
691     for (d = 0; d < dof; ++d) newCoords[noff + d] = coords[off + d];
692   }
693   PetscCall(VecRestoreArray(coordinates, &coords));
694   PetscCall(VecRestoreArray(newCoordinates, &newCoords));
695   PetscCall(VecDestroy(&newCoordinates));
696   PetscCall(PetscSectionDestroy(&newCoordSection));
697   PetscFunctionReturn(PETSC_SUCCESS);
698 }
699 
700 static PetscErrorCode DMPlexShiftSF_Single(DM dm, PetscInt depthShift[], PetscSF sf, PetscSF sfNew)
701 {
702   const PetscSFNode *remotePoints;
703   PetscSFNode       *gremotePoints;
704   const PetscInt    *localPoints;
705   PetscInt          *glocalPoints, *newLocation, *newRemoteLocation;
706   PetscInt           numRoots, numLeaves, l, pStart, pEnd, depth = 0, totShift = 0;
707 
708   PetscFunctionBegin;
709   PetscCall(DMPlexGetDepth(dm, &depth));
710   PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
711   PetscCall(PetscSFGetGraph(sf, &numRoots, &numLeaves, &localPoints, &remotePoints));
712   totShift = DMPlexShiftPoint_Internal(pEnd, depth, depthShift) - pEnd;
713   if (numRoots >= 0) {
714     PetscCall(PetscMalloc2(numRoots, &newLocation, pEnd - pStart, &newRemoteLocation));
715     for (l = 0; l < numRoots; ++l) newLocation[l] = DMPlexShiftPoint_Internal(l, depth, depthShift);
716     PetscCall(PetscSFBcastBegin(sf, MPIU_INT, newLocation, newRemoteLocation, MPI_REPLACE));
717     PetscCall(PetscSFBcastEnd(sf, MPIU_INT, newLocation, newRemoteLocation, MPI_REPLACE));
718     PetscCall(PetscMalloc1(numLeaves, &glocalPoints));
719     PetscCall(PetscMalloc1(numLeaves, &gremotePoints));
720     for (l = 0; l < numLeaves; ++l) {
721       glocalPoints[l]        = DMPlexShiftPoint_Internal(localPoints[l], depth, depthShift);
722       gremotePoints[l].rank  = remotePoints[l].rank;
723       gremotePoints[l].index = newRemoteLocation[localPoints[l]];
724     }
725     PetscCall(PetscFree2(newLocation, newRemoteLocation));
726     PetscCall(PetscSFSetGraph(sfNew, numRoots + totShift, numLeaves, glocalPoints, PETSC_OWN_POINTER, gremotePoints, PETSC_OWN_POINTER));
727   }
728   PetscFunctionReturn(PETSC_SUCCESS);
729 }
730 
731 static PetscErrorCode DMPlexShiftSF_Internal(DM dm, PetscInt depthShift[], DM dmNew)
732 {
733   PetscSF   sfPoint, sfPointNew;
734   PetscBool useNatural;
735 
736   PetscFunctionBegin;
737   /* Step 9: Convert pointSF */
738   PetscCall(DMGetPointSF(dm, &sfPoint));
739   PetscCall(DMGetPointSF(dmNew, &sfPointNew));
740   PetscCall(DMPlexShiftSF_Single(dm, depthShift, sfPoint, sfPointNew));
741   /* Step 9b: Convert naturalSF */
742   PetscCall(DMGetUseNatural(dm, &useNatural));
743   if (useNatural) {
744     PetscSF sfNat, sfNatNew;
745 
746     PetscCall(DMSetUseNatural(dmNew, useNatural));
747     PetscCall(DMGetNaturalSF(dm, &sfNat));
748     PetscCall(DMGetNaturalSF(dmNew, &sfNatNew));
749     PetscCall(DMPlexShiftSF_Single(dm, depthShift, sfNat, sfNatNew));
750   }
751   PetscFunctionReturn(PETSC_SUCCESS);
752 }
753 
754 static PetscErrorCode DMPlexShiftLabels_Internal(DM dm, PetscInt depthShift[], DM dmNew)
755 {
756   PetscInt depth = 0, numLabels, l;
757 
758   PetscFunctionBegin;
759   PetscCall(DMPlexGetDepth(dm, &depth));
760   /* Step 10: Convert labels */
761   PetscCall(DMGetNumLabels(dm, &numLabels));
762   for (l = 0; l < numLabels; ++l) {
763     DMLabel         label, newlabel;
764     const char     *lname;
765     PetscBool       isDepth, isDim;
766     IS              valueIS;
767     const PetscInt *values;
768     PetscInt        numValues, val;
769 
770     PetscCall(DMGetLabelName(dm, l, &lname));
771     PetscCall(PetscStrcmp(lname, "depth", &isDepth));
772     if (isDepth) continue;
773     PetscCall(PetscStrcmp(lname, "dim", &isDim));
774     if (isDim) continue;
775     PetscCall(DMCreateLabel(dmNew, lname));
776     PetscCall(DMGetLabel(dm, lname, &label));
777     PetscCall(DMGetLabel(dmNew, lname, &newlabel));
778     PetscCall(DMLabelGetDefaultValue(label, &val));
779     PetscCall(DMLabelSetDefaultValue(newlabel, val));
780     PetscCall(DMLabelGetValueIS(label, &valueIS));
781     PetscCall(ISGetLocalSize(valueIS, &numValues));
782     PetscCall(ISGetIndices(valueIS, &values));
783     for (val = 0; val < numValues; ++val) {
784       IS              pointIS;
785       const PetscInt *points;
786       PetscInt        numPoints, p;
787 
788       PetscCall(DMLabelGetStratumIS(label, values[val], &pointIS));
789       PetscCall(ISGetLocalSize(pointIS, &numPoints));
790       PetscCall(ISGetIndices(pointIS, &points));
791       for (p = 0; p < numPoints; ++p) {
792         const PetscInt newpoint = DMPlexShiftPoint_Internal(points[p], depth, depthShift);
793 
794         PetscCall(DMLabelSetValue(newlabel, newpoint, values[val]));
795       }
796       PetscCall(ISRestoreIndices(pointIS, &points));
797       PetscCall(ISDestroy(&pointIS));
798     }
799     PetscCall(ISRestoreIndices(valueIS, &values));
800     PetscCall(ISDestroy(&valueIS));
801   }
802   PetscFunctionReturn(PETSC_SUCCESS);
803 }
804 
805 static PetscErrorCode DMPlexCreateVTKLabel_Internal(DM dm, PetscBool createGhostLabel, DM dmNew)
806 {
807   PetscSF            sfPoint;
808   DMLabel            vtkLabel, ghostLabel = NULL;
809   const PetscSFNode *leafRemote;
810   const PetscInt    *leafLocal;
811   PetscInt           cellHeight, cStart, cEnd, c, fStart, fEnd, f, numLeaves, l;
812   PetscMPIInt        rank;
813 
814   PetscFunctionBegin;
815   /* Step 11: Make label for output (vtk) and to mark ghost points (ghost) */
816   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
817   PetscCall(DMGetPointSF(dm, &sfPoint));
818   PetscCall(DMPlexGetVTKCellHeight(dmNew, &cellHeight));
819   PetscCall(DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd));
820   PetscCall(PetscSFGetGraph(sfPoint, NULL, &numLeaves, &leafLocal, &leafRemote));
821   PetscCall(DMCreateLabel(dmNew, "vtk"));
822   PetscCall(DMGetLabel(dmNew, "vtk", &vtkLabel));
823   if (createGhostLabel) {
824     PetscCall(DMCreateLabel(dmNew, "ghost"));
825     PetscCall(DMGetLabel(dmNew, "ghost", &ghostLabel));
826   }
827   for (l = 0, c = cStart; l < numLeaves && c < cEnd; ++l, ++c) {
828     for (; c < leafLocal[l] && c < cEnd; ++c) PetscCall(DMLabelSetValue(vtkLabel, c, 1));
829     if (leafLocal[l] >= cEnd) break;
830     if (leafRemote[l].rank == rank) {
831       PetscCall(DMLabelSetValue(vtkLabel, c, 1));
832     } else if (ghostLabel) PetscCall(DMLabelSetValue(ghostLabel, c, 2));
833   }
834   for (; c < cEnd; ++c) PetscCall(DMLabelSetValue(vtkLabel, c, 1));
835   if (ghostLabel) {
836     PetscCall(DMPlexGetHeightStratum(dmNew, 1, &fStart, &fEnd));
837     for (f = fStart; f < fEnd; ++f) {
838       PetscInt numCells;
839 
840       PetscCall(DMPlexGetSupportSize(dmNew, f, &numCells));
841       if (numCells < 2) {
842         PetscCall(DMLabelSetValue(ghostLabel, f, 1));
843       } else {
844         const PetscInt *cells = NULL;
845         PetscInt        vA, vB;
846 
847         PetscCall(DMPlexGetSupport(dmNew, f, &cells));
848         PetscCall(DMLabelGetValue(vtkLabel, cells[0], &vA));
849         PetscCall(DMLabelGetValue(vtkLabel, cells[1], &vB));
850         if (vA != 1 && vB != 1) PetscCall(DMLabelSetValue(ghostLabel, f, 1));
851       }
852     }
853   }
854   PetscFunctionReturn(PETSC_SUCCESS);
855 }
856 
857 static PetscErrorCode DMPlexShiftTree_Internal(DM dm, PetscInt depthShift[], DM dmNew)
858 {
859   DM           refTree;
860   PetscSection pSec;
861   PetscInt    *parents, *childIDs;
862 
863   PetscFunctionBegin;
864   PetscCall(DMPlexGetReferenceTree(dm, &refTree));
865   PetscCall(DMPlexSetReferenceTree(dmNew, refTree));
866   PetscCall(DMPlexGetTree(dm, &pSec, &parents, &childIDs, NULL, NULL));
867   if (pSec) {
868     PetscInt     p, pStart, pEnd, *parentsShifted, pStartShifted, pEndShifted, depth;
869     PetscInt    *childIDsShifted;
870     PetscSection pSecShifted;
871 
872     PetscCall(PetscSectionGetChart(pSec, &pStart, &pEnd));
873     PetscCall(DMPlexGetDepth(dm, &depth));
874     pStartShifted = DMPlexShiftPoint_Internal(pStart, depth, depthShift);
875     pEndShifted   = DMPlexShiftPoint_Internal(pEnd, depth, depthShift);
876     PetscCall(PetscMalloc2(pEndShifted - pStartShifted, &parentsShifted, pEndShifted - pStartShifted, &childIDsShifted));
877     PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dmNew), &pSecShifted));
878     PetscCall(PetscSectionSetChart(pSecShifted, pStartShifted, pEndShifted));
879     for (p = pStartShifted; p < pEndShifted; p++) {
880       /* start off assuming no children */
881       PetscCall(PetscSectionSetDof(pSecShifted, p, 0));
882     }
883     for (p = pStart; p < pEnd; p++) {
884       PetscInt dof;
885       PetscInt pNew = DMPlexShiftPoint_Internal(p, depth, depthShift);
886 
887       PetscCall(PetscSectionGetDof(pSec, p, &dof));
888       PetscCall(PetscSectionSetDof(pSecShifted, pNew, dof));
889     }
890     PetscCall(PetscSectionSetUp(pSecShifted));
891     for (p = pStart; p < pEnd; p++) {
892       PetscInt dof;
893       PetscInt pNew = DMPlexShiftPoint_Internal(p, depth, depthShift);
894 
895       PetscCall(PetscSectionGetDof(pSec, p, &dof));
896       if (dof) {
897         PetscInt off, offNew;
898 
899         PetscCall(PetscSectionGetOffset(pSec, p, &off));
900         PetscCall(PetscSectionGetOffset(pSecShifted, pNew, &offNew));
901         parentsShifted[offNew]  = DMPlexShiftPoint_Internal(parents[off], depth, depthShift);
902         childIDsShifted[offNew] = childIDs[off];
903       }
904     }
905     PetscCall(DMPlexSetTree(dmNew, pSecShifted, parentsShifted, childIDsShifted));
906     PetscCall(PetscFree2(parentsShifted, childIDsShifted));
907     PetscCall(PetscSectionDestroy(&pSecShifted));
908   }
909   PetscFunctionReturn(PETSC_SUCCESS);
910 }
911 
912 static PetscErrorCode DMPlexConstructGhostCells_Internal(DM dm, DMLabel label, PetscInt *numGhostCells, DM gdm)
913 {
914   PetscSF          sf;
915   IS               valueIS;
916   const PetscInt  *values, *leaves;
917   PetscInt        *depthShift;
918   PetscInt         d, depth = 0, nleaves, loc, Ng, numFS, fs, fStart, fEnd, ghostCell, cEnd, c;
919   const PetscReal *maxCell, *Lstart, *L;
920 
921   PetscFunctionBegin;
922   PetscCall(DMGetPointSF(dm, &sf));
923   PetscCall(PetscSFGetGraph(sf, NULL, &nleaves, &leaves, NULL));
924   nleaves = PetscMax(0, nleaves);
925   PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
926   /* Count ghost cells */
927   PetscCall(DMLabelGetValueIS(label, &valueIS));
928   PetscCall(ISGetLocalSize(valueIS, &numFS));
929   PetscCall(ISGetIndices(valueIS, &values));
930   Ng = 0;
931   for (fs = 0; fs < numFS; ++fs) {
932     IS              faceIS;
933     const PetscInt *faces;
934     PetscInt        numFaces, f, numBdFaces = 0;
935 
936     PetscCall(DMLabelGetStratumIS(label, values[fs], &faceIS));
937     PetscCall(ISGetLocalSize(faceIS, &numFaces));
938     PetscCall(ISGetIndices(faceIS, &faces));
939     for (f = 0; f < numFaces; ++f) {
940       PetscInt numChildren;
941 
942       PetscCall(PetscFindInt(faces[f], nleaves, leaves, &loc));
943       PetscCall(DMPlexGetTreeChildren(dm, faces[f], &numChildren, NULL));
944       /* non-local and ancestors points don't get to register ghosts */
945       if (loc >= 0 || numChildren) continue;
946       if ((faces[f] >= fStart) && (faces[f] < fEnd)) ++numBdFaces;
947     }
948     Ng += numBdFaces;
949     PetscCall(ISRestoreIndices(faceIS, &faces));
950     PetscCall(ISDestroy(&faceIS));
951   }
952   PetscCall(DMPlexGetDepth(dm, &depth));
953   PetscCall(PetscMalloc1(2 * (depth + 1), &depthShift));
954   for (d = 0; d <= depth; d++) {
955     PetscInt dEnd;
956 
957     PetscCall(DMPlexGetDepthStratum(dm, d, NULL, &dEnd));
958     depthShift[2 * d]     = dEnd;
959     depthShift[2 * d + 1] = 0;
960   }
961   if (depth >= 0) depthShift[2 * depth + 1] = Ng;
962   PetscCall(DMPlexShiftPointSetUp_Internal(depth, depthShift));
963   PetscCall(DMPlexShiftSizes_Internal(dm, depthShift, gdm));
964   /* Step 3: Set cone/support sizes for new points */
965   PetscCall(DMPlexGetHeightStratum(dm, 0, NULL, &cEnd));
966   for (c = cEnd; c < cEnd + Ng; ++c) PetscCall(DMPlexSetConeSize(gdm, c, 1));
967   for (fs = 0; fs < numFS; ++fs) {
968     IS              faceIS;
969     const PetscInt *faces;
970     PetscInt        numFaces, f;
971 
972     PetscCall(DMLabelGetStratumIS(label, values[fs], &faceIS));
973     PetscCall(ISGetLocalSize(faceIS, &numFaces));
974     PetscCall(ISGetIndices(faceIS, &faces));
975     for (f = 0; f < numFaces; ++f) {
976       PetscInt size, numChildren;
977 
978       PetscCall(PetscFindInt(faces[f], nleaves, leaves, &loc));
979       PetscCall(DMPlexGetTreeChildren(dm, faces[f], &numChildren, NULL));
980       if (loc >= 0 || numChildren) continue;
981       if ((faces[f] < fStart) || (faces[f] >= fEnd)) continue;
982       PetscCall(DMPlexGetSupportSize(dm, faces[f], &size));
983       PetscCheck(size == 1, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM has boundary face %" PetscInt_FMT " with %" PetscInt_FMT " support cells", faces[f], size);
984       PetscCall(DMPlexSetSupportSize(gdm, faces[f] + Ng, 2));
985     }
986     PetscCall(ISRestoreIndices(faceIS, &faces));
987     PetscCall(ISDestroy(&faceIS));
988   }
989   /* Step 4: Setup ghosted DM */
990   PetscCall(DMSetUp(gdm));
991   PetscCall(DMPlexShiftPoints_Internal(dm, depthShift, gdm));
992   /* Step 6: Set cones and supports for new points */
993   ghostCell = cEnd;
994   for (fs = 0; fs < numFS; ++fs) {
995     IS              faceIS;
996     const PetscInt *faces;
997     PetscInt        numFaces, f;
998 
999     PetscCall(DMLabelGetStratumIS(label, values[fs], &faceIS));
1000     PetscCall(ISGetLocalSize(faceIS, &numFaces));
1001     PetscCall(ISGetIndices(faceIS, &faces));
1002     for (f = 0; f < numFaces; ++f) {
1003       PetscInt newFace = faces[f] + Ng, numChildren;
1004 
1005       PetscCall(PetscFindInt(faces[f], nleaves, leaves, &loc));
1006       PetscCall(DMPlexGetTreeChildren(dm, faces[f], &numChildren, NULL));
1007       if (loc >= 0 || numChildren) continue;
1008       if ((faces[f] < fStart) || (faces[f] >= fEnd)) continue;
1009       PetscCall(DMPlexSetCone(gdm, ghostCell, &newFace));
1010       PetscCall(DMPlexInsertSupport(gdm, newFace, 1, ghostCell));
1011       ++ghostCell;
1012     }
1013     PetscCall(ISRestoreIndices(faceIS, &faces));
1014     PetscCall(ISDestroy(&faceIS));
1015   }
1016   PetscCall(ISRestoreIndices(valueIS, &values));
1017   PetscCall(ISDestroy(&valueIS));
1018   PetscCall(DMPlexShiftCoordinates_Internal(dm, depthShift, gdm));
1019   PetscCall(DMPlexShiftSF_Internal(dm, depthShift, gdm));
1020   PetscCall(DMPlexShiftLabels_Internal(dm, depthShift, gdm));
1021   PetscCall(DMPlexCreateVTKLabel_Internal(dm, PETSC_TRUE, gdm));
1022   PetscCall(DMPlexShiftTree_Internal(dm, depthShift, gdm));
1023   PetscCall(PetscFree(depthShift));
1024   for (c = cEnd; c < cEnd + Ng; ++c) PetscCall(DMPlexSetCellType(gdm, c, DM_POLYTOPE_FV_GHOST));
1025   /* Step 7: Periodicity */
1026   PetscCall(DMGetPeriodicity(dm, &maxCell, &Lstart, &L));
1027   PetscCall(DMSetPeriodicity(gdm, maxCell, Lstart, L));
1028   if (numGhostCells) *numGhostCells = Ng;
1029   PetscFunctionReturn(PETSC_SUCCESS);
1030 }
1031 
1032 /*@C
1033   DMPlexConstructGhostCells - Construct ghost cells which connect to every boundary face
1034 
1035   Collective
1036 
1037   Input Parameters:
1038 + dm        - The original `DM`
1039 - labelName - The label specifying the boundary faces, or "Face Sets" if this is `NULL`
1040 
1041   Output Parameters:
1042 + numGhostCells - The number of ghost cells added to the `DM`
1043 - dmGhosted     - The new `DM`
1044 
1045   Level: developer
1046 
1047   Note:
1048   If no label exists of that name, one will be created marking all boundary faces
1049 
1050 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMCreate()`
1051 @*/
1052 PetscErrorCode DMPlexConstructGhostCells(DM dm, const char labelName[], PetscInt *numGhostCells, DM *dmGhosted)
1053 {
1054   DM          gdm;
1055   DMLabel     label;
1056   const char *name = labelName ? labelName : "Face Sets";
1057   PetscInt    dim, Ng = 0;
1058   PetscBool   useCone, useClosure;
1059 
1060   PetscFunctionBegin;
1061   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1062   if (numGhostCells) PetscAssertPointer(numGhostCells, 3);
1063   PetscAssertPointer(dmGhosted, 4);
1064   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &gdm));
1065   PetscCall(DMSetType(gdm, DMPLEX));
1066   PetscCall(DMGetDimension(dm, &dim));
1067   PetscCall(DMSetDimension(gdm, dim));
1068   PetscCall(DMGetBasicAdjacency(dm, &useCone, &useClosure));
1069   PetscCall(DMSetBasicAdjacency(gdm, useCone, useClosure));
1070   PetscCall(DMGetLabel(dm, name, &label));
1071   if (!label) {
1072     /* Get label for boundary faces */
1073     PetscCall(DMCreateLabel(dm, name));
1074     PetscCall(DMGetLabel(dm, name, &label));
1075     PetscCall(DMPlexMarkBoundaryFaces(dm, 1, label));
1076   }
1077   PetscCall(DMPlexConstructGhostCells_Internal(dm, label, &Ng, gdm));
1078   PetscCall(DMCopyDisc(dm, gdm));
1079   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_TRUE, gdm));
1080   gdm->setfromoptionscalled = dm->setfromoptionscalled;
1081   if (numGhostCells) *numGhostCells = Ng;
1082   *dmGhosted = gdm;
1083   PetscFunctionReturn(PETSC_SUCCESS);
1084 }
1085 
1086 static PetscErrorCode DivideCells_Private(DM dm, DMLabel label, DMPlexPointQueue queue)
1087 {
1088   PetscInt dim, d, shift = 100, *pStart, *pEnd;
1089 
1090   PetscFunctionBegin;
1091   PetscCall(DMGetDimension(dm, &dim));
1092   PetscCall(PetscMalloc2(dim, &pStart, dim, &pEnd));
1093   for (d = 0; d < dim; ++d) PetscCall(DMPlexGetDepthStratum(dm, d, &pStart[d], &pEnd[d]));
1094   while (!DMPlexPointQueueEmpty(queue)) {
1095     PetscInt  cell    = -1;
1096     PetscInt *closure = NULL;
1097     PetscInt  closureSize, cl, cval;
1098 
1099     PetscCall(DMPlexPointQueueDequeue(queue, &cell));
1100     PetscCall(DMLabelGetValue(label, cell, &cval));
1101     PetscCall(DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
1102     /* Mark points in the cell closure that touch the fault */
1103     for (d = 0; d < dim; ++d) {
1104       for (cl = 0; cl < closureSize * 2; cl += 2) {
1105         const PetscInt clp = closure[cl];
1106         PetscInt       clval;
1107 
1108         if ((clp < pStart[d]) || (clp >= pEnd[d])) continue;
1109         PetscCall(DMLabelGetValue(label, clp, &clval));
1110         if (clval == -1) {
1111           const PetscInt *cone;
1112           PetscInt        coneSize, c;
1113 
1114           /* If a cone point touches the fault, then this point touches the fault */
1115           PetscCall(DMPlexGetCone(dm, clp, &cone));
1116           PetscCall(DMPlexGetConeSize(dm, clp, &coneSize));
1117           for (c = 0; c < coneSize; ++c) {
1118             PetscInt cpval;
1119 
1120             PetscCall(DMLabelGetValue(label, cone[c], &cpval));
1121             if (cpval != -1) {
1122               PetscInt dep;
1123 
1124               PetscCall(DMPlexGetPointDepth(dm, clp, &dep));
1125               clval = cval < 0 ? -(shift + dep) : shift + dep;
1126               PetscCall(DMLabelSetValue(label, clp, clval));
1127               break;
1128             }
1129           }
1130         }
1131         /* Mark neighbor cells through marked faces (these cells must also touch the fault) */
1132         if (d == dim - 1 && clval != -1) {
1133           const PetscInt *support;
1134           PetscInt        supportSize, s, nval;
1135 
1136           PetscCall(DMPlexGetSupport(dm, clp, &support));
1137           PetscCall(DMPlexGetSupportSize(dm, clp, &supportSize));
1138           for (s = 0; s < supportSize; ++s) {
1139             PetscCall(DMLabelGetValue(label, support[s], &nval));
1140             if (nval == -1) {
1141               PetscCall(DMLabelSetValue(label, support[s], clval < 0 ? clval - 1 : clval + 1));
1142               PetscCall(DMPlexPointQueueEnqueue(queue, support[s]));
1143             }
1144           }
1145         }
1146       }
1147     }
1148     PetscCall(DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
1149   }
1150   PetscCall(PetscFree2(pStart, pEnd));
1151   PetscFunctionReturn(PETSC_SUCCESS);
1152 }
1153 
1154 typedef struct {
1155   DM               dm;
1156   DMPlexPointQueue queue;
1157 } PointDivision;
1158 
1159 static PetscErrorCode divideCell(DMLabel label, PetscInt p, PetscInt val, void *ctx)
1160 {
1161   PointDivision  *div  = (PointDivision *)ctx;
1162   PetscInt        cval = val < 0 ? val - 1 : val + 1;
1163   const PetscInt *support;
1164   PetscInt        supportSize, s;
1165 
1166   PetscFunctionBegin;
1167   PetscCall(DMPlexGetSupport(div->dm, p, &support));
1168   PetscCall(DMPlexGetSupportSize(div->dm, p, &supportSize));
1169   for (s = 0; s < supportSize; ++s) {
1170     PetscCall(DMLabelSetValue(label, support[s], cval));
1171     PetscCall(DMPlexPointQueueEnqueue(div->queue, support[s]));
1172   }
1173   PetscFunctionReturn(PETSC_SUCCESS);
1174 }
1175 
1176 /* Mark cells by label propagation */
1177 static PetscErrorCode DMPlexLabelFaultHalo(DM dm, DMLabel faultLabel)
1178 {
1179   DMPlexPointQueue queue = NULL;
1180   PointDivision    div;
1181   PetscSF          pointSF;
1182   IS               pointIS;
1183   const PetscInt  *points;
1184   PetscBool        empty;
1185   PetscInt         dim, shift = 100, n, i;
1186 
1187   PetscFunctionBegin;
1188   PetscCall(DMGetDimension(dm, &dim));
1189   PetscCall(DMPlexPointQueueCreate(1024, &queue));
1190   div.dm    = dm;
1191   div.queue = queue;
1192   /* Enqueue cells on fault */
1193   PetscCall(DMLabelGetStratumIS(faultLabel, shift + dim, &pointIS));
1194   if (pointIS) {
1195     PetscCall(ISGetLocalSize(pointIS, &n));
1196     PetscCall(ISGetIndices(pointIS, &points));
1197     for (i = 0; i < n; ++i) PetscCall(DMPlexPointQueueEnqueue(queue, points[i]));
1198     PetscCall(ISRestoreIndices(pointIS, &points));
1199     PetscCall(ISDestroy(&pointIS));
1200   }
1201   PetscCall(DMLabelGetStratumIS(faultLabel, -(shift + dim), &pointIS));
1202   if (pointIS) {
1203     PetscCall(ISGetLocalSize(pointIS, &n));
1204     PetscCall(ISGetIndices(pointIS, &points));
1205     for (i = 0; i < n; ++i) PetscCall(DMPlexPointQueueEnqueue(queue, points[i]));
1206     PetscCall(ISRestoreIndices(pointIS, &points));
1207     PetscCall(ISDestroy(&pointIS));
1208   }
1209 
1210   PetscCall(DMGetPointSF(dm, &pointSF));
1211   PetscCall(DMLabelPropagateBegin(faultLabel, pointSF));
1212   /* While edge queue is not empty: */
1213   PetscCall(DMPlexPointQueueEmptyCollective((PetscObject)dm, queue, &empty));
1214   while (!empty) {
1215     PetscCall(DivideCells_Private(dm, faultLabel, queue));
1216     PetscCall(DMLabelPropagatePush(faultLabel, pointSF, divideCell, &div));
1217     PetscCall(DMPlexPointQueueEmptyCollective((PetscObject)dm, queue, &empty));
1218   }
1219   PetscCall(DMLabelPropagateEnd(faultLabel, pointSF));
1220   PetscCall(DMPlexPointQueueDestroy(&queue));
1221   PetscFunctionReturn(PETSC_SUCCESS);
1222 }
1223 
1224 /*
1225   We are adding three kinds of points here:
1226     Replicated:     Copies of points which exist in the mesh, such as vertices identified across a fault
1227     Non-replicated: Points which exist on the fault, but are not replicated
1228     Ghost:          These are shared fault faces which are not owned by this process. These do not produce hybrid cells and do not replicate
1229     Hybrid:         Entirely new points, such as cohesive cells
1230 
1231   When creating subsequent cohesive cells, we shift the old hybrid cells to the end of the numbering at
1232   each depth so that the new split/hybrid points can be inserted as a block.
1233 */
1234 static PetscErrorCode DMPlexConstructCohesiveCells_Internal(DM dm, DMLabel label, DMLabel splitLabel, DM sdm)
1235 {
1236   MPI_Comm         comm;
1237   IS               valueIS;
1238   PetscInt         numSP = 0; /* The number of depths for which we have replicated points */
1239   const PetscInt  *values;    /* List of depths for which we have replicated points */
1240   IS              *splitIS;
1241   IS              *unsplitIS;
1242   IS               ghostIS;
1243   PetscInt        *numSplitPoints;     /* The number of replicated points at each depth */
1244   PetscInt        *numUnsplitPoints;   /* The number of non-replicated points at each depth which still give rise to hybrid points */
1245   PetscInt        *numHybridPoints;    /* The number of new hybrid points at each depth */
1246   PetscInt        *numHybridPointsOld; /* The number of existing hybrid points at each depth */
1247   PetscInt         numGhostPoints;     /* The number of unowned, shared fault faces */
1248   const PetscInt **splitPoints;        /* Replicated points for each depth */
1249   const PetscInt **unsplitPoints;      /* Non-replicated points for each depth */
1250   const PetscInt  *ghostPoints;        /* Ghost fault faces */
1251   PetscSection     coordSection;
1252   Vec              coordinates;
1253   PetscScalar     *coords;
1254   PetscInt        *depthMax;   /* The first hybrid point at each depth in the original mesh */
1255   PetscInt        *depthEnd;   /* The point limit at each depth in the original mesh */
1256   PetscInt        *depthShift; /* Number of replicated+hybrid points at each depth */
1257   PetscInt        *pMaxNew;    /* The first replicated point at each depth in the new mesh, hybrids come after this */
1258   PetscInt        *coneNew, *coneONew, *supportNew;
1259   PetscInt         shift = 100, shift2 = 200, depth = 0, dep, dim, d, sp, maxConeSize, maxSupportSize, maxConeSizeNew, maxSupportSizeNew, numLabels, vStart, vEnd, pEnd, p, v;
1260 
1261   PetscFunctionBegin;
1262   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
1263   PetscCall(DMGetDimension(dm, &dim));
1264   PetscCall(DMPlexGetDepth(dm, &depth));
1265   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
1266   /* We do not want this label automatically computed, instead we compute it here */
1267   PetscCall(DMCreateLabel(sdm, "celltype"));
1268   /* Count split points and add cohesive cells */
1269   PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize));
1270   PetscCall(PetscMalloc5(depth + 1, &depthMax, depth + 1, &depthEnd, 2 * (depth + 1), &depthShift, depth + 1, &pMaxNew, depth + 1, &numHybridPointsOld));
1271   PetscCall(PetscMalloc7(depth + 1, &splitIS, depth + 1, &unsplitIS, depth + 1, &numSplitPoints, depth + 1, &numUnsplitPoints, depth + 1, &numHybridPoints, depth + 1, &splitPoints, depth + 1, &unsplitPoints));
1272   for (d = 0; d <= depth; ++d) {
1273     PetscCall(DMPlexGetDepthStratum(dm, d, NULL, &pMaxNew[d]));
1274     PetscCall(DMPlexGetTensorPrismBounds_Internal(dm, d, &depthMax[d], NULL));
1275     depthEnd[d]           = pMaxNew[d];
1276     depthMax[d]           = depthMax[d] < 0 ? depthEnd[d] : depthMax[d];
1277     numSplitPoints[d]     = 0;
1278     numUnsplitPoints[d]   = 0;
1279     numHybridPoints[d]    = 0;
1280     numHybridPointsOld[d] = depthMax[d] < 0 ? 0 : depthEnd[d] - depthMax[d];
1281     splitPoints[d]        = NULL;
1282     unsplitPoints[d]      = NULL;
1283     splitIS[d]            = NULL;
1284     unsplitIS[d]          = NULL;
1285     /* we are shifting the existing hybrid points with the stratum behind them, so
1286      * the split comes at the end of the normal points, i.e., at depthMax[d] */
1287     depthShift[2 * d]     = depthMax[d];
1288     depthShift[2 * d + 1] = 0;
1289   }
1290   numGhostPoints = 0;
1291   ghostPoints    = NULL;
1292   if (label) {
1293     PetscCall(DMLabelGetValueIS(label, &valueIS));
1294     PetscCall(ISGetLocalSize(valueIS, &numSP));
1295     PetscCall(ISGetIndices(valueIS, &values));
1296   }
1297   for (sp = 0; sp < numSP; ++sp) {
1298     const PetscInt dep = values[sp];
1299 
1300     if ((dep < 0) || (dep > depth)) continue;
1301     PetscCall(DMLabelGetStratumIS(label, dep, &splitIS[dep]));
1302     if (splitIS[dep]) {
1303       PetscCall(ISGetLocalSize(splitIS[dep], &numSplitPoints[dep]));
1304       PetscCall(ISGetIndices(splitIS[dep], &splitPoints[dep]));
1305     }
1306     PetscCall(DMLabelGetStratumIS(label, shift2 + dep, &unsplitIS[dep]));
1307     if (unsplitIS[dep]) {
1308       PetscCall(ISGetLocalSize(unsplitIS[dep], &numUnsplitPoints[dep]));
1309       PetscCall(ISGetIndices(unsplitIS[dep], &unsplitPoints[dep]));
1310     }
1311   }
1312   PetscCall(DMLabelGetStratumIS(label, shift2 + dim - 1, &ghostIS));
1313   if (ghostIS) {
1314     PetscCall(ISGetLocalSize(ghostIS, &numGhostPoints));
1315     PetscCall(ISGetIndices(ghostIS, &ghostPoints));
1316   }
1317   /* Calculate number of hybrid points */
1318   for (d = 1; d <= depth; ++d) numHybridPoints[d] = numSplitPoints[d - 1] + numUnsplitPoints[d - 1]; /* There is a hybrid cell/face/edge for every split face/edge/vertex   */
1319   for (d = 0; d <= depth; ++d) depthShift[2 * d + 1] = numSplitPoints[d] + numHybridPoints[d];
1320   PetscCall(DMPlexShiftPointSetUp_Internal(depth, depthShift));
1321   /* the end of the points in this stratum that come before the new points:
1322    * shifting pMaxNew[d] gets the new start of the next stratum, then count back the old hybrid points and the newly
1323    * added points */
1324   for (d = 0; d <= depth; ++d) pMaxNew[d] = DMPlexShiftPoint_Internal(pMaxNew[d], depth, depthShift) - (numHybridPointsOld[d] + numSplitPoints[d] + numHybridPoints[d]);
1325   PetscCall(DMPlexShiftSizes_Internal(dm, depthShift, sdm));
1326   /* Step 3: Set cone/support sizes for new points */
1327   for (dep = 0; dep <= depth; ++dep) {
1328     for (p = 0; p < numSplitPoints[dep]; ++p) {
1329       const PetscInt  oldp   = splitPoints[dep][p];
1330       const PetscInt  newp   = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*oldp + depthOffset[dep]*/;
1331       const PetscInt  splitp = p + pMaxNew[dep];
1332       const PetscInt *support;
1333       DMPolytopeType  ct;
1334       PetscInt        coneSize, supportSize, qf, qn, qp, e;
1335 
1336       PetscCall(DMPlexGetConeSize(dm, oldp, &coneSize));
1337       PetscCall(DMPlexSetConeSize(sdm, splitp, coneSize));
1338       PetscCall(DMPlexGetSupportSize(dm, oldp, &supportSize));
1339       PetscCall(DMPlexSetSupportSize(sdm, splitp, supportSize));
1340       PetscCall(DMPlexGetCellType(dm, oldp, &ct));
1341       PetscCall(DMPlexSetCellType(sdm, splitp, ct));
1342       if (dep == depth - 1) {
1343         const PetscInt hybcell = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1];
1344 
1345         /* Add cohesive cells, they are prisms */
1346         PetscCall(DMPlexSetConeSize(sdm, hybcell, 2 + coneSize));
1347         switch (coneSize) {
1348         case 2:
1349           PetscCall(DMPlexSetCellType(sdm, hybcell, DM_POLYTOPE_SEG_PRISM_TENSOR));
1350           break;
1351         case 3:
1352           PetscCall(DMPlexSetCellType(sdm, hybcell, DM_POLYTOPE_TRI_PRISM_TENSOR));
1353           break;
1354         case 4:
1355           PetscCall(DMPlexSetCellType(sdm, hybcell, DM_POLYTOPE_QUAD_PRISM_TENSOR));
1356           break;
1357         }
1358         /* Shared fault faces with only one support cell now have two with the cohesive cell */
1359         /*   TODO Check thaat oldp has rootdegree == 1 */
1360         if (supportSize == 1) {
1361           const PetscInt *support;
1362           PetscInt        val;
1363 
1364           PetscCall(DMPlexGetSupport(dm, oldp, &support));
1365           PetscCall(DMLabelGetValue(label, support[0], &val));
1366           if (val < 0) PetscCall(DMPlexSetSupportSize(sdm, splitp, 2));
1367           else PetscCall(DMPlexSetSupportSize(sdm, newp, 2));
1368         }
1369       } else if (dep == 0) {
1370         const PetscInt hybedge = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1];
1371 
1372         PetscCall(DMPlexGetSupport(dm, oldp, &support));
1373         for (e = 0, qn = 0, qp = 0, qf = 0; e < supportSize; ++e) {
1374           PetscInt val;
1375 
1376           PetscCall(DMLabelGetValue(label, support[e], &val));
1377           if (val == 1) ++qf;
1378           if ((val == 1) || (val == (shift + 1))) ++qn;
1379           if ((val == 1) || (val == -(shift + 1))) ++qp;
1380         }
1381         /* Split old vertex: Edges into original vertex and new cohesive edge */
1382         PetscCall(DMPlexSetSupportSize(sdm, newp, qn + 1));
1383         /* Split new vertex: Edges into split vertex and new cohesive edge */
1384         PetscCall(DMPlexSetSupportSize(sdm, splitp, qp + 1));
1385         /* Add hybrid edge */
1386         PetscCall(DMPlexSetConeSize(sdm, hybedge, 2));
1387         PetscCall(DMPlexSetSupportSize(sdm, hybedge, qf));
1388         PetscCall(DMPlexSetCellType(sdm, hybedge, DM_POLYTOPE_POINT_PRISM_TENSOR));
1389       } else if (dep == dim - 2) {
1390         const PetscInt hybface = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1];
1391 
1392         PetscCall(DMPlexGetSupport(dm, oldp, &support));
1393         for (e = 0, qn = 0, qp = 0, qf = 0; e < supportSize; ++e) {
1394           PetscInt val;
1395 
1396           PetscCall(DMLabelGetValue(label, support[e], &val));
1397           if (val == dim - 1) ++qf;
1398           if ((val == dim - 1) || (val == (shift + dim - 1))) ++qn;
1399           if ((val == dim - 1) || (val == -(shift + dim - 1))) ++qp;
1400         }
1401         /* Split old edge: Faces into original edge and cohesive face (positive side?) */
1402         PetscCall(DMPlexSetSupportSize(sdm, newp, qn + 1));
1403         /* Split new edge: Faces into split edge and cohesive face (negative side?) */
1404         PetscCall(DMPlexSetSupportSize(sdm, splitp, qp + 1));
1405         /* Add hybrid face */
1406         PetscCall(DMPlexSetConeSize(sdm, hybface, 4));
1407         PetscCall(DMPlexSetSupportSize(sdm, hybface, qf));
1408         PetscCall(DMPlexSetCellType(sdm, hybface, DM_POLYTOPE_SEG_PRISM_TENSOR));
1409       }
1410     }
1411   }
1412   for (dep = 0; dep <= depth; ++dep) {
1413     for (p = 0; p < numUnsplitPoints[dep]; ++p) {
1414       const PetscInt  oldp = unsplitPoints[dep][p];
1415       const PetscInt  newp = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*oldp + depthOffset[dep]*/;
1416       const PetscInt *support;
1417       PetscInt        coneSize, supportSize, qf, e, s;
1418 
1419       PetscCall(DMPlexGetConeSize(dm, oldp, &coneSize));
1420       PetscCall(DMPlexGetSupportSize(dm, oldp, &supportSize));
1421       PetscCall(DMPlexGetSupport(dm, oldp, &support));
1422       if (dep == 0) {
1423         const PetscInt hybedge = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1] + numSplitPoints[dep];
1424 
1425         /* Unsplit vertex: Edges into original vertex, split edges, and new cohesive edge twice */
1426         for (s = 0, qf = 0; s < supportSize; ++s, ++qf) {
1427           PetscCall(PetscFindInt(support[s], numSplitPoints[dep + 1], splitPoints[dep + 1], &e));
1428           if (e >= 0) ++qf;
1429         }
1430         PetscCall(DMPlexSetSupportSize(sdm, newp, qf + 2));
1431         /* Add hybrid edge */
1432         PetscCall(DMPlexSetConeSize(sdm, hybedge, 2));
1433         for (e = 0, qf = 0; e < supportSize; ++e) {
1434           PetscInt val;
1435 
1436           PetscCall(DMLabelGetValue(label, support[e], &val));
1437           /* Split and unsplit edges produce hybrid faces */
1438           if (val == 1) ++qf;
1439           if (val == (shift2 + 1)) ++qf;
1440         }
1441         PetscCall(DMPlexSetSupportSize(sdm, hybedge, qf));
1442         PetscCall(DMPlexSetCellType(sdm, hybedge, DM_POLYTOPE_POINT_PRISM_TENSOR));
1443       } else if (dep == dim - 2) {
1444         const PetscInt hybface = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1] + numSplitPoints[dep];
1445         PetscInt       val;
1446 
1447         for (e = 0, qf = 0; e < supportSize; ++e) {
1448           PetscCall(DMLabelGetValue(label, support[e], &val));
1449           if (val == dim - 1) qf += 2;
1450           else ++qf;
1451         }
1452         /* Unsplit edge: Faces into original edge, split face, and cohesive face twice */
1453         PetscCall(DMPlexSetSupportSize(sdm, newp, qf + 2));
1454         /* Add hybrid face */
1455         for (e = 0, qf = 0; e < supportSize; ++e) {
1456           PetscCall(DMLabelGetValue(label, support[e], &val));
1457           if (val == dim - 1) ++qf;
1458         }
1459         PetscCall(DMPlexSetConeSize(sdm, hybface, 4));
1460         PetscCall(DMPlexSetSupportSize(sdm, hybface, qf));
1461         PetscCall(DMPlexSetCellType(sdm, hybface, DM_POLYTOPE_SEG_PRISM_TENSOR));
1462       }
1463     }
1464   }
1465   /* Step 4: Setup split DM */
1466   PetscCall(DMSetUp(sdm));
1467   PetscCall(DMPlexShiftPoints_Internal(dm, depthShift, sdm));
1468   PetscCall(DMPlexGetMaxSizes(sdm, &maxConeSizeNew, &maxSupportSizeNew));
1469   PetscCall(PetscMalloc3(PetscMax(maxConeSize, maxConeSizeNew) * 3, &coneNew, PetscMax(maxConeSize, maxConeSizeNew) * 3, &coneONew, PetscMax(maxSupportSize, maxSupportSizeNew), &supportNew));
1470   /* Step 6: Set cones and supports for new points */
1471   for (dep = 0; dep <= depth; ++dep) {
1472     for (p = 0; p < numSplitPoints[dep]; ++p) {
1473       const PetscInt  oldp   = splitPoints[dep][p];
1474       const PetscInt  newp   = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*oldp + depthOffset[dep]*/;
1475       const PetscInt  splitp = p + pMaxNew[dep];
1476       const PetscInt *cone, *support, *ornt;
1477       DMPolytopeType  ct;
1478       PetscInt        coneSize, supportSize, q, qf, qn, qp, v, e, s;
1479 
1480       PetscCall(DMPlexGetCellType(dm, oldp, &ct));
1481       PetscCall(DMPlexGetConeSize(dm, oldp, &coneSize));
1482       PetscCall(DMPlexGetCone(dm, oldp, &cone));
1483       PetscCall(DMPlexGetConeOrientation(dm, oldp, &ornt));
1484       PetscCall(DMPlexGetSupportSize(dm, oldp, &supportSize));
1485       PetscCall(DMPlexGetSupport(dm, oldp, &support));
1486       if (dep == depth - 1) {
1487         PetscBool       hasUnsplit = PETSC_FALSE;
1488         const PetscInt  hybcell    = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1];
1489         const PetscInt *supportF;
1490 
1491         coneONew[0] = coneONew[1] = -1000;
1492         /* Split face:       copy in old face to new face to start */
1493         PetscCall(DMPlexGetSupport(sdm, newp, &supportF));
1494         PetscCall(DMPlexSetSupport(sdm, splitp, supportF));
1495         /* Split old face:   old vertices/edges in cone so no change */
1496         /* Split new face:   new vertices/edges in cone */
1497         for (q = 0; q < coneSize; ++q) {
1498           PetscCall(PetscFindInt(cone[q], numSplitPoints[dep - 1], splitPoints[dep - 1], &v));
1499           if (v < 0) {
1500             PetscCall(PetscFindInt(cone[q], numUnsplitPoints[dep - 1], unsplitPoints[dep - 1], &v));
1501             PetscCheck(v >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %" PetscInt_FMT " in split or unsplit points of depth %" PetscInt_FMT, cone[q], dep - 1);
1502             coneNew[2 + q] = DMPlexShiftPoint_Internal(cone[q], depth, depthShift) /*cone[q] + depthOffset[dep-1]*/;
1503             hasUnsplit     = PETSC_TRUE;
1504           } else {
1505             coneNew[2 + q] = v + pMaxNew[dep - 1];
1506             if (dep > 1) {
1507               const PetscInt *econe;
1508               PetscInt        econeSize, r, vs, vu;
1509 
1510               PetscCall(DMPlexGetConeSize(dm, cone[q], &econeSize));
1511               PetscCall(DMPlexGetCone(dm, cone[q], &econe));
1512               for (r = 0; r < econeSize; ++r) {
1513                 PetscCall(PetscFindInt(econe[r], numSplitPoints[dep - 2], splitPoints[dep - 2], &vs));
1514                 PetscCall(PetscFindInt(econe[r], numUnsplitPoints[dep - 2], unsplitPoints[dep - 2], &vu));
1515                 if (vs >= 0) continue;
1516                 PetscCheck(vu >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %" PetscInt_FMT " in split or unsplit points of depth %" PetscInt_FMT, econe[r], dep - 2);
1517                 hasUnsplit = PETSC_TRUE;
1518               }
1519             }
1520           }
1521         }
1522         PetscCall(DMPlexSetCone(sdm, splitp, &coneNew[2]));
1523         PetscCall(DMPlexSetConeOrientation(sdm, splitp, ornt));
1524         /* Face support */
1525         PetscInt vals[2];
1526 
1527         PetscCall(DMLabelGetValue(label, support[0], &vals[0]));
1528         if (supportSize > 1) PetscCall(DMLabelGetValue(label, support[1], &vals[1]));
1529         else vals[1] = -vals[0];
1530         PetscCheck(vals[0] * vals[1] < 0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid support labels %" PetscInt_FMT " %" PetscInt_FMT, vals[0], vals[1]);
1531 
1532         for (s = 0; s < 2; ++s) {
1533           if (s >= supportSize) {
1534             if (vals[s] < 0) {
1535               /* Ghost old face:   Replace negative side cell with cohesive cell */
1536               PetscCall(DMPlexInsertSupport(sdm, newp, s, hybcell));
1537             } else {
1538               /* Ghost new face:   Replace positive side cell with cohesive cell */
1539               PetscCall(DMPlexInsertSupport(sdm, splitp, s, hybcell));
1540             }
1541           } else {
1542             if (vals[s] < 0) {
1543               /* Split old face:   Replace negative side cell with cohesive cell */
1544               PetscCall(DMPlexInsertSupport(sdm, newp, s, hybcell));
1545             } else {
1546               /* Split new face:   Replace positive side cell with cohesive cell */
1547               PetscCall(DMPlexInsertSupport(sdm, splitp, s, hybcell));
1548             }
1549           }
1550         }
1551         /* Get orientation for cohesive face using the positive side cell */
1552         {
1553           const PetscInt *ncone, *nconeO;
1554           PetscInt        nconeSize, nc, ocell;
1555           PetscBool       flip = PETSC_FALSE;
1556 
1557           if (supportSize > 1) {
1558             ocell = vals[0] < 0 ? support[1] : support[0];
1559           } else {
1560             ocell = support[0];
1561             flip  = vals[0] < 0 ? PETSC_TRUE : PETSC_FALSE;
1562           }
1563           PetscCall(DMPlexGetConeSize(dm, ocell, &nconeSize));
1564           PetscCall(DMPlexGetCone(dm, ocell, &ncone));
1565           PetscCall(DMPlexGetConeOrientation(dm, ocell, &nconeO));
1566           for (nc = 0; nc < nconeSize; ++nc) {
1567             if (ncone[nc] == oldp) {
1568               coneONew[0] = flip ? -(nconeO[nc] + 1) : nconeO[nc];
1569               break;
1570             }
1571           }
1572           PetscCheck(nc < nconeSize, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate face %" PetscInt_FMT " in neighboring cell %" PetscInt_FMT, oldp, ocell);
1573         }
1574         /* Cohesive cell:    Old and new split face, then new cohesive faces */
1575         {
1576           const PetscInt No = DMPolytopeTypeGetNumArrangments(ct) / 2;
1577           PetscCheck((coneONew[0] >= -No) && (coneONew[0] < No), PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid %s orientation %" PetscInt_FMT, DMPolytopeTypes[ct], coneONew[0]);
1578         }
1579         const PetscInt *arr = DMPolytopeTypeGetArrangment(ct, coneONew[0]);
1580 
1581         coneNew[0]  = newp; /* Extracted negative side orientation above */
1582         coneNew[1]  = splitp;
1583         coneONew[1] = coneONew[0];
1584         for (q = 0; q < coneSize; ++q) {
1585           /* Hybrid faces must follow order from oriented end face */
1586           const PetscInt qa = arr[q * 2 + 0];
1587           const PetscInt qo = arr[q * 2 + 1];
1588           DMPolytopeType ft = dep == 2 ? DM_POLYTOPE_SEGMENT : DM_POLYTOPE_POINT;
1589 
1590           PetscCall(PetscFindInt(cone[qa], numSplitPoints[dep - 1], splitPoints[dep - 1], &v));
1591           if (v < 0) {
1592             PetscCall(PetscFindInt(cone[qa], numUnsplitPoints[dep - 1], unsplitPoints[dep - 1], &v));
1593             coneNew[2 + q] = v + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep - 1];
1594           } else {
1595             coneNew[2 + q] = v + pMaxNew[dep] + numSplitPoints[dep];
1596           }
1597           coneONew[2 + q] = DMPolytopeTypeComposeOrientation(ft, qo, ornt[qa]);
1598         }
1599         PetscCall(DMPlexSetCone(sdm, hybcell, coneNew));
1600         PetscCall(DMPlexSetConeOrientation(sdm, hybcell, coneONew));
1601         /* Label the hybrid cells on the boundary of the split */
1602         if (hasUnsplit) PetscCall(DMLabelSetValue(label, -hybcell, dim));
1603       } else if (dep == 0) {
1604         const PetscInt hybedge = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1];
1605 
1606         /* Split old vertex: Edges in old split faces and new cohesive edge */
1607         for (e = 0, qn = 0; e < supportSize; ++e) {
1608           PetscInt val;
1609 
1610           PetscCall(DMLabelGetValue(label, support[e], &val));
1611           if ((val == 1) || (val == (shift + 1))) supportNew[qn++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1612         }
1613         supportNew[qn] = hybedge;
1614         PetscCall(DMPlexSetSupport(sdm, newp, supportNew));
1615         /* Split new vertex: Edges in new split faces and new cohesive edge */
1616         for (e = 0, qp = 0; e < supportSize; ++e) {
1617           PetscInt val, edge;
1618 
1619           PetscCall(DMLabelGetValue(label, support[e], &val));
1620           if (val == 1) {
1621             PetscCall(PetscFindInt(support[e], numSplitPoints[dep + 1], splitPoints[dep + 1], &edge));
1622             PetscCheck(edge >= 0, comm, PETSC_ERR_ARG_WRONG, "Edge %" PetscInt_FMT " is not a split edge", support[e]);
1623             supportNew[qp++] = edge + pMaxNew[dep + 1];
1624           } else if (val == -(shift + 1)) {
1625             supportNew[qp++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1626           }
1627         }
1628         supportNew[qp] = hybedge;
1629         PetscCall(DMPlexSetSupport(sdm, splitp, supportNew));
1630         /* Hybrid edge:    Old and new split vertex */
1631         coneNew[0] = newp;
1632         coneNew[1] = splitp;
1633         PetscCall(DMPlexSetCone(sdm, hybedge, coneNew));
1634         for (e = 0, qf = 0; e < supportSize; ++e) {
1635           PetscInt val, edge;
1636 
1637           PetscCall(DMLabelGetValue(label, support[e], &val));
1638           if (val == 1) {
1639             PetscCall(PetscFindInt(support[e], numSplitPoints[dep + 1], splitPoints[dep + 1], &edge));
1640             PetscCheck(edge >= 0, comm, PETSC_ERR_ARG_WRONG, "Edge %" PetscInt_FMT " is not a split edge", support[e]);
1641             supportNew[qf++] = edge + pMaxNew[dep + 2] + numSplitPoints[dep + 2];
1642           }
1643         }
1644         PetscCall(DMPlexSetSupport(sdm, hybedge, supportNew));
1645       } else if (dep == dim - 2) {
1646         const PetscInt hybface = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1];
1647 
1648         /* Split old edge:   old vertices in cone so no change */
1649         /* Split new edge:   new vertices in cone */
1650         for (q = 0; q < coneSize; ++q) {
1651           PetscCall(PetscFindInt(cone[q], numSplitPoints[dep - 1], splitPoints[dep - 1], &v));
1652           if (v < 0) {
1653             PetscCall(PetscFindInt(cone[q], numUnsplitPoints[dep - 1], unsplitPoints[dep - 1], &v));
1654             PetscCheck(v >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %" PetscInt_FMT " in split or unsplit points of depth %" PetscInt_FMT, cone[q], dep - 1);
1655             coneNew[q] = DMPlexShiftPoint_Internal(cone[q], depth, depthShift) /*cone[q] + depthOffset[dep-1]*/;
1656           } else {
1657             coneNew[q] = v + pMaxNew[dep - 1];
1658           }
1659         }
1660         PetscCall(DMPlexSetCone(sdm, splitp, coneNew));
1661         /* Split old edge: Faces in positive side cells and old split faces */
1662         for (e = 0, q = 0; e < supportSize; ++e) {
1663           PetscInt val;
1664 
1665           PetscCall(DMLabelGetValue(label, support[e], &val));
1666           if (val == dim - 1) {
1667             supportNew[q++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1668           } else if (val == (shift + dim - 1)) {
1669             supportNew[q++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1670           }
1671         }
1672         supportNew[q++] = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1];
1673         PetscCall(DMPlexSetSupport(sdm, newp, supportNew));
1674         /* Split new edge: Faces in negative side cells and new split faces */
1675         for (e = 0, q = 0; e < supportSize; ++e) {
1676           PetscInt val, face;
1677 
1678           PetscCall(DMLabelGetValue(label, support[e], &val));
1679           if (val == dim - 1) {
1680             PetscCall(PetscFindInt(support[e], numSplitPoints[dep + 1], splitPoints[dep + 1], &face));
1681             PetscCheck(face >= 0, comm, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " is not a split face", support[e]);
1682             supportNew[q++] = face + pMaxNew[dep + 1];
1683           } else if (val == -(shift + dim - 1)) {
1684             supportNew[q++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1685           }
1686         }
1687         supportNew[q++] = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1];
1688         PetscCall(DMPlexSetSupport(sdm, splitp, supportNew));
1689         /* Hybrid face */
1690         coneNew[0] = newp;
1691         coneNew[1] = splitp;
1692         for (v = 0; v < coneSize; ++v) {
1693           PetscInt vertex;
1694           PetscCall(PetscFindInt(cone[v], numSplitPoints[dep - 1], splitPoints[dep - 1], &vertex));
1695           if (vertex < 0) {
1696             PetscCall(PetscFindInt(cone[v], numUnsplitPoints[dep - 1], unsplitPoints[dep - 1], &vertex));
1697             PetscCheck(vertex >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %" PetscInt_FMT " in split or unsplit points of depth %" PetscInt_FMT, cone[v], dep - 1);
1698             coneNew[2 + v] = vertex + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep - 1];
1699           } else {
1700             coneNew[2 + v] = vertex + pMaxNew[dep] + numSplitPoints[dep];
1701           }
1702         }
1703         PetscCall(DMPlexSetCone(sdm, hybface, coneNew));
1704         for (e = 0, qf = 0; e < supportSize; ++e) {
1705           PetscInt val, face;
1706 
1707           PetscCall(DMLabelGetValue(label, support[e], &val));
1708           if (val == dim - 1) {
1709             PetscCall(PetscFindInt(support[e], numSplitPoints[dep + 1], splitPoints[dep + 1], &face));
1710             PetscCheck(face >= 0, comm, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " is not a split face", support[e]);
1711             supportNew[qf++] = face + pMaxNew[dep + 2] + numSplitPoints[dep + 2];
1712           }
1713         }
1714         PetscCall(DMPlexSetSupport(sdm, hybface, supportNew));
1715       }
1716     }
1717   }
1718   for (dep = 0; dep <= depth; ++dep) {
1719     for (p = 0; p < numUnsplitPoints[dep]; ++p) {
1720       const PetscInt  oldp = unsplitPoints[dep][p];
1721       const PetscInt  newp = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*oldp + depthOffset[dep]*/;
1722       const PetscInt *cone, *support;
1723       PetscInt        coneSize, supportSize, supportSizeNew, q, qf, e, f, s;
1724 
1725       PetscCall(DMPlexGetConeSize(dm, oldp, &coneSize));
1726       PetscCall(DMPlexGetCone(dm, oldp, &cone));
1727       PetscCall(DMPlexGetSupportSize(dm, oldp, &supportSize));
1728       PetscCall(DMPlexGetSupport(dm, oldp, &support));
1729       if (dep == 0) {
1730         const PetscInt hybedge = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1] + numSplitPoints[dep];
1731 
1732         /* Unsplit vertex */
1733         PetscCall(DMPlexGetSupportSize(sdm, newp, &supportSizeNew));
1734         for (s = 0, q = 0; s < supportSize; ++s) {
1735           supportNew[q++] = DMPlexShiftPoint_Internal(support[s], depth, depthShift) /*support[s] + depthOffset[dep+1]*/;
1736           PetscCall(PetscFindInt(support[s], numSplitPoints[dep + 1], splitPoints[dep + 1], &e));
1737           if (e >= 0) supportNew[q++] = e + pMaxNew[dep + 1];
1738         }
1739         supportNew[q++] = hybedge;
1740         supportNew[q++] = hybedge;
1741         PetscCheck(q == supportSizeNew, comm, PETSC_ERR_ARG_WRONG, "Support size %" PetscInt_FMT " != %" PetscInt_FMT " for vertex %" PetscInt_FMT, q, supportSizeNew, newp);
1742         PetscCall(DMPlexSetSupport(sdm, newp, supportNew));
1743         /* Hybrid edge */
1744         coneNew[0] = newp;
1745         coneNew[1] = newp;
1746         PetscCall(DMPlexSetCone(sdm, hybedge, coneNew));
1747         for (e = 0, qf = 0; e < supportSize; ++e) {
1748           PetscInt val, edge;
1749 
1750           PetscCall(DMLabelGetValue(label, support[e], &val));
1751           if (val == 1) {
1752             PetscCall(PetscFindInt(support[e], numSplitPoints[dep + 1], splitPoints[dep + 1], &edge));
1753             PetscCheck(edge >= 0, comm, PETSC_ERR_ARG_WRONG, "Edge %" PetscInt_FMT " is not a split edge", support[e]);
1754             supportNew[qf++] = edge + pMaxNew[dep + 2] + numSplitPoints[dep + 2];
1755           } else if (val == (shift2 + 1)) {
1756             PetscCall(PetscFindInt(support[e], numUnsplitPoints[dep + 1], unsplitPoints[dep + 1], &edge));
1757             PetscCheck(edge >= 0, comm, PETSC_ERR_ARG_WRONG, "Edge %" PetscInt_FMT " is not a unsplit edge", support[e]);
1758             supportNew[qf++] = edge + pMaxNew[dep + 2] + numSplitPoints[dep + 2] + numSplitPoints[dep + 1];
1759           }
1760         }
1761         PetscCall(DMPlexSetSupport(sdm, hybedge, supportNew));
1762       } else if (dep == dim - 2) {
1763         const PetscInt hybface = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1] + numSplitPoints[dep];
1764 
1765         /* Unsplit edge: Faces into original edge, split face, and hybrid face twice */
1766         for (f = 0, qf = 0; f < supportSize; ++f) {
1767           PetscInt val, face;
1768 
1769           PetscCall(DMLabelGetValue(label, support[f], &val));
1770           if (val == dim - 1) {
1771             PetscCall(PetscFindInt(support[f], numSplitPoints[dep + 1], splitPoints[dep + 1], &face));
1772             PetscCheck(face >= 0, comm, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " is not a split face", support[f]);
1773             supportNew[qf++] = DMPlexShiftPoint_Internal(support[f], depth, depthShift) /*support[f] + depthOffset[dep+1]*/;
1774             supportNew[qf++] = face + pMaxNew[dep + 1];
1775           } else {
1776             supportNew[qf++] = DMPlexShiftPoint_Internal(support[f], depth, depthShift) /*support[f] + depthOffset[dep+1]*/;
1777           }
1778         }
1779         supportNew[qf++] = hybface;
1780         supportNew[qf++] = hybface;
1781         PetscCall(DMPlexGetSupportSize(sdm, newp, &supportSizeNew));
1782         PetscCheck(qf == supportSizeNew, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Support size for unsplit edge %" PetscInt_FMT " is %" PetscInt_FMT " != %" PetscInt_FMT, newp, qf, supportSizeNew);
1783         PetscCall(DMPlexSetSupport(sdm, newp, supportNew));
1784         /* Add hybrid face */
1785         coneNew[0] = newp;
1786         coneNew[1] = newp;
1787         PetscCall(PetscFindInt(cone[0], numUnsplitPoints[dep - 1], unsplitPoints[dep - 1], &v));
1788         PetscCheck(v >= 0, comm, PETSC_ERR_ARG_WRONG, "Vertex %" PetscInt_FMT " is not an unsplit vertex", cone[0]);
1789         coneNew[2] = v + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep - 1];
1790         PetscCall(PetscFindInt(cone[1], numUnsplitPoints[dep - 1], unsplitPoints[dep - 1], &v));
1791         PetscCheck(v >= 0, comm, PETSC_ERR_ARG_WRONG, "Vertex %" PetscInt_FMT " is not an unsplit vertex", cone[1]);
1792         coneNew[3] = v + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep - 1];
1793         PetscCall(DMPlexSetCone(sdm, hybface, coneNew));
1794         for (f = 0, qf = 0; f < supportSize; ++f) {
1795           PetscInt val, face;
1796 
1797           PetscCall(DMLabelGetValue(label, support[f], &val));
1798           if (val == dim - 1) {
1799             PetscCall(PetscFindInt(support[f], numSplitPoints[dep + 1], splitPoints[dep + 1], &face));
1800             supportNew[qf++] = face + pMaxNew[dep + 2] + numSplitPoints[dep + 2];
1801           }
1802         }
1803         PetscCall(DMPlexGetSupportSize(sdm, hybface, &supportSizeNew));
1804         PetscCheck(qf == supportSizeNew, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Support size for hybrid face %" PetscInt_FMT " is %" PetscInt_FMT " != %" PetscInt_FMT, hybface, qf, supportSizeNew);
1805         PetscCall(DMPlexSetSupport(sdm, hybface, supportNew));
1806       }
1807     }
1808   }
1809   /* Step 6b: Replace split points in negative side cones */
1810   for (sp = 0; sp < numSP; ++sp) {
1811     PetscInt        dep = values[sp];
1812     IS              pIS;
1813     PetscInt        numPoints;
1814     const PetscInt *points;
1815 
1816     if (dep >= 0) continue;
1817     PetscCall(DMLabelGetStratumIS(label, dep, &pIS));
1818     if (!pIS) continue;
1819     dep = -dep - shift;
1820     PetscCall(ISGetLocalSize(pIS, &numPoints));
1821     PetscCall(ISGetIndices(pIS, &points));
1822     for (p = 0; p < numPoints; ++p) {
1823       const PetscInt  oldp = points[p];
1824       const PetscInt  newp = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*depthOffset[dep] + oldp*/;
1825       const PetscInt *cone;
1826       PetscInt        coneSize, c;
1827       /* PetscBool       replaced = PETSC_FALSE; */
1828 
1829       /* Negative edge: replace split vertex */
1830       /* Negative cell: replace split face */
1831       PetscCall(DMPlexGetConeSize(sdm, newp, &coneSize));
1832       PetscCall(DMPlexGetCone(sdm, newp, &cone));
1833       for (c = 0; c < coneSize; ++c) {
1834         const PetscInt coldp = DMPlexShiftPointInverse_Internal(cone[c], depth, depthShift);
1835         PetscInt       csplitp, cp, val;
1836 
1837         PetscCall(DMLabelGetValue(label, coldp, &val));
1838         if (val == dep - 1) {
1839           PetscCall(PetscFindInt(coldp, numSplitPoints[dep - 1], splitPoints[dep - 1], &cp));
1840           PetscCheck(cp >= 0, comm, PETSC_ERR_ARG_WRONG, "Point %" PetscInt_FMT " is not a split point of dimension %" PetscInt_FMT, oldp, dep - 1);
1841           csplitp = pMaxNew[dep - 1] + cp;
1842           PetscCall(DMPlexInsertCone(sdm, newp, c, csplitp));
1843           /* replaced = PETSC_TRUE; */
1844         }
1845       }
1846       /* Cells with only a vertex or edge on the submesh have no replacement */
1847       /* PetscCheck(replaced,comm, PETSC_ERR_ARG_WRONG, "The cone of point %d does not contain split points", oldp); */
1848     }
1849     PetscCall(ISRestoreIndices(pIS, &points));
1850     PetscCall(ISDestroy(&pIS));
1851   }
1852   PetscCall(DMPlexReorderCohesiveSupports(sdm));
1853   /* Step 7: Coordinates */
1854   PetscCall(DMPlexShiftCoordinates_Internal(dm, depthShift, sdm));
1855   PetscCall(DMGetCoordinateSection(sdm, &coordSection));
1856   PetscCall(DMGetCoordinatesLocal(sdm, &coordinates));
1857   PetscCall(VecGetArray(coordinates, &coords));
1858   for (v = 0; v < (numSplitPoints ? numSplitPoints[0] : 0); ++v) {
1859     const PetscInt newp   = DMPlexShiftPoint_Internal(splitPoints[0][v], depth, depthShift) /*depthOffset[0] + splitPoints[0][v]*/;
1860     const PetscInt splitp = pMaxNew[0] + v;
1861     PetscInt       dof, off, soff, d;
1862 
1863     PetscCall(PetscSectionGetDof(coordSection, newp, &dof));
1864     PetscCall(PetscSectionGetOffset(coordSection, newp, &off));
1865     PetscCall(PetscSectionGetOffset(coordSection, splitp, &soff));
1866     for (d = 0; d < dof; ++d) coords[soff + d] = coords[off + d];
1867   }
1868   PetscCall(VecRestoreArray(coordinates, &coords));
1869   /* Step 8: SF, if I can figure this out we can split the mesh in parallel */
1870   PetscCall(DMPlexShiftSF_Internal(dm, depthShift, sdm));
1871   /*   TODO We need to associate the ghost points with the correct replica */
1872   /* Step 9: Labels */
1873   PetscCall(DMPlexShiftLabels_Internal(dm, depthShift, sdm));
1874   PetscCall(DMPlexCreateVTKLabel_Internal(dm, PETSC_FALSE, sdm));
1875   PetscCall(DMGetNumLabels(sdm, &numLabels));
1876   for (dep = 0; dep <= depth; ++dep) {
1877     for (p = 0; p < numSplitPoints[dep]; ++p) {
1878       const PetscInt newp   = DMPlexShiftPoint_Internal(splitPoints[dep][p], depth, depthShift) /*depthOffset[dep] + splitPoints[dep][p]*/;
1879       const PetscInt splitp = pMaxNew[dep] + p;
1880       PetscInt       l;
1881 
1882       if (splitLabel) {
1883         const PetscInt val = 100 + dep;
1884 
1885         PetscCall(DMLabelSetValue(splitLabel, newp, val));
1886         PetscCall(DMLabelSetValue(splitLabel, splitp, -val));
1887       }
1888       for (l = 0; l < numLabels; ++l) {
1889         DMLabel     mlabel;
1890         const char *lname;
1891         PetscInt    val;
1892         PetscBool   isDepth;
1893 
1894         PetscCall(DMGetLabelName(sdm, l, &lname));
1895         PetscCall(PetscStrcmp(lname, "depth", &isDepth));
1896         if (isDepth) continue;
1897         PetscCall(DMGetLabel(sdm, lname, &mlabel));
1898         PetscCall(DMLabelGetValue(mlabel, newp, &val));
1899         if (val >= 0) PetscCall(DMLabelSetValue(mlabel, splitp, val));
1900       }
1901     }
1902   }
1903   for (sp = 0; sp < numSP; ++sp) {
1904     const PetscInt dep = values[sp];
1905 
1906     if ((dep < 0) || (dep > depth)) continue;
1907     if (splitIS[dep]) PetscCall(ISRestoreIndices(splitIS[dep], &splitPoints[dep]));
1908     PetscCall(ISDestroy(&splitIS[dep]));
1909     if (unsplitIS[dep]) PetscCall(ISRestoreIndices(unsplitIS[dep], &unsplitPoints[dep]));
1910     PetscCall(ISDestroy(&unsplitIS[dep]));
1911   }
1912   if (ghostIS) PetscCall(ISRestoreIndices(ghostIS, &ghostPoints));
1913   PetscCall(ISDestroy(&ghostIS));
1914   if (label) {
1915     PetscCall(ISRestoreIndices(valueIS, &values));
1916     PetscCall(ISDestroy(&valueIS));
1917   }
1918   for (d = 0; d <= depth; ++d) {
1919     PetscCall(DMPlexGetDepthStratum(sdm, d, NULL, &pEnd));
1920     pMaxNew[d] = pEnd - numHybridPoints[d] - numHybridPointsOld[d];
1921   }
1922   PetscCall(PetscFree3(coneNew, coneONew, supportNew));
1923   PetscCall(PetscFree5(depthMax, depthEnd, depthShift, pMaxNew, numHybridPointsOld));
1924   PetscCall(PetscFree7(splitIS, unsplitIS, numSplitPoints, numUnsplitPoints, numHybridPoints, splitPoints, unsplitPoints));
1925   PetscFunctionReturn(PETSC_SUCCESS);
1926 }
1927 
1928 /*@C
1929   DMPlexConstructCohesiveCells - Construct cohesive cells which split the face along an internal interface
1930 
1931   Collective
1932 
1933   Input Parameters:
1934 + dm    - The original `DM`
1935 - label - The `DMLabel` specifying the boundary faces (this could be auto-generated)
1936 
1937   Output Parameters:
1938 + splitLabel - The `DMLabel` containing the split points, or `NULL` if no output is desired
1939 - dmSplit    - The new `DM`
1940 
1941   Level: developer
1942 
1943 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMCreate()`, `DMPlexLabelCohesiveComplete()`
1944 @*/
1945 PetscErrorCode DMPlexConstructCohesiveCells(DM dm, DMLabel label, DMLabel splitLabel, DM *dmSplit)
1946 {
1947   DM       sdm;
1948   PetscInt dim;
1949 
1950   PetscFunctionBegin;
1951   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1952   PetscAssertPointer(dmSplit, 4);
1953   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &sdm));
1954   PetscCall(DMSetType(sdm, DMPLEX));
1955   PetscCall(DMGetDimension(dm, &dim));
1956   PetscCall(DMSetDimension(sdm, dim));
1957   switch (dim) {
1958   case 2:
1959   case 3:
1960     PetscCall(DMPlexConstructCohesiveCells_Internal(dm, label, splitLabel, sdm));
1961     break;
1962   default:
1963     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cannot construct cohesive cells for dimension %" PetscInt_FMT, dim);
1964   }
1965   *dmSplit = sdm;
1966   PetscFunctionReturn(PETSC_SUCCESS);
1967 }
1968 
1969 /* Returns the side of the surface for a given cell with a face on the surface */
1970 static PetscErrorCode GetSurfaceSide_Static(DM dm, DM subdm, PetscInt numSubpoints, const PetscInt *subpoints, PetscInt cell, PetscInt face, PetscBool *pos)
1971 {
1972   const PetscInt *cone, *ornt;
1973   PetscInt        dim, coneSize, c;
1974 
1975   PetscFunctionBegin;
1976   *pos = PETSC_TRUE;
1977   PetscCall(DMGetDimension(dm, &dim));
1978   PetscCall(DMPlexGetConeSize(dm, cell, &coneSize));
1979   PetscCall(DMPlexGetCone(dm, cell, &cone));
1980   PetscCall(DMPlexGetConeOrientation(dm, cell, &ornt));
1981   for (c = 0; c < coneSize; ++c) {
1982     if (cone[c] == face) {
1983       PetscInt o = ornt[c];
1984 
1985       if (subdm) {
1986         const PetscInt *subcone, *subornt;
1987         PetscInt        subpoint, subface, subconeSize, sc;
1988 
1989         PetscCall(PetscFindInt(cell, numSubpoints, subpoints, &subpoint));
1990         PetscCall(PetscFindInt(face, numSubpoints, subpoints, &subface));
1991         PetscCall(DMPlexGetConeSize(subdm, subpoint, &subconeSize));
1992         PetscCall(DMPlexGetCone(subdm, subpoint, &subcone));
1993         PetscCall(DMPlexGetConeOrientation(subdm, subpoint, &subornt));
1994         for (sc = 0; sc < subconeSize; ++sc) {
1995           if (subcone[sc] == subface) {
1996             o = subornt[0];
1997             break;
1998           }
1999         }
2000         PetscCheck(sc < subconeSize, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not find subpoint %" PetscInt_FMT " (%" PetscInt_FMT ") in cone for subpoint %" PetscInt_FMT " (%" PetscInt_FMT ")", subface, face, subpoint, cell);
2001       }
2002       if (o >= 0) *pos = PETSC_TRUE;
2003       else *pos = PETSC_FALSE;
2004       break;
2005     }
2006   }
2007   PetscCheck(c != coneSize, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Cell %" PetscInt_FMT " in split face %" PetscInt_FMT " support does not have it in the cone", cell, face);
2008   PetscFunctionReturn(PETSC_SUCCESS);
2009 }
2010 
2011 static PetscErrorCode CheckFaultEdge_Private(DM dm, DMLabel label)
2012 {
2013   IS              facePosIS, faceNegIS, dimIS;
2014   const PetscInt *points;
2015   PetscInt        dim, numPoints, p, shift = 100, shift2 = 200;
2016 
2017   PetscFunctionBegin;
2018   PetscCall(DMGetDimension(dm, &dim));
2019   /* If any faces touching the fault divide cells on either side, split them */
2020   PetscCall(DMLabelGetStratumIS(label, shift + dim - 1, &facePosIS));
2021   PetscCall(DMLabelGetStratumIS(label, -(shift + dim - 1), &faceNegIS));
2022   if (!facePosIS || !faceNegIS) {
2023     PetscCall(ISDestroy(&facePosIS));
2024     PetscCall(ISDestroy(&faceNegIS));
2025     PetscFunctionReturn(PETSC_SUCCESS);
2026   }
2027   PetscCall(ISExpand(facePosIS, faceNegIS, &dimIS));
2028   PetscCall(ISDestroy(&facePosIS));
2029   PetscCall(ISDestroy(&faceNegIS));
2030   PetscCall(ISGetLocalSize(dimIS, &numPoints));
2031   PetscCall(ISGetIndices(dimIS, &points));
2032   for (p = 0; p < numPoints; ++p) {
2033     const PetscInt  point = points[p];
2034     const PetscInt *support;
2035     PetscInt        supportSize, valA, valB;
2036 
2037     PetscCall(DMPlexGetSupportSize(dm, point, &supportSize));
2038     if (supportSize != 2) continue;
2039     PetscCall(DMPlexGetSupport(dm, point, &support));
2040     PetscCall(DMLabelGetValue(label, support[0], &valA));
2041     PetscCall(DMLabelGetValue(label, support[1], &valB));
2042     if ((valA == -1) || (valB == -1)) continue;
2043     if (valA * valB > 0) continue;
2044     /* Check that this face is not incident on only unsplit faces, meaning has at least one split face */
2045     {
2046       PetscInt *closure = NULL;
2047       PetscBool split   = PETSC_FALSE;
2048       PetscInt  closureSize, cl;
2049 
2050       PetscCall(DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
2051       for (cl = 0; cl < closureSize * 2; cl += 2) {
2052         PetscCall(DMLabelGetValue(label, closure[cl], &valA));
2053         if ((valA >= 0) && (valA <= dim)) {
2054           split = PETSC_TRUE;
2055           break;
2056         }
2057       }
2058       PetscCall(DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
2059       if (!split) continue;
2060     }
2061     /* Split the face */
2062     PetscCall(DMLabelGetValue(label, point, &valA));
2063     PetscCall(DMLabelClearValue(label, point, valA));
2064     PetscCall(DMLabelSetValue(label, point, dim - 1));
2065     /* Label its closure:
2066       unmarked: label as unsplit
2067       incident: relabel as split
2068       split:    do nothing
2069     */
2070     {
2071       PetscInt *closure = NULL;
2072       PetscInt  closureSize, cl, dep;
2073 
2074       PetscCall(DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
2075       for (cl = 0; cl < closureSize * 2; cl += 2) {
2076         PetscCall(DMLabelGetValue(label, closure[cl], &valA));
2077         if (valA == -1) { /* Mark as unsplit */
2078           PetscCall(DMPlexGetPointDepth(dm, closure[cl], &dep));
2079           PetscCall(DMLabelSetValue(label, closure[cl], shift2 + dep));
2080         } else if (((valA >= shift) && (valA < shift2)) || ((valA <= -shift) && (valA > -shift2))) {
2081           PetscCall(DMPlexGetPointDepth(dm, closure[cl], &dep));
2082           PetscCall(DMLabelClearValue(label, closure[cl], valA));
2083           PetscCall(DMLabelSetValue(label, closure[cl], dep));
2084         }
2085       }
2086       PetscCall(DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
2087     }
2088   }
2089   PetscCall(ISRestoreIndices(dimIS, &points));
2090   PetscCall(ISDestroy(&dimIS));
2091   PetscFunctionReturn(PETSC_SUCCESS);
2092 }
2093 
2094 /*@
2095   DMPlexLabelCohesiveComplete - Starting with a label marking points on an internal surface, we add all other mesh pieces
2096   to complete the surface
2097 
2098   Input Parameters:
2099 + dm     - The `DM`
2100 . label  - A `DMLabel` marking the surface
2101 . blabel - A `DMLabel` marking the vertices on the boundary which will not be duplicated, or `NULL` to find them automatically
2102 . bvalue - Value of `DMLabel` marking the vertices on the boundary
2103 . flip   - Flag to flip the submesh normal and replace points on the other side
2104 - subdm  - The `DM` associated with the label, or `NULL`
2105 
2106   Output Parameter:
2107 . label - A `DMLabel` marking all surface points
2108 
2109   Level: developer
2110 
2111   Note:
2112   The vertices in blabel are called "unsplit" in the terminology from hybrid cell creation.
2113 
2114 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexConstructCohesiveCells()`, `DMPlexLabelComplete()`
2115 @*/
2116 PetscErrorCode DMPlexLabelCohesiveComplete(DM dm, DMLabel label, DMLabel blabel, PetscInt bvalue, PetscBool flip, DM subdm)
2117 {
2118   DMLabel         depthLabel;
2119   IS              dimIS, subpointIS = NULL;
2120   const PetscInt *points, *subpoints;
2121   const PetscInt  rev   = flip ? -1 : 1;
2122   PetscInt        shift = 100, shift2 = 200, shift3 = 300, dim, depth, numPoints, numSubpoints, p, val;
2123 
2124   PetscFunctionBegin;
2125   PetscCall(DMPlexGetDepth(dm, &depth));
2126   PetscCall(DMGetDimension(dm, &dim));
2127   PetscCall(DMPlexGetDepthLabel(dm, &depthLabel));
2128   if (subdm) {
2129     PetscCall(DMPlexGetSubpointIS(subdm, &subpointIS));
2130     if (subpointIS) {
2131       PetscCall(ISGetLocalSize(subpointIS, &numSubpoints));
2132       PetscCall(ISGetIndices(subpointIS, &subpoints));
2133     }
2134   }
2135   /* Mark cell on the fault, and its faces which touch the fault: cell orientation for face gives the side of the fault */
2136   PetscCall(DMLabelGetStratumIS(label, dim - 1, &dimIS));
2137   if (!dimIS) goto divide;
2138   PetscCall(ISGetLocalSize(dimIS, &numPoints));
2139   PetscCall(ISGetIndices(dimIS, &points));
2140   for (p = 0; p < numPoints; ++p) { /* Loop over fault faces */
2141     const PetscInt *support;
2142     PetscInt        supportSize, s;
2143 
2144     PetscCall(DMPlexGetSupportSize(dm, points[p], &supportSize));
2145 #if 0
2146     if (supportSize != 2) {
2147       const PetscInt *lp;
2148       PetscInt        Nlp, pind;
2149 
2150       /* Check that for a cell with a single support face, that face is in the SF */
2151       /*   THis check only works for the remote side. We would need root side information */
2152       PetscCall(PetscSFGetGraph(dm->sf, NULL, &Nlp, &lp, NULL));
2153       PetscCall(PetscFindInt(points[p], Nlp, lp, &pind));
2154       PetscCheck(pind >= 0,PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Split face %" PetscInt_FMT " has %" PetscInt_FMT " != 2 supports, and the face is not shared with another process", points[p], supportSize);
2155     }
2156 #endif
2157     PetscCall(DMPlexGetSupport(dm, points[p], &support));
2158     for (s = 0; s < supportSize; ++s) {
2159       const PetscInt *cone;
2160       PetscInt        coneSize, c;
2161       PetscBool       pos;
2162 
2163       PetscCall(GetSurfaceSide_Static(dm, subdm, numSubpoints, subpoints, support[s], points[p], &pos));
2164       if (pos) PetscCall(DMLabelSetValue(label, support[s], rev * (shift + dim)));
2165       else PetscCall(DMLabelSetValue(label, support[s], -rev * (shift + dim)));
2166       if (rev < 0) pos = !pos ? PETSC_TRUE : PETSC_FALSE;
2167       /* Put faces touching the fault in the label */
2168       PetscCall(DMPlexGetConeSize(dm, support[s], &coneSize));
2169       PetscCall(DMPlexGetCone(dm, support[s], &cone));
2170       for (c = 0; c < coneSize; ++c) {
2171         const PetscInt point = cone[c];
2172 
2173         PetscCall(DMLabelGetValue(label, point, &val));
2174         if (val == -1) {
2175           PetscInt *closure = NULL;
2176           PetscInt  closureSize, cl;
2177 
2178           PetscCall(DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
2179           for (cl = 0; cl < closureSize * 2; cl += 2) {
2180             const PetscInt clp  = closure[cl];
2181             PetscInt       bval = -1;
2182 
2183             PetscCall(DMLabelGetValue(label, clp, &val));
2184             if (blabel) PetscCall(DMLabelGetValue(blabel, clp, &bval));
2185             if ((val >= 0) && (val < dim - 1) && (bval < 0)) {
2186               PetscCall(DMLabelSetValue(label, point, pos == PETSC_TRUE ? shift + dim - 1 : -(shift + dim - 1)));
2187               break;
2188             }
2189           }
2190           PetscCall(DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
2191         }
2192       }
2193     }
2194   }
2195   PetscCall(ISRestoreIndices(dimIS, &points));
2196   PetscCall(ISDestroy(&dimIS));
2197   /* Mark boundary points as unsplit */
2198   if (blabel) {
2199     IS bdIS;
2200 
2201     PetscCall(DMLabelGetStratumIS(blabel, bvalue, &bdIS));
2202     PetscCall(ISGetLocalSize(bdIS, &numPoints));
2203     PetscCall(ISGetIndices(bdIS, &points));
2204     for (p = 0; p < numPoints; ++p) {
2205       const PetscInt point = points[p];
2206       PetscInt       val, bval;
2207 
2208       PetscCall(DMLabelGetValue(blabel, point, &bval));
2209       if (bval >= 0) {
2210         PetscCall(DMLabelGetValue(label, point, &val));
2211         if ((val < 0) || (val > dim)) {
2212           /* This could be a point added from splitting a vertex on an adjacent fault, otherwise its just wrong */
2213           PetscCall(DMLabelClearValue(blabel, point, bval));
2214         }
2215       }
2216     }
2217     for (p = 0; p < numPoints; ++p) {
2218       const PetscInt point = points[p];
2219       PetscInt       val, bval;
2220 
2221       PetscCall(DMLabelGetValue(blabel, point, &bval));
2222       if (bval >= 0) {
2223         const PetscInt *cone, *support;
2224         PetscInt        coneSize, supportSize, s, valA, valB, valE;
2225 
2226         /* Mark as unsplit */
2227         PetscCall(DMLabelGetValue(label, point, &val));
2228         PetscCheck(val >= 0 && val <= dim, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %" PetscInt_FMT " has label value %" PetscInt_FMT ", should be part of the fault", point, val);
2229         PetscCall(DMLabelClearValue(label, point, val));
2230         PetscCall(DMLabelSetValue(label, point, shift2 + val));
2231         /* Check for cross-edge
2232              A cross-edge has endpoints which are both on the boundary of the surface, but the edge itself is not. */
2233         if (val != 0) continue;
2234         PetscCall(DMPlexGetSupport(dm, point, &support));
2235         PetscCall(DMPlexGetSupportSize(dm, point, &supportSize));
2236         for (s = 0; s < supportSize; ++s) {
2237           PetscCall(DMPlexGetCone(dm, support[s], &cone));
2238           PetscCall(DMPlexGetConeSize(dm, support[s], &coneSize));
2239           PetscCheck(coneSize == 2, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Edge %" PetscInt_FMT " has %" PetscInt_FMT " vertices != 2", support[s], coneSize);
2240           PetscCall(DMLabelGetValue(blabel, cone[0], &valA));
2241           PetscCall(DMLabelGetValue(blabel, cone[1], &valB));
2242           PetscCall(DMLabelGetValue(blabel, support[s], &valE));
2243           if ((valE < 0) && (valA >= 0) && (valB >= 0) && (cone[0] != cone[1])) PetscCall(DMLabelSetValue(blabel, support[s], 2));
2244         }
2245       }
2246     }
2247     PetscCall(ISRestoreIndices(bdIS, &points));
2248     PetscCall(ISDestroy(&bdIS));
2249   }
2250   /* Mark ghost fault cells */
2251   {
2252     PetscSF         sf;
2253     const PetscInt *leaves;
2254     PetscInt        Nl, l;
2255 
2256     PetscCall(DMGetPointSF(dm, &sf));
2257     PetscCall(PetscSFGetGraph(sf, NULL, &Nl, &leaves, NULL));
2258     PetscCall(DMLabelGetStratumIS(label, dim - 1, &dimIS));
2259     if (!dimIS) goto divide;
2260     PetscCall(ISGetLocalSize(dimIS, &numPoints));
2261     PetscCall(ISGetIndices(dimIS, &points));
2262     if (Nl > 0) {
2263       for (p = 0; p < numPoints; ++p) {
2264         const PetscInt point = points[p];
2265         PetscInt       val;
2266 
2267         PetscCall(PetscFindInt(point, Nl, leaves, &l));
2268         if (l >= 0) {
2269           PetscInt *closure = NULL;
2270           PetscInt  closureSize, cl;
2271 
2272           PetscCall(DMLabelGetValue(label, point, &val));
2273           PetscCheck((val == dim - 1) || (val == shift2 + dim - 1), PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %" PetscInt_FMT " has label value %" PetscInt_FMT ", should be a fault face", point, val);
2274           PetscCall(DMLabelClearValue(label, point, val));
2275           PetscCall(DMLabelSetValue(label, point, shift3 + val));
2276           PetscCall(DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
2277           for (cl = 2; cl < closureSize * 2; cl += 2) {
2278             const PetscInt clp = closure[cl];
2279 
2280             PetscCall(DMLabelGetValue(label, clp, &val));
2281             PetscCheck(val != -1, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %" PetscInt_FMT " is missing from label, but is in the closure of a fault face", point);
2282             PetscCall(DMLabelClearValue(label, clp, val));
2283             PetscCall(DMLabelSetValue(label, clp, shift3 + val));
2284           }
2285           PetscCall(DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
2286         }
2287       }
2288     }
2289     PetscCall(ISRestoreIndices(dimIS, &points));
2290     PetscCall(ISDestroy(&dimIS));
2291   }
2292 divide:
2293   if (subpointIS) PetscCall(ISRestoreIndices(subpointIS, &subpoints));
2294   PetscCall(DMPlexLabelFaultHalo(dm, label));
2295   PetscCall(CheckFaultEdge_Private(dm, label));
2296   PetscFunctionReturn(PETSC_SUCCESS);
2297 }
2298 
2299 /* Check that no cell have all vertices on the fault */
2300 static PetscErrorCode DMPlexCheckValidSubmesh_Private(DM dm, DMLabel label, DM subdm)
2301 {
2302   IS              subpointIS;
2303   const PetscInt *dmpoints;
2304   PetscInt        defaultValue, cStart, cEnd, c, vStart, vEnd;
2305 
2306   PetscFunctionBegin;
2307   if (!label) PetscFunctionReturn(PETSC_SUCCESS);
2308   PetscCall(DMLabelGetDefaultValue(label, &defaultValue));
2309   PetscCall(DMPlexGetSubpointIS(subdm, &subpointIS));
2310   if (!subpointIS) PetscFunctionReturn(PETSC_SUCCESS);
2311   PetscCall(DMPlexGetHeightStratum(subdm, 0, &cStart, &cEnd));
2312   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
2313   PetscCall(ISGetIndices(subpointIS, &dmpoints));
2314   for (c = cStart; c < cEnd; ++c) {
2315     PetscBool invalidCell = PETSC_TRUE;
2316     PetscInt *closure     = NULL;
2317     PetscInt  closureSize, cl;
2318 
2319     PetscCall(DMPlexGetTransitiveClosure(dm, dmpoints[c], PETSC_TRUE, &closureSize, &closure));
2320     for (cl = 0; cl < closureSize * 2; cl += 2) {
2321       PetscInt value = 0;
2322 
2323       if ((closure[cl] < vStart) || (closure[cl] >= vEnd)) continue;
2324       PetscCall(DMLabelGetValue(label, closure[cl], &value));
2325       if (value == defaultValue) {
2326         invalidCell = PETSC_FALSE;
2327         break;
2328       }
2329     }
2330     PetscCall(DMPlexRestoreTransitiveClosure(dm, dmpoints[c], PETSC_TRUE, &closureSize, &closure));
2331     if (invalidCell) {
2332       PetscCall(ISRestoreIndices(subpointIS, &dmpoints));
2333       PetscCall(ISDestroy(&subpointIS));
2334       PetscCall(DMDestroy(&subdm));
2335       SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Ambiguous submesh. Cell %" PetscInt_FMT " has all of its vertices on the submesh.", dmpoints[c]);
2336     }
2337   }
2338   PetscCall(ISRestoreIndices(subpointIS, &dmpoints));
2339   PetscFunctionReturn(PETSC_SUCCESS);
2340 }
2341 
2342 /*@
2343   DMPlexCreateHybridMesh - Create a mesh with hybrid cells along an internal interface
2344 
2345   Collective
2346 
2347   Input Parameters:
2348 + dm      - The original `DM`
2349 . label   - The label specifying the interface vertices
2350 . bdlabel - The optional label specifying the interface boundary vertices
2351 - bdvalue - Value of optional label specifying the interface boundary vertices
2352 
2353   Output Parameters:
2354 + hybridLabel - The label fully marking the interface, or `NULL` if no output is desired
2355 . splitLabel  - The label containing the split points, or `NULL` if no output is desired
2356 . dmInterface - The new interface `DM`, or `NULL`
2357 - dmHybrid    - The new `DM` with cohesive cells
2358 
2359   Level: developer
2360 
2361   Note:
2362   The hybridLabel indicates what parts of the original mesh impinged on the division surface. For points
2363   directly on the division surface, they are labeled with their dimension, so an edge 7 on the division surface would be
2364   7 (1) in hybridLabel. For points that impinge from the positive side, they are labeled with 100+dim, so an edge 6 with
2365   one vertex 3 on the surface would be 6 (101) and 3 (0) in hybridLabel. If an edge 9 from the negative side of the
2366   surface also hits vertex 3, it would be 9 (-101) in hybridLabel.
2367 
2368   The splitLabel indicates what points in the new hybrid mesh were the result of splitting points in the original
2369   mesh. The label value is $\pm 100+dim$ for each point. For example, if two edges 10 and 14 in the hybrid resulting from
2370   splitting an edge in the original mesh, you would have 10 (101) and 14 (-101) in the splitLabel.
2371 
2372   The dmInterface is a `DM` built from the original division surface. It has a label which can be retrieved using
2373   `DMPlexGetSubpointMap()` which maps each point back to the point in the surface of the original mesh.
2374 
2375 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexConstructCohesiveCells()`, `DMPlexLabelCohesiveComplete()`, `DMPlexGetSubpointMap()`, `DMCreate()`
2376 @*/
2377 PetscErrorCode DMPlexCreateHybridMesh(DM dm, DMLabel label, DMLabel bdlabel, PetscInt bdvalue, DMLabel *hybridLabel, DMLabel *splitLabel, DM *dmInterface, DM *dmHybrid)
2378 {
2379   DM       idm;
2380   DMLabel  subpointMap, hlabel, slabel = NULL;
2381   PetscInt dim;
2382 
2383   PetscFunctionBegin;
2384   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2385   if (label) PetscAssertPointer(label, 2);
2386   if (bdlabel) PetscAssertPointer(bdlabel, 3);
2387   if (hybridLabel) PetscAssertPointer(hybridLabel, 5);
2388   if (splitLabel) PetscAssertPointer(splitLabel, 6);
2389   if (dmInterface) PetscAssertPointer(dmInterface, 7);
2390   PetscAssertPointer(dmHybrid, 8);
2391   PetscCall(DMGetDimension(dm, &dim));
2392   PetscCall(DMPlexCreateSubmesh(dm, label, 1, PETSC_FALSE, &idm));
2393   PetscCall(DMPlexCheckValidSubmesh_Private(dm, label, idm));
2394   PetscCall(DMPlexOrient(idm));
2395   PetscCall(DMPlexGetSubpointMap(idm, &subpointMap));
2396   PetscCall(DMLabelDuplicate(subpointMap, &hlabel));
2397   PetscCall(DMLabelClearStratum(hlabel, dim));
2398   if (splitLabel) {
2399     const char *name;
2400     char        sname[PETSC_MAX_PATH_LEN];
2401 
2402     PetscCall(PetscObjectGetName((PetscObject)hlabel, &name));
2403     PetscCall(PetscStrncpy(sname, name, sizeof(sname)));
2404     PetscCall(PetscStrlcat(sname, " split", sizeof(sname)));
2405     PetscCall(DMLabelCreate(PETSC_COMM_SELF, sname, &slabel));
2406   }
2407   PetscCall(DMPlexLabelCohesiveComplete(dm, hlabel, bdlabel, bdvalue, PETSC_FALSE, idm));
2408   if (dmInterface) {
2409     *dmInterface = idm;
2410   } else PetscCall(DMDestroy(&idm));
2411   PetscCall(DMPlexConstructCohesiveCells(dm, hlabel, slabel, dmHybrid));
2412   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_TRUE, *dmHybrid));
2413   if (hybridLabel) *hybridLabel = hlabel;
2414   else PetscCall(DMLabelDestroy(&hlabel));
2415   if (splitLabel) *splitLabel = slabel;
2416   {
2417     DM      cdm;
2418     DMLabel ctLabel;
2419 
2420     /* We need to somehow share the celltype label with the coordinate dm */
2421     PetscCall(DMGetCoordinateDM(*dmHybrid, &cdm));
2422     PetscCall(DMPlexGetCellTypeLabel(*dmHybrid, &ctLabel));
2423     PetscCall(DMSetLabel(cdm, ctLabel));
2424   }
2425   PetscFunctionReturn(PETSC_SUCCESS);
2426 }
2427 
2428 /* Here we need the explicit assumption that:
2429 
2430      For any marked cell, the marked vertices constitute a single face
2431 */
2432 static PetscErrorCode DMPlexMarkSubmesh_Uninterpolated(DM dm, DMLabel vertexLabel, PetscInt value, DMLabel subpointMap, PetscInt *numFaces, PetscInt *nFV, DM subdm)
2433 {
2434   IS              subvertexIS = NULL;
2435   const PetscInt *subvertices;
2436   PetscInt       *pStart, *pEnd, pSize;
2437   PetscInt        depth, dim, d, numSubVerticesInitial = 0, v;
2438 
2439   PetscFunctionBegin;
2440   *numFaces = 0;
2441   *nFV      = 0;
2442   PetscCall(DMPlexGetDepth(dm, &depth));
2443   PetscCall(DMGetDimension(dm, &dim));
2444   pSize = PetscMax(depth, dim) + 1;
2445   PetscCall(PetscMalloc2(pSize, &pStart, pSize, &pEnd));
2446   for (d = 0; d <= depth; ++d) PetscCall(DMPlexGetSimplexOrBoxCells(dm, depth - d, &pStart[d], &pEnd[d]));
2447   /* Loop over initial vertices and mark all faces in the collective star() */
2448   if (vertexLabel) PetscCall(DMLabelGetStratumIS(vertexLabel, value, &subvertexIS));
2449   if (subvertexIS) {
2450     PetscCall(ISGetSize(subvertexIS, &numSubVerticesInitial));
2451     PetscCall(ISGetIndices(subvertexIS, &subvertices));
2452   }
2453   for (v = 0; v < numSubVerticesInitial; ++v) {
2454     const PetscInt vertex = subvertices[v];
2455     PetscInt      *star   = NULL;
2456     PetscInt       starSize, s, numCells = 0, c;
2457 
2458     PetscCall(DMPlexGetTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star));
2459     for (s = 0; s < starSize * 2; s += 2) {
2460       const PetscInt point = star[s];
2461       if ((point >= pStart[depth]) && (point < pEnd[depth])) star[numCells++] = point;
2462     }
2463     for (c = 0; c < numCells; ++c) {
2464       const PetscInt cell    = star[c];
2465       PetscInt      *closure = NULL;
2466       PetscInt       closureSize, cl;
2467       PetscInt       cellLoc, numCorners = 0, faceSize = 0;
2468 
2469       PetscCall(DMLabelGetValue(subpointMap, cell, &cellLoc));
2470       if (cellLoc == 2) continue;
2471       PetscCheck(cellLoc < 0, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Cell %" PetscInt_FMT " has dimension %" PetscInt_FMT " in the surface label", cell, cellLoc);
2472       PetscCall(DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
2473       for (cl = 0; cl < closureSize * 2; cl += 2) {
2474         const PetscInt point = closure[cl];
2475         PetscInt       vertexLoc;
2476 
2477         if ((point >= pStart[0]) && (point < pEnd[0])) {
2478           ++numCorners;
2479           PetscCall(DMLabelGetValue(vertexLabel, point, &vertexLoc));
2480           if (vertexLoc == value) closure[faceSize++] = point;
2481         }
2482       }
2483       if (!(*nFV)) PetscCall(DMPlexGetNumFaceVertices(dm, dim, numCorners, nFV));
2484       PetscCheck(faceSize <= *nFV, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Invalid submesh: Too many vertices %" PetscInt_FMT " of an element on the surface", faceSize);
2485       if (faceSize == *nFV) {
2486         const PetscInt *cells = NULL;
2487         PetscInt        numCells, nc;
2488 
2489         ++(*numFaces);
2490         for (cl = 0; cl < faceSize; ++cl) PetscCall(DMLabelSetValue(subpointMap, closure[cl], 0));
2491         PetscCall(DMPlexGetJoin(dm, faceSize, closure, &numCells, &cells));
2492         for (nc = 0; nc < numCells; ++nc) PetscCall(DMLabelSetValue(subpointMap, cells[nc], 2));
2493         PetscCall(DMPlexRestoreJoin(dm, faceSize, closure, &numCells, &cells));
2494       }
2495       PetscCall(DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
2496     }
2497     PetscCall(DMPlexRestoreTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star));
2498   }
2499   if (subvertexIS) PetscCall(ISRestoreIndices(subvertexIS, &subvertices));
2500   PetscCall(ISDestroy(&subvertexIS));
2501   PetscCall(PetscFree2(pStart, pEnd));
2502   PetscFunctionReturn(PETSC_SUCCESS);
2503 }
2504 
2505 static PetscErrorCode DMPlexMarkSubmesh_Interpolated(DM dm, DMLabel vertexLabel, PetscInt value, PetscBool markedFaces, DMLabel subpointMap, DM subdm)
2506 {
2507   IS              subvertexIS = NULL;
2508   const PetscInt *subvertices;
2509   PetscInt       *pStart, *pEnd;
2510   PetscInt        dim, d, numSubVerticesInitial = 0, v;
2511 
2512   PetscFunctionBegin;
2513   PetscCall(DMGetDimension(dm, &dim));
2514   PetscCall(PetscMalloc2(dim + 1, &pStart, dim + 1, &pEnd));
2515   for (d = 0; d <= dim; ++d) PetscCall(DMPlexGetSimplexOrBoxCells(dm, dim - d, &pStart[d], &pEnd[d]));
2516   /* Loop over initial vertices and mark all faces in the collective star() */
2517   if (vertexLabel) {
2518     PetscCall(DMLabelGetStratumIS(vertexLabel, value, &subvertexIS));
2519     if (subvertexIS) {
2520       PetscCall(ISGetSize(subvertexIS, &numSubVerticesInitial));
2521       PetscCall(ISGetIndices(subvertexIS, &subvertices));
2522     }
2523   }
2524   for (v = 0; v < numSubVerticesInitial; ++v) {
2525     const PetscInt vertex = subvertices[v];
2526     PetscInt      *star   = NULL;
2527     PetscInt       starSize, s, numFaces = 0, f;
2528 
2529     PetscCall(DMPlexGetTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star));
2530     for (s = 0; s < starSize * 2; s += 2) {
2531       const PetscInt point = star[s];
2532       PetscInt       faceLoc;
2533 
2534       if ((point >= pStart[dim - 1]) && (point < pEnd[dim - 1])) {
2535         if (markedFaces) {
2536           PetscCall(DMLabelGetValue(vertexLabel, point, &faceLoc));
2537           if (faceLoc < 0) continue;
2538         }
2539         star[numFaces++] = point;
2540       }
2541     }
2542     for (f = 0; f < numFaces; ++f) {
2543       const PetscInt face    = star[f];
2544       PetscInt      *closure = NULL;
2545       PetscInt       closureSize, c;
2546       PetscInt       faceLoc;
2547 
2548       PetscCall(DMLabelGetValue(subpointMap, face, &faceLoc));
2549       if (faceLoc == dim - 1) continue;
2550       PetscCheck(faceLoc < 0, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Face %" PetscInt_FMT " has dimension %" PetscInt_FMT " in the surface label", face, faceLoc);
2551       PetscCall(DMPlexGetTransitiveClosure(dm, face, PETSC_TRUE, &closureSize, &closure));
2552       for (c = 0; c < closureSize * 2; c += 2) {
2553         const PetscInt point = closure[c];
2554         PetscInt       vertexLoc;
2555 
2556         if ((point >= pStart[0]) && (point < pEnd[0])) {
2557           PetscCall(DMLabelGetValue(vertexLabel, point, &vertexLoc));
2558           if (vertexLoc != value) break;
2559         }
2560       }
2561       if (c == closureSize * 2) {
2562         const PetscInt *support;
2563         PetscInt        supportSize, s;
2564 
2565         for (c = 0; c < closureSize * 2; c += 2) {
2566           const PetscInt point = closure[c];
2567 
2568           for (d = 0; d < dim; ++d) {
2569             if ((point >= pStart[d]) && (point < pEnd[d])) {
2570               PetscCall(DMLabelSetValue(subpointMap, point, d));
2571               break;
2572             }
2573           }
2574         }
2575         PetscCall(DMPlexGetSupportSize(dm, face, &supportSize));
2576         PetscCall(DMPlexGetSupport(dm, face, &support));
2577         for (s = 0; s < supportSize; ++s) PetscCall(DMLabelSetValue(subpointMap, support[s], dim));
2578       }
2579       PetscCall(DMPlexRestoreTransitiveClosure(dm, face, PETSC_TRUE, &closureSize, &closure));
2580     }
2581     PetscCall(DMPlexRestoreTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star));
2582   }
2583   if (subvertexIS) PetscCall(ISRestoreIndices(subvertexIS, &subvertices));
2584   PetscCall(ISDestroy(&subvertexIS));
2585   PetscCall(PetscFree2(pStart, pEnd));
2586   PetscFunctionReturn(PETSC_SUCCESS);
2587 }
2588 
2589 static PetscErrorCode DMPlexMarkCohesiveSubmesh_Uninterpolated(DM dm, PetscBool hasLagrange, const char labelname[], PetscInt value, DMLabel subpointMap, PetscInt *numFaces, PetscInt *nFV, PetscInt *subCells[], DM subdm)
2590 {
2591   DMLabel         label = NULL;
2592   const PetscInt *cone;
2593   PetscInt        dim, cMax, cEnd, c, subc = 0, p, coneSize = -1;
2594 
2595   PetscFunctionBegin;
2596   *numFaces = 0;
2597   *nFV      = 0;
2598   if (labelname) PetscCall(DMGetLabel(dm, labelname, &label));
2599   *subCells = NULL;
2600   PetscCall(DMGetDimension(dm, &dim));
2601   PetscCall(DMPlexGetTensorPrismBounds_Internal(dm, dim, &cMax, &cEnd));
2602   if (cMax < 0) PetscFunctionReturn(PETSC_SUCCESS);
2603   if (label) {
2604     for (c = cMax; c < cEnd; ++c) {
2605       PetscInt val;
2606 
2607       PetscCall(DMLabelGetValue(label, c, &val));
2608       if (val == value) {
2609         ++(*numFaces);
2610         PetscCall(DMPlexGetConeSize(dm, c, &coneSize));
2611       }
2612     }
2613   } else {
2614     *numFaces = cEnd - cMax;
2615     PetscCall(DMPlexGetConeSize(dm, cMax, &coneSize));
2616   }
2617   PetscCall(PetscMalloc1(*numFaces * 2, subCells));
2618   if (!(*numFaces)) PetscFunctionReturn(PETSC_SUCCESS);
2619   *nFV = hasLagrange ? coneSize / 3 : coneSize / 2;
2620   for (c = cMax; c < cEnd; ++c) {
2621     const PetscInt *cells;
2622     PetscInt        numCells;
2623 
2624     if (label) {
2625       PetscInt val;
2626 
2627       PetscCall(DMLabelGetValue(label, c, &val));
2628       if (val != value) continue;
2629     }
2630     PetscCall(DMPlexGetCone(dm, c, &cone));
2631     for (p = 0; p < *nFV; ++p) PetscCall(DMLabelSetValue(subpointMap, cone[p], 0));
2632     /* Negative face */
2633     PetscCall(DMPlexGetJoin(dm, *nFV, cone, &numCells, &cells));
2634     /* Not true in parallel
2635     PetscCheck(numCells == 2,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive cells should separate two cells"); */
2636     for (p = 0; p < numCells; ++p) {
2637       PetscCall(DMLabelSetValue(subpointMap, cells[p], 2));
2638       (*subCells)[subc++] = cells[p];
2639     }
2640     PetscCall(DMPlexRestoreJoin(dm, *nFV, cone, &numCells, &cells));
2641     /* Positive face is not included */
2642   }
2643   PetscFunctionReturn(PETSC_SUCCESS);
2644 }
2645 
2646 static PetscErrorCode DMPlexMarkCohesiveSubmesh_Interpolated(DM dm, DMLabel label, PetscInt value, DMLabel subpointMap, DM subdm)
2647 {
2648   PetscInt *pStart, *pEnd;
2649   PetscInt  dim, cMax, cEnd, c, d;
2650 
2651   PetscFunctionBegin;
2652   PetscCall(DMGetDimension(dm, &dim));
2653   PetscCall(DMPlexGetTensorPrismBounds_Internal(dm, dim, &cMax, &cEnd));
2654   if (cMax < 0) PetscFunctionReturn(PETSC_SUCCESS);
2655   PetscCall(PetscMalloc2(dim + 1, &pStart, dim + 1, &pEnd));
2656   for (d = 0; d <= dim; ++d) PetscCall(DMPlexGetDepthStratum(dm, d, &pStart[d], &pEnd[d]));
2657   for (c = cMax; c < cEnd; ++c) {
2658     const PetscInt *cone;
2659     PetscInt       *closure = NULL;
2660     PetscInt        fconeSize, coneSize, closureSize, cl, val;
2661 
2662     if (label) {
2663       PetscCall(DMLabelGetValue(label, c, &val));
2664       if (val != value) continue;
2665     }
2666     PetscCall(DMPlexGetConeSize(dm, c, &coneSize));
2667     PetscCall(DMPlexGetCone(dm, c, &cone));
2668     PetscCall(DMPlexGetConeSize(dm, cone[0], &fconeSize));
2669     PetscCheck(coneSize == (fconeSize ? fconeSize : 1) + 2, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive cells should separate two cells");
2670     /* Negative face */
2671     PetscCall(DMPlexGetTransitiveClosure(dm, cone[0], PETSC_TRUE, &closureSize, &closure));
2672     for (cl = 0; cl < closureSize * 2; cl += 2) {
2673       const PetscInt point = closure[cl];
2674 
2675       for (d = 0; d <= dim; ++d) {
2676         if ((point >= pStart[d]) && (point < pEnd[d])) {
2677           PetscCall(DMLabelSetValue(subpointMap, point, d));
2678           break;
2679         }
2680       }
2681     }
2682     PetscCall(DMPlexRestoreTransitiveClosure(dm, cone[0], PETSC_TRUE, &closureSize, &closure));
2683     /* Cells -- positive face is not included */
2684     for (cl = 0; cl < 1; ++cl) {
2685       const PetscInt *support;
2686       PetscInt        supportSize, s;
2687 
2688       PetscCall(DMPlexGetSupportSize(dm, cone[cl], &supportSize));
2689       /* PetscCheck(supportSize == 2,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive faces should separate two cells"); */
2690       PetscCall(DMPlexGetSupport(dm, cone[cl], &support));
2691       for (s = 0; s < supportSize; ++s) PetscCall(DMLabelSetValue(subpointMap, support[s], dim));
2692     }
2693   }
2694   PetscCall(PetscFree2(pStart, pEnd));
2695   PetscFunctionReturn(PETSC_SUCCESS);
2696 }
2697 
2698 static PetscErrorCode DMPlexGetFaceOrientation(DM dm, PetscInt cell, PetscInt numCorners, PetscInt indices[], PetscInt oppositeVertex, PetscInt origVertices[], PetscInt faceVertices[], PetscBool *posOriented)
2699 {
2700   MPI_Comm       comm;
2701   PetscBool      posOrient = PETSC_FALSE;
2702   const PetscInt debug     = 0;
2703   PetscInt       cellDim, faceSize, f;
2704 
2705   PetscFunctionBegin;
2706   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
2707   PetscCall(DMGetDimension(dm, &cellDim));
2708   if (debug) PetscCall(PetscPrintf(comm, "cellDim: %" PetscInt_FMT " numCorners: %" PetscInt_FMT "\n", cellDim, numCorners));
2709 
2710   if (cellDim == 1 && numCorners == 2) {
2711     /* Triangle */
2712     faceSize  = numCorners - 1;
2713     posOrient = !(oppositeVertex % 2) ? PETSC_TRUE : PETSC_FALSE;
2714   } else if (cellDim == 2 && numCorners == 3) {
2715     /* Triangle */
2716     faceSize  = numCorners - 1;
2717     posOrient = !(oppositeVertex % 2) ? PETSC_TRUE : PETSC_FALSE;
2718   } else if (cellDim == 3 && numCorners == 4) {
2719     /* Tetrahedron */
2720     faceSize  = numCorners - 1;
2721     posOrient = (oppositeVertex % 2) ? PETSC_TRUE : PETSC_FALSE;
2722   } else if (cellDim == 1 && numCorners == 3) {
2723     /* Quadratic line */
2724     faceSize  = 1;
2725     posOrient = PETSC_TRUE;
2726   } else if (cellDim == 2 && numCorners == 4) {
2727     /* Quads */
2728     faceSize = 2;
2729     if ((indices[1] > indices[0]) && (indices[1] - indices[0] == 1)) {
2730       posOrient = PETSC_TRUE;
2731     } else if ((indices[0] == 3) && (indices[1] == 0)) {
2732       posOrient = PETSC_TRUE;
2733     } else {
2734       if (((indices[0] > indices[1]) && (indices[0] - indices[1] == 1)) || ((indices[0] == 0) && (indices[1] == 3))) {
2735         posOrient = PETSC_FALSE;
2736       } else SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid quad crossedge");
2737     }
2738   } else if (cellDim == 2 && numCorners == 6) {
2739     /* Quadratic triangle (I hate this) */
2740     /* Edges are determined by the first 2 vertices (corners of edges) */
2741     const PetscInt faceSizeTri = 3;
2742     PetscInt       sortedIndices[3], i, iFace;
2743     PetscBool      found                    = PETSC_FALSE;
2744     PetscInt       faceVerticesTriSorted[9] = {
2745       0, 3, 4, /* bottom */
2746       1, 4, 5, /* right */
2747       2, 3, 5, /* left */
2748     };
2749     PetscInt faceVerticesTri[9] = {
2750       0, 3, 4, /* bottom */
2751       1, 4, 5, /* right */
2752       2, 5, 3, /* left */
2753     };
2754 
2755     for (i = 0; i < faceSizeTri; ++i) sortedIndices[i] = indices[i];
2756     PetscCall(PetscSortInt(faceSizeTri, sortedIndices));
2757     for (iFace = 0; iFace < 3; ++iFace) {
2758       const PetscInt ii = iFace * faceSizeTri;
2759       PetscInt       fVertex, cVertex;
2760 
2761       if ((sortedIndices[0] == faceVerticesTriSorted[ii + 0]) && (sortedIndices[1] == faceVerticesTriSorted[ii + 1])) {
2762         for (fVertex = 0; fVertex < faceSizeTri; ++fVertex) {
2763           for (cVertex = 0; cVertex < faceSizeTri; ++cVertex) {
2764             if (indices[cVertex] == faceVerticesTri[ii + fVertex]) {
2765               faceVertices[fVertex] = origVertices[cVertex];
2766               break;
2767             }
2768           }
2769         }
2770         found = PETSC_TRUE;
2771         break;
2772       }
2773     }
2774     PetscCheck(found, comm, PETSC_ERR_ARG_WRONG, "Invalid tri crossface");
2775     if (posOriented) *posOriented = PETSC_TRUE;
2776     PetscFunctionReturn(PETSC_SUCCESS);
2777   } else if (cellDim == 2 && numCorners == 9) {
2778     /* Quadratic quad (I hate this) */
2779     /* Edges are determined by the first 2 vertices (corners of edges) */
2780     const PetscInt faceSizeQuad = 3;
2781     PetscInt       sortedIndices[3], i, iFace;
2782     PetscBool      found                      = PETSC_FALSE;
2783     PetscInt       faceVerticesQuadSorted[12] = {
2784       0, 1, 4, /* bottom */
2785       1, 2, 5, /* right */
2786       2, 3, 6, /* top */
2787       0, 3, 7, /* left */
2788     };
2789     PetscInt faceVerticesQuad[12] = {
2790       0, 1, 4, /* bottom */
2791       1, 2, 5, /* right */
2792       2, 3, 6, /* top */
2793       3, 0, 7, /* left */
2794     };
2795 
2796     for (i = 0; i < faceSizeQuad; ++i) sortedIndices[i] = indices[i];
2797     PetscCall(PetscSortInt(faceSizeQuad, sortedIndices));
2798     for (iFace = 0; iFace < 4; ++iFace) {
2799       const PetscInt ii = iFace * faceSizeQuad;
2800       PetscInt       fVertex, cVertex;
2801 
2802       if ((sortedIndices[0] == faceVerticesQuadSorted[ii + 0]) && (sortedIndices[1] == faceVerticesQuadSorted[ii + 1])) {
2803         for (fVertex = 0; fVertex < faceSizeQuad; ++fVertex) {
2804           for (cVertex = 0; cVertex < faceSizeQuad; ++cVertex) {
2805             if (indices[cVertex] == faceVerticesQuad[ii + fVertex]) {
2806               faceVertices[fVertex] = origVertices[cVertex];
2807               break;
2808             }
2809           }
2810         }
2811         found = PETSC_TRUE;
2812         break;
2813       }
2814     }
2815     PetscCheck(found, comm, PETSC_ERR_ARG_WRONG, "Invalid quad crossface");
2816     if (posOriented) *posOriented = PETSC_TRUE;
2817     PetscFunctionReturn(PETSC_SUCCESS);
2818   } else if (cellDim == 3 && numCorners == 8) {
2819     /* Hexes
2820        A hex is two oriented quads with the normal of the first
2821        pointing up at the second.
2822 
2823           7---6
2824          /|  /|
2825         4---5 |
2826         | 1-|-2
2827         |/  |/
2828         0---3
2829 
2830         Faces are determined by the first 4 vertices (corners of faces) */
2831     const PetscInt faceSizeHex = 4;
2832     PetscInt       sortedIndices[4], i, iFace;
2833     PetscBool      found                     = PETSC_FALSE;
2834     PetscInt       faceVerticesHexSorted[24] = {
2835       0, 1, 2, 3, /* bottom */
2836       4, 5, 6, 7, /* top */
2837       0, 3, 4, 5, /* front */
2838       2, 3, 5, 6, /* right */
2839       1, 2, 6, 7, /* back */
2840       0, 1, 4, 7, /* left */
2841     };
2842     PetscInt faceVerticesHex[24] = {
2843       1, 2, 3, 0, /* bottom */
2844       4, 5, 6, 7, /* top */
2845       0, 3, 5, 4, /* front */
2846       3, 2, 6, 5, /* right */
2847       2, 1, 7, 6, /* back */
2848       1, 0, 4, 7, /* left */
2849     };
2850 
2851     for (i = 0; i < faceSizeHex; ++i) sortedIndices[i] = indices[i];
2852     PetscCall(PetscSortInt(faceSizeHex, sortedIndices));
2853     for (iFace = 0; iFace < 6; ++iFace) {
2854       const PetscInt ii = iFace * faceSizeHex;
2855       PetscInt       fVertex, cVertex;
2856 
2857       if ((sortedIndices[0] == faceVerticesHexSorted[ii + 0]) && (sortedIndices[1] == faceVerticesHexSorted[ii + 1]) && (sortedIndices[2] == faceVerticesHexSorted[ii + 2]) && (sortedIndices[3] == faceVerticesHexSorted[ii + 3])) {
2858         for (fVertex = 0; fVertex < faceSizeHex; ++fVertex) {
2859           for (cVertex = 0; cVertex < faceSizeHex; ++cVertex) {
2860             if (indices[cVertex] == faceVerticesHex[ii + fVertex]) {
2861               faceVertices[fVertex] = origVertices[cVertex];
2862               break;
2863             }
2864           }
2865         }
2866         found = PETSC_TRUE;
2867         break;
2868       }
2869     }
2870     PetscCheck(found, comm, PETSC_ERR_ARG_WRONG, "Invalid hex crossface");
2871     if (posOriented) *posOriented = PETSC_TRUE;
2872     PetscFunctionReturn(PETSC_SUCCESS);
2873   } else if (cellDim == 3 && numCorners == 10) {
2874     /* Quadratic tet */
2875     /* Faces are determined by the first 3 vertices (corners of faces) */
2876     const PetscInt faceSizeTet = 6;
2877     PetscInt       sortedIndices[6], i, iFace;
2878     PetscBool      found                     = PETSC_FALSE;
2879     PetscInt       faceVerticesTetSorted[24] = {
2880       0, 1, 2, 6, 7, 8, /* bottom */
2881       0, 3, 4, 6, 7, 9, /* front */
2882       1, 4, 5, 7, 8, 9, /* right */
2883       2, 3, 5, 6, 8, 9, /* left */
2884     };
2885     PetscInt faceVerticesTet[24] = {
2886       0, 1, 2, 6, 7, 8, /* bottom */
2887       0, 4, 3, 6, 7, 9, /* front */
2888       1, 5, 4, 7, 8, 9, /* right */
2889       2, 3, 5, 8, 6, 9, /* left */
2890     };
2891 
2892     for (i = 0; i < faceSizeTet; ++i) sortedIndices[i] = indices[i];
2893     PetscCall(PetscSortInt(faceSizeTet, sortedIndices));
2894     for (iFace = 0; iFace < 4; ++iFace) {
2895       const PetscInt ii = iFace * faceSizeTet;
2896       PetscInt       fVertex, cVertex;
2897 
2898       if ((sortedIndices[0] == faceVerticesTetSorted[ii + 0]) && (sortedIndices[1] == faceVerticesTetSorted[ii + 1]) && (sortedIndices[2] == faceVerticesTetSorted[ii + 2]) && (sortedIndices[3] == faceVerticesTetSorted[ii + 3])) {
2899         for (fVertex = 0; fVertex < faceSizeTet; ++fVertex) {
2900           for (cVertex = 0; cVertex < faceSizeTet; ++cVertex) {
2901             if (indices[cVertex] == faceVerticesTet[ii + fVertex]) {
2902               faceVertices[fVertex] = origVertices[cVertex];
2903               break;
2904             }
2905           }
2906         }
2907         found = PETSC_TRUE;
2908         break;
2909       }
2910     }
2911     PetscCheck(found, comm, PETSC_ERR_ARG_WRONG, "Invalid tet crossface");
2912     if (posOriented) *posOriented = PETSC_TRUE;
2913     PetscFunctionReturn(PETSC_SUCCESS);
2914   } else if (cellDim == 3 && numCorners == 27) {
2915     /* Quadratic hexes (I hate this)
2916        A hex is two oriented quads with the normal of the first
2917        pointing up at the second.
2918 
2919          7---6
2920         /|  /|
2921        4---5 |
2922        | 3-|-2
2923        |/  |/
2924        0---1
2925 
2926        Faces are determined by the first 4 vertices (corners of faces) */
2927     const PetscInt faceSizeQuadHex = 9;
2928     PetscInt       sortedIndices[9], i, iFace;
2929     PetscBool      found                         = PETSC_FALSE;
2930     PetscInt       faceVerticesQuadHexSorted[54] = {
2931       0, 1, 2, 3, 8,  9,  10, 11, 24, /* bottom */
2932       4, 5, 6, 7, 12, 13, 14, 15, 25, /* top */
2933       0, 1, 4, 5, 8,  12, 16, 17, 22, /* front */
2934       1, 2, 5, 6, 9,  13, 17, 18, 21, /* right */
2935       2, 3, 6, 7, 10, 14, 18, 19, 23, /* back */
2936       0, 3, 4, 7, 11, 15, 16, 19, 20, /* left */
2937     };
2938     PetscInt faceVerticesQuadHex[54] = {
2939       3, 2, 1, 0, 10, 9,  8,  11, 24, /* bottom */
2940       4, 5, 6, 7, 12, 13, 14, 15, 25, /* top */
2941       0, 1, 5, 4, 8,  17, 12, 16, 22, /* front */
2942       1, 2, 6, 5, 9,  18, 13, 17, 21, /* right */
2943       2, 3, 7, 6, 10, 19, 14, 18, 23, /* back */
2944       3, 0, 4, 7, 11, 16, 15, 19, 20  /* left */
2945     };
2946 
2947     for (i = 0; i < faceSizeQuadHex; ++i) sortedIndices[i] = indices[i];
2948     PetscCall(PetscSortInt(faceSizeQuadHex, sortedIndices));
2949     for (iFace = 0; iFace < 6; ++iFace) {
2950       const PetscInt ii = iFace * faceSizeQuadHex;
2951       PetscInt       fVertex, cVertex;
2952 
2953       if ((sortedIndices[0] == faceVerticesQuadHexSorted[ii + 0]) && (sortedIndices[1] == faceVerticesQuadHexSorted[ii + 1]) && (sortedIndices[2] == faceVerticesQuadHexSorted[ii + 2]) && (sortedIndices[3] == faceVerticesQuadHexSorted[ii + 3])) {
2954         for (fVertex = 0; fVertex < faceSizeQuadHex; ++fVertex) {
2955           for (cVertex = 0; cVertex < faceSizeQuadHex; ++cVertex) {
2956             if (indices[cVertex] == faceVerticesQuadHex[ii + fVertex]) {
2957               faceVertices[fVertex] = origVertices[cVertex];
2958               break;
2959             }
2960           }
2961         }
2962         found = PETSC_TRUE;
2963         break;
2964       }
2965     }
2966     PetscCheck(found, comm, PETSC_ERR_ARG_WRONG, "Invalid hex crossface");
2967     if (posOriented) *posOriented = PETSC_TRUE;
2968     PetscFunctionReturn(PETSC_SUCCESS);
2969   } else SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Unknown cell type for faceOrientation().");
2970   if (!posOrient) {
2971     if (debug) PetscCall(PetscPrintf(comm, "  Reversing initial face orientation\n"));
2972     for (f = 0; f < faceSize; ++f) faceVertices[f] = origVertices[faceSize - 1 - f];
2973   } else {
2974     if (debug) PetscCall(PetscPrintf(comm, "  Keeping initial face orientation\n"));
2975     for (f = 0; f < faceSize; ++f) faceVertices[f] = origVertices[f];
2976   }
2977   if (posOriented) *posOriented = posOrient;
2978   PetscFunctionReturn(PETSC_SUCCESS);
2979 }
2980 
2981 /*@
2982   DMPlexGetOrientedFace - Given a cell and a face, as a set of vertices, return the oriented face, as a set of vertices,
2983   in faceVertices. The orientation is such that the face normal points out of the cell
2984 
2985   Not Collective
2986 
2987   Input Parameters:
2988 + dm           - The original mesh
2989 . cell         - The cell mesh point
2990 . faceSize     - The number of vertices on the face
2991 . face         - The face vertices
2992 . numCorners   - The number of vertices on the cell
2993 . indices      - Local numbering of face vertices in cell cone
2994 - origVertices - Original face vertices
2995 
2996   Output Parameters:
2997 + faceVertices - The face vertices properly oriented
2998 - posOriented  - `PETSC_TRUE` if the face was oriented with outward normal
2999 
3000   Level: developer
3001 
3002 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetCone()`
3003 @*/
3004 PetscErrorCode DMPlexGetOrientedFace(DM dm, PetscInt cell, PetscInt faceSize, const PetscInt face[], PetscInt numCorners, PetscInt indices[], PetscInt origVertices[], PetscInt faceVertices[], PetscBool *posOriented)
3005 {
3006   const PetscInt *cone = NULL;
3007   PetscInt        coneSize, v, f, v2;
3008   PetscInt        oppositeVertex = -1;
3009 
3010   PetscFunctionBegin;
3011   PetscCall(DMPlexGetConeSize(dm, cell, &coneSize));
3012   PetscCall(DMPlexGetCone(dm, cell, &cone));
3013   for (v = 0, v2 = 0; v < coneSize; ++v) {
3014     PetscBool found = PETSC_FALSE;
3015 
3016     for (f = 0; f < faceSize; ++f) {
3017       if (face[f] == cone[v]) {
3018         found = PETSC_TRUE;
3019         break;
3020       }
3021     }
3022     if (found) {
3023       indices[v2]      = v;
3024       origVertices[v2] = cone[v];
3025       ++v2;
3026     } else {
3027       oppositeVertex = v;
3028     }
3029   }
3030   PetscCall(DMPlexGetFaceOrientation(dm, cell, numCorners, indices, oppositeVertex, origVertices, faceVertices, posOriented));
3031   PetscFunctionReturn(PETSC_SUCCESS);
3032 }
3033 
3034 /*
3035   DMPlexInsertFace_Internal - Puts a face into the mesh
3036 
3037   Not Collective
3038 
3039   Input Parameters:
3040   + dm              - The `DMPLEX`
3041   . numFaceVertex   - The number of vertices in the face
3042   . faceVertices    - The vertices in the face for `dm`
3043   . subfaceVertices - The vertices in the face for subdm
3044   . numCorners      - The number of vertices in the `cell`
3045   . cell            - A cell in `dm` containing the face
3046   . subcell         - A cell in subdm containing the face
3047   . firstFace       - First face in the mesh
3048   - newFacePoint    - Next face in the mesh
3049 
3050   Output Parameter:
3051   . newFacePoint - Contains next face point number on input, updated on output
3052 
3053   Level: developer
3054 */
3055 static PetscErrorCode DMPlexInsertFace_Internal(DM dm, DM subdm, PetscInt numFaceVertices, const PetscInt faceVertices[], const PetscInt subfaceVertices[], PetscInt numCorners, PetscInt cell, PetscInt subcell, PetscInt firstFace, PetscInt *newFacePoint)
3056 {
3057   MPI_Comm        comm;
3058   DM_Plex        *submesh = (DM_Plex *)subdm->data;
3059   const PetscInt *faces;
3060   PetscInt        numFaces, coneSize;
3061 
3062   PetscFunctionBegin;
3063   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
3064   PetscCall(DMPlexGetConeSize(subdm, subcell, &coneSize));
3065   PetscCheck(coneSize == 1, comm, PETSC_ERR_ARG_OUTOFRANGE, "Cone size of cell %" PetscInt_FMT " is %" PetscInt_FMT " != 1", cell, coneSize);
3066 #if 0
3067   /* Cannot use this because support() has not been constructed yet */
3068   PetscCall(DMPlexGetJoin(subdm, numFaceVertices, subfaceVertices, &numFaces, &faces));
3069 #else
3070   {
3071     PetscInt f;
3072 
3073     numFaces = 0;
3074     PetscCall(DMGetWorkArray(subdm, 1, MPIU_INT, (void **)&faces));
3075     for (f = firstFace; f < *newFacePoint; ++f) {
3076       PetscInt dof, off, d;
3077 
3078       PetscCall(PetscSectionGetDof(submesh->coneSection, f, &dof));
3079       PetscCall(PetscSectionGetOffset(submesh->coneSection, f, &off));
3080       /* Yes, I know this is quadratic, but I expect the sizes to be <5 */
3081       for (d = 0; d < dof; ++d) {
3082         const PetscInt p = submesh->cones[off + d];
3083         PetscInt       v;
3084 
3085         for (v = 0; v < numFaceVertices; ++v) {
3086           if (subfaceVertices[v] == p) break;
3087         }
3088         if (v == numFaceVertices) break;
3089       }
3090       if (d == dof) {
3091         numFaces               = 1;
3092         ((PetscInt *)faces)[0] = f;
3093       }
3094     }
3095   }
3096 #endif
3097   PetscCheck(numFaces <= 1, comm, PETSC_ERR_ARG_WRONG, "Vertex set had %" PetscInt_FMT " faces, not one", numFaces);
3098   if (numFaces == 1) {
3099     /* Add the other cell neighbor for this face */
3100     PetscCall(DMPlexSetCone(subdm, subcell, faces));
3101   } else {
3102     PetscInt *indices, *origVertices, *orientedVertices, *orientedSubVertices, v, ov;
3103     PetscBool posOriented;
3104 
3105     PetscCall(DMGetWorkArray(subdm, 4 * numFaceVertices * sizeof(PetscInt), MPIU_INT, &orientedVertices));
3106     origVertices        = &orientedVertices[numFaceVertices];
3107     indices             = &orientedVertices[numFaceVertices * 2];
3108     orientedSubVertices = &orientedVertices[numFaceVertices * 3];
3109     PetscCall(DMPlexGetOrientedFace(dm, cell, numFaceVertices, faceVertices, numCorners, indices, origVertices, orientedVertices, &posOriented));
3110     /* TODO: I know that routine should return a permutation, not the indices */
3111     for (v = 0; v < numFaceVertices; ++v) {
3112       const PetscInt vertex = faceVertices[v], subvertex = subfaceVertices[v];
3113       for (ov = 0; ov < numFaceVertices; ++ov) {
3114         if (orientedVertices[ov] == vertex) {
3115           orientedSubVertices[ov] = subvertex;
3116           break;
3117         }
3118       }
3119       PetscCheck(ov != numFaceVertices, comm, PETSC_ERR_PLIB, "Could not find face vertex %" PetscInt_FMT " in orientated set", vertex);
3120     }
3121     PetscCall(DMPlexSetCone(subdm, *newFacePoint, orientedSubVertices));
3122     PetscCall(DMPlexSetCone(subdm, subcell, newFacePoint));
3123     PetscCall(DMRestoreWorkArray(subdm, 4 * numFaceVertices * sizeof(PetscInt), MPIU_INT, &orientedVertices));
3124     ++(*newFacePoint);
3125   }
3126 #if 0
3127   PetscCall(DMPlexRestoreJoin(subdm, numFaceVertices, subfaceVertices, &numFaces, &faces));
3128 #else
3129   PetscCall(DMRestoreWorkArray(subdm, 1, MPIU_INT, (void **)&faces));
3130 #endif
3131   PetscFunctionReturn(PETSC_SUCCESS);
3132 }
3133 
3134 static PetscErrorCode DMPlexCreateSubmesh_Uninterpolated(DM dm, DMLabel vertexLabel, PetscInt value, DM subdm)
3135 {
3136   MPI_Comm        comm;
3137   DMLabel         subpointMap;
3138   IS              subvertexIS, subcellIS;
3139   const PetscInt *subVertices, *subCells;
3140   PetscInt        numSubVertices, firstSubVertex, numSubCells;
3141   PetscInt       *subface, maxConeSize, numSubFaces = 0, firstSubFace, newFacePoint, nFV = 0;
3142   PetscInt        vStart, vEnd, c, f;
3143 
3144   PetscFunctionBegin;
3145   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
3146   /* Create subpointMap which marks the submesh */
3147   PetscCall(DMLabelCreate(PETSC_COMM_SELF, "subpoint_map", &subpointMap));
3148   PetscCall(DMPlexSetSubpointMap(subdm, subpointMap));
3149   if (vertexLabel) PetscCall(DMPlexMarkSubmesh_Uninterpolated(dm, vertexLabel, value, subpointMap, &numSubFaces, &nFV, subdm));
3150   /* Setup chart */
3151   PetscCall(DMLabelGetStratumSize(subpointMap, 0, &numSubVertices));
3152   PetscCall(DMLabelGetStratumSize(subpointMap, 2, &numSubCells));
3153   PetscCall(DMPlexSetChart(subdm, 0, numSubCells + numSubFaces + numSubVertices));
3154   PetscCall(DMPlexSetVTKCellHeight(subdm, 1));
3155   /* Set cone sizes */
3156   firstSubVertex = numSubCells;
3157   firstSubFace   = numSubCells + numSubVertices;
3158   newFacePoint   = firstSubFace;
3159   PetscCall(DMLabelGetStratumIS(subpointMap, 0, &subvertexIS));
3160   if (subvertexIS) PetscCall(ISGetIndices(subvertexIS, &subVertices));
3161   PetscCall(DMLabelGetStratumIS(subpointMap, 2, &subcellIS));
3162   if (subcellIS) PetscCall(ISGetIndices(subcellIS, &subCells));
3163   for (c = 0; c < numSubCells; ++c) PetscCall(DMPlexSetConeSize(subdm, c, 1));
3164   for (f = firstSubFace; f < firstSubFace + numSubFaces; ++f) PetscCall(DMPlexSetConeSize(subdm, f, nFV));
3165   PetscCall(DMSetUp(subdm));
3166   PetscCall(DMLabelDestroy(&subpointMap));
3167   /* Create face cones */
3168   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
3169   PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, NULL));
3170   PetscCall(DMGetWorkArray(subdm, maxConeSize, MPIU_INT, (void **)&subface));
3171   for (c = 0; c < numSubCells; ++c) {
3172     const PetscInt cell    = subCells[c];
3173     const PetscInt subcell = c;
3174     PetscInt      *closure = NULL;
3175     PetscInt       closureSize, cl, numCorners = 0, faceSize = 0;
3176 
3177     PetscCall(DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
3178     for (cl = 0; cl < closureSize * 2; cl += 2) {
3179       const PetscInt point = closure[cl];
3180       PetscInt       subVertex;
3181 
3182       if ((point >= vStart) && (point < vEnd)) {
3183         ++numCorners;
3184         PetscCall(PetscFindInt(point, numSubVertices, subVertices, &subVertex));
3185         if (subVertex >= 0) {
3186           closure[faceSize] = point;
3187           subface[faceSize] = firstSubVertex + subVertex;
3188           ++faceSize;
3189         }
3190       }
3191     }
3192     PetscCheck(faceSize <= nFV, comm, PETSC_ERR_ARG_WRONG, "Invalid submesh: Too many vertices %" PetscInt_FMT " of an element on the surface", faceSize);
3193     if (faceSize == nFV) PetscCall(DMPlexInsertFace_Internal(dm, subdm, faceSize, closure, subface, numCorners, cell, subcell, firstSubFace, &newFacePoint));
3194     PetscCall(DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
3195   }
3196   PetscCall(DMRestoreWorkArray(subdm, maxConeSize, MPIU_INT, (void **)&subface));
3197   PetscCall(DMPlexSymmetrize(subdm));
3198   PetscCall(DMPlexStratify(subdm));
3199   /* Build coordinates */
3200   {
3201     PetscSection coordSection, subCoordSection;
3202     Vec          coordinates, subCoordinates;
3203     PetscScalar *coords, *subCoords;
3204     PetscInt     numComp, coordSize, v;
3205     const char  *name;
3206 
3207     PetscCall(DMGetCoordinateSection(dm, &coordSection));
3208     PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
3209     PetscCall(DMGetCoordinateSection(subdm, &subCoordSection));
3210     PetscCall(PetscSectionSetNumFields(subCoordSection, 1));
3211     PetscCall(PetscSectionGetFieldComponents(coordSection, 0, &numComp));
3212     PetscCall(PetscSectionSetFieldComponents(subCoordSection, 0, numComp));
3213     PetscCall(PetscSectionSetChart(subCoordSection, firstSubVertex, firstSubVertex + numSubVertices));
3214     for (v = 0; v < numSubVertices; ++v) {
3215       const PetscInt vertex    = subVertices[v];
3216       const PetscInt subvertex = firstSubVertex + v;
3217       PetscInt       dof;
3218 
3219       PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3220       PetscCall(PetscSectionSetDof(subCoordSection, subvertex, dof));
3221       PetscCall(PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof));
3222     }
3223     PetscCall(PetscSectionSetUp(subCoordSection));
3224     PetscCall(PetscSectionGetStorageSize(subCoordSection, &coordSize));
3225     PetscCall(VecCreate(PETSC_COMM_SELF, &subCoordinates));
3226     PetscCall(PetscObjectGetName((PetscObject)coordinates, &name));
3227     PetscCall(PetscObjectSetName((PetscObject)subCoordinates, name));
3228     PetscCall(VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE));
3229     PetscCall(VecSetType(subCoordinates, VECSTANDARD));
3230     if (coordSize) {
3231       PetscCall(VecGetArray(coordinates, &coords));
3232       PetscCall(VecGetArray(subCoordinates, &subCoords));
3233       for (v = 0; v < numSubVertices; ++v) {
3234         const PetscInt vertex    = subVertices[v];
3235         const PetscInt subvertex = firstSubVertex + v;
3236         PetscInt       dof, off, sdof, soff, d;
3237 
3238         PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3239         PetscCall(PetscSectionGetOffset(coordSection, vertex, &off));
3240         PetscCall(PetscSectionGetDof(subCoordSection, subvertex, &sdof));
3241         PetscCall(PetscSectionGetOffset(subCoordSection, subvertex, &soff));
3242         PetscCheck(dof == sdof, comm, PETSC_ERR_PLIB, "Coordinate dimension %" PetscInt_FMT " on subvertex %" PetscInt_FMT ", vertex %" PetscInt_FMT " should be %" PetscInt_FMT, sdof, subvertex, vertex, dof);
3243         for (d = 0; d < dof; ++d) subCoords[soff + d] = coords[off + d];
3244       }
3245       PetscCall(VecRestoreArray(coordinates, &coords));
3246       PetscCall(VecRestoreArray(subCoordinates, &subCoords));
3247     }
3248     PetscCall(DMSetCoordinatesLocal(subdm, subCoordinates));
3249     PetscCall(VecDestroy(&subCoordinates));
3250   }
3251   /* Cleanup */
3252   if (subvertexIS) PetscCall(ISRestoreIndices(subvertexIS, &subVertices));
3253   PetscCall(ISDestroy(&subvertexIS));
3254   if (subcellIS) PetscCall(ISRestoreIndices(subcellIS, &subCells));
3255   PetscCall(ISDestroy(&subcellIS));
3256   PetscFunctionReturn(PETSC_SUCCESS);
3257 }
3258 
3259 /* TODO: Fix this to properly propagate up error conditions it may find */
3260 static inline PetscInt DMPlexFilterPoint_Internal(PetscInt point, PetscInt firstSubPoint, PetscInt numSubPoints, const PetscInt subPoints[])
3261 {
3262   PetscInt       subPoint;
3263   PetscErrorCode ierr;
3264 
3265   ierr = PetscFindInt(point, numSubPoints, subPoints, &subPoint);
3266   if (ierr) return -1;
3267   return subPoint < 0 ? subPoint : firstSubPoint + subPoint;
3268 }
3269 
3270 /* TODO: Fix this to properly propagate up error conditions it may find */
3271 static inline PetscInt DMPlexFilterPointPerm_Internal(PetscInt point, PetscInt firstSubPoint, PetscInt numSubPoints, const PetscInt subPoints[], const PetscInt subIndices[])
3272 {
3273   PetscInt       subPoint;
3274   PetscErrorCode ierr;
3275 
3276   ierr = PetscFindInt(point, numSubPoints, subPoints, &subPoint);
3277   if (ierr) return -1;
3278   return subPoint < 0 ? subPoint : firstSubPoint + (subIndices ? subIndices[subPoint] : subPoint);
3279 }
3280 
3281 static PetscErrorCode DMPlexFilterLabels_Internal(DM dm, const PetscInt numSubPoints[], const PetscInt *subpoints[], const PetscInt firstSubPoint[], DM subdm)
3282 {
3283   DMLabel  depthLabel;
3284   PetscInt Nl, l, d;
3285 
3286   PetscFunctionBegin;
3287   // Reset depth label for fast lookup
3288   PetscCall(DMPlexGetDepthLabel(dm, &depthLabel));
3289   PetscCall(DMLabelMakeAllInvalid_Internal(depthLabel));
3290   PetscCall(DMGetNumLabels(dm, &Nl));
3291   for (l = 0; l < Nl; ++l) {
3292     DMLabel         label, newlabel;
3293     const char     *lname;
3294     PetscBool       isDepth, isDim, isCelltype, isVTK;
3295     IS              valueIS;
3296     const PetscInt *values;
3297     PetscInt        Nv, v;
3298 
3299     PetscCall(DMGetLabelName(dm, l, &lname));
3300     PetscCall(PetscStrcmp(lname, "depth", &isDepth));
3301     PetscCall(PetscStrcmp(lname, "dim", &isDim));
3302     PetscCall(PetscStrcmp(lname, "celltype", &isCelltype));
3303     PetscCall(PetscStrcmp(lname, "vtk", &isVTK));
3304     if (isDepth || isDim || isCelltype || isVTK) continue;
3305     PetscCall(DMCreateLabel(subdm, lname));
3306     PetscCall(DMGetLabel(dm, lname, &label));
3307     PetscCall(DMGetLabel(subdm, lname, &newlabel));
3308     PetscCall(DMLabelGetDefaultValue(label, &v));
3309     PetscCall(DMLabelSetDefaultValue(newlabel, v));
3310     PetscCall(DMLabelGetValueIS(label, &valueIS));
3311     PetscCall(ISGetLocalSize(valueIS, &Nv));
3312     PetscCall(ISGetIndices(valueIS, &values));
3313     for (v = 0; v < Nv; ++v) {
3314       IS              pointIS;
3315       const PetscInt *points;
3316       PetscInt        Np, p;
3317 
3318       PetscCall(DMLabelGetStratumIS(label, values[v], &pointIS));
3319       PetscCall(ISGetLocalSize(pointIS, &Np));
3320       PetscCall(ISGetIndices(pointIS, &points));
3321       for (p = 0; p < Np; ++p) {
3322         const PetscInt point = points[p];
3323         PetscInt       subp;
3324 
3325         PetscCall(DMPlexGetPointDepth(dm, point, &d));
3326         subp = DMPlexFilterPoint_Internal(point, firstSubPoint[d], numSubPoints[d], subpoints[d]);
3327         if (subp >= 0) PetscCall(DMLabelSetValue(newlabel, subp, values[v]));
3328       }
3329       PetscCall(ISRestoreIndices(pointIS, &points));
3330       PetscCall(ISDestroy(&pointIS));
3331     }
3332     PetscCall(ISRestoreIndices(valueIS, &values));
3333     PetscCall(ISDestroy(&valueIS));
3334   }
3335   PetscFunctionReturn(PETSC_SUCCESS);
3336 }
3337 
3338 static PetscErrorCode DMPlexCreateSubmeshGeneric_Interpolated(DM dm, DMLabel label, PetscInt value, PetscBool markedFaces, PetscBool isCohesive, PetscInt cellHeight, DM subdm)
3339 {
3340   MPI_Comm         comm;
3341   DMLabel          subpointMap;
3342   IS              *subpointIS;
3343   const PetscInt **subpoints;
3344   PetscInt        *numSubPoints, *firstSubPoint, *coneNew, *orntNew;
3345   PetscInt         totSubPoints = 0, maxConeSize, dim, sdim, cdim, p, d, v;
3346   PetscMPIInt      rank;
3347 
3348   PetscFunctionBegin;
3349   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
3350   PetscCallMPI(MPI_Comm_rank(comm, &rank));
3351   /* Create subpointMap which marks the submesh */
3352   PetscCall(DMLabelCreate(PETSC_COMM_SELF, "subpoint_map", &subpointMap));
3353   PetscCall(DMPlexSetSubpointMap(subdm, subpointMap));
3354   if (cellHeight) {
3355     if (isCohesive) PetscCall(DMPlexMarkCohesiveSubmesh_Interpolated(dm, label, value, subpointMap, subdm));
3356     else PetscCall(DMPlexMarkSubmesh_Interpolated(dm, label, value, markedFaces, subpointMap, subdm));
3357   } else {
3358     DMLabel         depth;
3359     IS              pointIS;
3360     const PetscInt *points;
3361     PetscInt        numPoints = 0;
3362 
3363     PetscCall(DMPlexGetDepthLabel(dm, &depth));
3364     PetscCall(DMLabelGetStratumIS(label, value, &pointIS));
3365     if (pointIS) {
3366       PetscCall(ISGetIndices(pointIS, &points));
3367       PetscCall(ISGetLocalSize(pointIS, &numPoints));
3368     }
3369     for (p = 0; p < numPoints; ++p) {
3370       PetscInt *closure = NULL;
3371       PetscInt  closureSize, c, pdim;
3372 
3373       PetscCall(DMPlexGetTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closure));
3374       for (c = 0; c < closureSize * 2; c += 2) {
3375         PetscCall(DMLabelGetValue(depth, closure[c], &pdim));
3376         PetscCall(DMLabelSetValue(subpointMap, closure[c], pdim));
3377       }
3378       PetscCall(DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closure));
3379     }
3380     if (pointIS) PetscCall(ISRestoreIndices(pointIS, &points));
3381     PetscCall(ISDestroy(&pointIS));
3382   }
3383   /* Setup chart */
3384   PetscCall(DMGetDimension(dm, &dim));
3385   PetscCall(DMGetCoordinateDim(dm, &cdim));
3386   PetscCall(PetscMalloc4(dim + 1, &numSubPoints, dim + 1, &firstSubPoint, dim + 1, &subpointIS, dim + 1, &subpoints));
3387   for (d = 0; d <= dim; ++d) {
3388     PetscCall(DMLabelGetStratumSize(subpointMap, d, &numSubPoints[d]));
3389     totSubPoints += numSubPoints[d];
3390   }
3391   // Determine submesh dimension
3392   PetscCall(DMGetDimension(subdm, &sdim));
3393   if (sdim > 0) {
3394     // Calling function knows what dimension to use, and we include neighboring cells as well
3395     sdim = dim;
3396   } else {
3397     // We reset the subdimension based on what is being selected
3398     PetscInt lsdim;
3399     for (lsdim = dim; lsdim >= 0; --lsdim)
3400       if (numSubPoints[lsdim]) break;
3401     PetscCall(MPIU_Allreduce(&lsdim, &sdim, 1, MPIU_INT, MPI_MAX, comm));
3402     PetscCall(DMSetDimension(subdm, sdim));
3403     PetscCall(DMSetCoordinateDim(subdm, cdim));
3404   }
3405   PetscCall(DMPlexSetChart(subdm, 0, totSubPoints));
3406   PetscCall(DMPlexSetVTKCellHeight(subdm, cellHeight));
3407   /* Set cone sizes */
3408   firstSubPoint[sdim] = 0;
3409   firstSubPoint[0]    = firstSubPoint[sdim] + numSubPoints[sdim];
3410   if (sdim > 1) firstSubPoint[sdim - 1] = firstSubPoint[0] + numSubPoints[0];
3411   if (sdim > 2) firstSubPoint[sdim - 2] = firstSubPoint[sdim - 1] + numSubPoints[sdim - 1];
3412   for (d = 0; d <= sdim; ++d) {
3413     PetscCall(DMLabelGetStratumIS(subpointMap, d, &subpointIS[d]));
3414     if (subpointIS[d]) PetscCall(ISGetIndices(subpointIS[d], &subpoints[d]));
3415   }
3416   /* We do not want this label automatically computed, instead we compute it here */
3417   PetscCall(DMCreateLabel(subdm, "celltype"));
3418   for (d = 0; d <= sdim; ++d) {
3419     for (p = 0; p < numSubPoints[d]; ++p) {
3420       const PetscInt  point    = subpoints[d][p];
3421       const PetscInt  subpoint = firstSubPoint[d] + p;
3422       const PetscInt *cone;
3423       PetscInt        coneSize;
3424 
3425       PetscCall(DMPlexGetConeSize(dm, point, &coneSize));
3426       if (cellHeight && (d == sdim)) {
3427         PetscInt coneSizeNew, c, val;
3428 
3429         PetscCall(DMPlexGetCone(dm, point, &cone));
3430         for (c = 0, coneSizeNew = 0; c < coneSize; ++c) {
3431           PetscCall(DMLabelGetValue(subpointMap, cone[c], &val));
3432           if (val >= 0) coneSizeNew++;
3433         }
3434         PetscCall(DMPlexSetConeSize(subdm, subpoint, coneSizeNew));
3435         PetscCall(DMPlexSetCellType(subdm, subpoint, DM_POLYTOPE_FV_GHOST));
3436       } else {
3437         DMPolytopeType ct;
3438 
3439         PetscCall(DMPlexSetConeSize(subdm, subpoint, coneSize));
3440         PetscCall(DMPlexGetCellType(dm, point, &ct));
3441         PetscCall(DMPlexSetCellType(subdm, subpoint, ct));
3442       }
3443     }
3444   }
3445   PetscCall(DMLabelDestroy(&subpointMap));
3446   PetscCall(DMSetUp(subdm));
3447   /* Set cones */
3448   PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, NULL));
3449   PetscCall(PetscMalloc2(maxConeSize, &coneNew, maxConeSize, &orntNew));
3450   for (d = 0; d <= sdim; ++d) {
3451     for (p = 0; p < numSubPoints[d]; ++p) {
3452       const PetscInt  point    = subpoints[d][p];
3453       const PetscInt  subpoint = firstSubPoint[d] + p;
3454       const PetscInt *cone, *ornt;
3455       PetscInt        coneSize, subconeSize, coneSizeNew, c, subc, fornt = 0;
3456 
3457       if (d == sdim - 1) {
3458         const PetscInt *support, *cone, *ornt;
3459         PetscInt        supportSize, coneSize, s, subc;
3460 
3461         PetscCall(DMPlexGetSupport(dm, point, &support));
3462         PetscCall(DMPlexGetSupportSize(dm, point, &supportSize));
3463         for (s = 0; s < supportSize; ++s) {
3464           PetscBool isHybrid = PETSC_FALSE;
3465 
3466           PetscCall(DMPlexCellIsHybrid_Internal(dm, support[s], &isHybrid));
3467           if (!isHybrid) continue;
3468           PetscCall(PetscFindInt(support[s], numSubPoints[d + 1], subpoints[d + 1], &subc));
3469           if (subc >= 0) {
3470             const PetscInt ccell = subpoints[d + 1][subc];
3471 
3472             PetscCall(DMPlexGetCone(dm, ccell, &cone));
3473             PetscCall(DMPlexGetConeSize(dm, ccell, &coneSize));
3474             PetscCall(DMPlexGetConeOrientation(dm, ccell, &ornt));
3475             for (c = 0; c < coneSize; ++c) {
3476               if (cone[c] == point) {
3477                 fornt = ornt[c];
3478                 break;
3479               }
3480             }
3481             break;
3482           }
3483         }
3484       }
3485       PetscCall(DMPlexGetConeSize(dm, point, &coneSize));
3486       PetscCall(DMPlexGetConeSize(subdm, subpoint, &subconeSize));
3487       PetscCall(DMPlexGetCone(dm, point, &cone));
3488       PetscCall(DMPlexGetConeOrientation(dm, point, &ornt));
3489       for (c = 0, coneSizeNew = 0; c < coneSize; ++c) {
3490         PetscCall(PetscFindInt(cone[c], numSubPoints[d - 1], subpoints[d - 1], &subc));
3491         if (subc >= 0) {
3492           coneNew[coneSizeNew] = firstSubPoint[d - 1] + subc;
3493           orntNew[coneSizeNew] = ornt[c];
3494           ++coneSizeNew;
3495         }
3496       }
3497       PetscCheck(coneSizeNew == subconeSize, comm, PETSC_ERR_PLIB, "Number of cone points located %" PetscInt_FMT " does not match subcone size %" PetscInt_FMT, coneSizeNew, subconeSize);
3498       PetscCall(DMPlexSetCone(subdm, subpoint, coneNew));
3499       PetscCall(DMPlexSetConeOrientation(subdm, subpoint, orntNew));
3500       if (fornt < 0) PetscCall(DMPlexOrientPoint(subdm, subpoint, fornt));
3501     }
3502   }
3503   PetscCall(PetscFree2(coneNew, orntNew));
3504   PetscCall(DMPlexSymmetrize(subdm));
3505   PetscCall(DMPlexStratify(subdm));
3506   /* Build coordinates */
3507   {
3508     PetscSection coordSection, subCoordSection;
3509     Vec          coordinates, subCoordinates;
3510     PetscScalar *coords, *subCoords;
3511     PetscInt     cdim, numComp, coordSize;
3512     const char  *name;
3513 
3514     PetscCall(DMGetCoordinateDim(dm, &cdim));
3515     PetscCall(DMGetCoordinateSection(dm, &coordSection));
3516     PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
3517     PetscCall(DMGetCoordinateSection(subdm, &subCoordSection));
3518     PetscCall(PetscSectionSetNumFields(subCoordSection, 1));
3519     PetscCall(PetscSectionGetFieldComponents(coordSection, 0, &numComp));
3520     PetscCall(PetscSectionSetFieldComponents(subCoordSection, 0, numComp));
3521     PetscCall(PetscSectionSetChart(subCoordSection, firstSubPoint[0], firstSubPoint[0] + numSubPoints[0]));
3522     for (v = 0; v < numSubPoints[0]; ++v) {
3523       const PetscInt vertex    = subpoints[0][v];
3524       const PetscInt subvertex = firstSubPoint[0] + v;
3525       PetscInt       dof;
3526 
3527       PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3528       PetscCall(PetscSectionSetDof(subCoordSection, subvertex, dof));
3529       PetscCall(PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof));
3530     }
3531     PetscCall(PetscSectionSetUp(subCoordSection));
3532     PetscCall(PetscSectionGetStorageSize(subCoordSection, &coordSize));
3533     PetscCall(VecCreate(PETSC_COMM_SELF, &subCoordinates));
3534     PetscCall(PetscObjectGetName((PetscObject)coordinates, &name));
3535     PetscCall(PetscObjectSetName((PetscObject)subCoordinates, name));
3536     PetscCall(VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE));
3537     PetscCall(VecSetBlockSize(subCoordinates, cdim));
3538     PetscCall(VecSetType(subCoordinates, VECSTANDARD));
3539     PetscCall(VecGetArray(coordinates, &coords));
3540     PetscCall(VecGetArray(subCoordinates, &subCoords));
3541     for (v = 0; v < numSubPoints[0]; ++v) {
3542       const PetscInt vertex    = subpoints[0][v];
3543       const PetscInt subvertex = firstSubPoint[0] + v;
3544       PetscInt       dof, off, sdof, soff, d;
3545 
3546       PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3547       PetscCall(PetscSectionGetOffset(coordSection, vertex, &off));
3548       PetscCall(PetscSectionGetDof(subCoordSection, subvertex, &sdof));
3549       PetscCall(PetscSectionGetOffset(subCoordSection, subvertex, &soff));
3550       PetscCheck(dof == sdof, comm, PETSC_ERR_PLIB, "Coordinate dimension %" PetscInt_FMT " on subvertex %" PetscInt_FMT ", vertex %" PetscInt_FMT " should be %" PetscInt_FMT, sdof, subvertex, vertex, dof);
3551       for (d = 0; d < dof; ++d) subCoords[soff + d] = coords[off + d];
3552     }
3553     PetscCall(VecRestoreArray(coordinates, &coords));
3554     PetscCall(VecRestoreArray(subCoordinates, &subCoords));
3555     PetscCall(DMSetCoordinatesLocal(subdm, subCoordinates));
3556     PetscCall(VecDestroy(&subCoordinates));
3557   }
3558   /* Build SF: We need this complexity because subpoints might not be selected on the owning process */
3559   {
3560     PetscSF            sfPoint, sfPointSub;
3561     IS                 subpIS;
3562     const PetscSFNode *remotePoints;
3563     PetscSFNode       *sremotePoints = NULL, *newLocalPoints = NULL, *newOwners = NULL;
3564     const PetscInt    *localPoints, *subpoints, *rootdegree;
3565     PetscInt          *slocalPoints = NULL, *sortedPoints = NULL, *sortedIndices = NULL;
3566     PetscInt           numRoots, numLeaves, numSubpoints = 0, numSubroots, numSubleaves = 0, l, sl = 0, ll = 0, pStart, pEnd, p;
3567     PetscMPIInt        rank, size;
3568 
3569     PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
3570     PetscCallMPI(MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size));
3571     PetscCall(DMGetPointSF(dm, &sfPoint));
3572     PetscCall(DMGetPointSF(subdm, &sfPointSub));
3573     PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
3574     PetscCall(DMPlexGetChart(subdm, NULL, &numSubroots));
3575     PetscCall(DMPlexGetSubpointIS(subdm, &subpIS));
3576     if (subpIS) {
3577       PetscBool sorted = PETSC_TRUE;
3578 
3579       PetscCall(ISGetIndices(subpIS, &subpoints));
3580       PetscCall(ISGetLocalSize(subpIS, &numSubpoints));
3581       for (p = 1; p < numSubpoints; ++p) sorted = sorted && (subpoints[p] >= subpoints[p - 1]) ? PETSC_TRUE : PETSC_FALSE;
3582       if (!sorted) {
3583         PetscCall(PetscMalloc2(numSubpoints, &sortedPoints, numSubpoints, &sortedIndices));
3584         for (p = 0; p < numSubpoints; ++p) sortedIndices[p] = p;
3585         PetscCall(PetscArraycpy(sortedPoints, subpoints, numSubpoints));
3586         PetscCall(PetscSortIntWithArray(numSubpoints, sortedPoints, sortedIndices));
3587       }
3588     }
3589     PetscCall(PetscSFGetGraph(sfPoint, &numRoots, &numLeaves, &localPoints, &remotePoints));
3590     if (numRoots >= 0) {
3591       PetscCall(PetscSFComputeDegreeBegin(sfPoint, &rootdegree));
3592       PetscCall(PetscSFComputeDegreeEnd(sfPoint, &rootdegree));
3593       PetscCall(PetscMalloc2(pEnd - pStart, &newLocalPoints, numRoots, &newOwners));
3594       for (p = 0; p < pEnd - pStart; ++p) {
3595         newLocalPoints[p].rank  = -2;
3596         newLocalPoints[p].index = -2;
3597       }
3598       /* Set subleaves */
3599       for (l = 0; l < numLeaves; ++l) {
3600         const PetscInt point    = localPoints[l];
3601         const PetscInt subpoint = DMPlexFilterPointPerm_Internal(point, 0, numSubpoints, sortedPoints ? sortedPoints : subpoints, sortedIndices);
3602 
3603         if (subpoint < 0) continue;
3604         newLocalPoints[point - pStart].rank  = rank;
3605         newLocalPoints[point - pStart].index = subpoint;
3606         ++numSubleaves;
3607       }
3608       /* Must put in owned subpoints */
3609       for (p = pStart; p < pEnd; ++p) {
3610         newOwners[p - pStart].rank  = -3;
3611         newOwners[p - pStart].index = -3;
3612       }
3613       for (p = 0; p < numSubpoints; ++p) {
3614         /* Hold on to currently owned points */
3615         if (rootdegree[subpoints[p] - pStart]) newOwners[subpoints[p] - pStart].rank = rank + size;
3616         else newOwners[subpoints[p] - pStart].rank = rank;
3617         newOwners[subpoints[p] - pStart].index = p;
3618       }
3619       PetscCall(PetscSFReduceBegin(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC));
3620       PetscCall(PetscSFReduceEnd(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC));
3621       for (p = pStart; p < pEnd; ++p)
3622         if (newOwners[p - pStart].rank >= size) newOwners[p - pStart].rank -= size;
3623       PetscCall(PetscSFBcastBegin(sfPoint, MPIU_2INT, newOwners, newLocalPoints, MPI_REPLACE));
3624       PetscCall(PetscSFBcastEnd(sfPoint, MPIU_2INT, newOwners, newLocalPoints, MPI_REPLACE));
3625       PetscCall(PetscMalloc1(numSubleaves, &slocalPoints));
3626       PetscCall(PetscMalloc1(numSubleaves, &sremotePoints));
3627       for (l = 0; l < numLeaves; ++l) {
3628         const PetscInt point    = localPoints[l];
3629         const PetscInt subpoint = DMPlexFilterPointPerm_Internal(point, 0, numSubpoints, sortedPoints ? sortedPoints : subpoints, sortedIndices);
3630 
3631         if (subpoint < 0) continue;
3632         if (newLocalPoints[point].rank == rank) {
3633           ++ll;
3634           continue;
3635         }
3636         slocalPoints[sl]        = subpoint;
3637         sremotePoints[sl].rank  = newLocalPoints[point].rank;
3638         sremotePoints[sl].index = newLocalPoints[point].index;
3639         PetscCheck(sremotePoints[sl].rank >= 0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote rank for local point %" PetscInt_FMT, point);
3640         PetscCheck(sremotePoints[sl].index >= 0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote subpoint for local point %" PetscInt_FMT, point);
3641         ++sl;
3642       }
3643       PetscCheck(sl + ll == numSubleaves, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatch in number of subleaves %" PetscInt_FMT " + %" PetscInt_FMT " != %" PetscInt_FMT, sl, ll, numSubleaves);
3644       PetscCall(PetscFree2(newLocalPoints, newOwners));
3645       PetscCall(PetscSFSetGraph(sfPointSub, numSubroots, sl, slocalPoints, PETSC_OWN_POINTER, sremotePoints, PETSC_OWN_POINTER));
3646     }
3647     if (subpIS) PetscCall(ISRestoreIndices(subpIS, &subpoints));
3648     PetscCall(PetscFree2(sortedPoints, sortedIndices));
3649   }
3650   /* Filter labels */
3651   PetscCall(DMPlexFilterLabels_Internal(dm, numSubPoints, subpoints, firstSubPoint, subdm));
3652   /* Cleanup */
3653   for (d = 0; d <= sdim; ++d) {
3654     if (subpointIS[d]) PetscCall(ISRestoreIndices(subpointIS[d], &subpoints[d]));
3655     PetscCall(ISDestroy(&subpointIS[d]));
3656   }
3657   PetscCall(PetscFree4(numSubPoints, firstSubPoint, subpointIS, subpoints));
3658   PetscFunctionReturn(PETSC_SUCCESS);
3659 }
3660 
3661 static PetscErrorCode DMPlexCreateSubmesh_Interpolated(DM dm, DMLabel vertexLabel, PetscInt value, PetscBool markedFaces, DM subdm)
3662 {
3663   PetscFunctionBegin;
3664   PetscCall(DMPlexCreateSubmeshGeneric_Interpolated(dm, vertexLabel, value, markedFaces, PETSC_FALSE, 1, subdm));
3665   PetscFunctionReturn(PETSC_SUCCESS);
3666 }
3667 
3668 /*@
3669   DMPlexCreateSubmesh - Extract a hypersurface from the mesh using vertices defined by a label
3670 
3671   Input Parameters:
3672 + dm          - The original mesh
3673 . vertexLabel - The `DMLabel` marking points contained in the surface
3674 . value       - The label value to use
3675 - markedFaces - `PETSC_TRUE` if surface faces are marked in addition to vertices, `PETSC_FALSE` if only vertices are marked
3676 
3677   Output Parameter:
3678 . subdm - The surface mesh
3679 
3680   Level: developer
3681 
3682   Note:
3683   This function produces a `DMLabel` mapping original points in the submesh to their depth. This can be obtained using `DMPlexGetSubpointMap()`.
3684 
3685 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetSubpointMap()`, `DMGetLabel()`, `DMLabelSetValue()`
3686 @*/
3687 PetscErrorCode DMPlexCreateSubmesh(DM dm, DMLabel vertexLabel, PetscInt value, PetscBool markedFaces, DM *subdm)
3688 {
3689   DMPlexInterpolatedFlag interpolated;
3690   PetscInt               dim, cdim;
3691 
3692   PetscFunctionBegin;
3693   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3694   PetscAssertPointer(subdm, 5);
3695   PetscCall(DMGetDimension(dm, &dim));
3696   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), subdm));
3697   PetscCall(DMSetType(*subdm, DMPLEX));
3698   PetscCall(DMSetDimension(*subdm, dim - 1));
3699   PetscCall(DMGetCoordinateDim(dm, &cdim));
3700   PetscCall(DMSetCoordinateDim(*subdm, cdim));
3701   PetscCall(DMPlexIsInterpolated(dm, &interpolated));
3702   PetscCheck(interpolated != DMPLEX_INTERPOLATED_PARTIAL, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Not for partially interpolated meshes");
3703   if (interpolated) {
3704     PetscCall(DMPlexCreateSubmesh_Interpolated(dm, vertexLabel, value, markedFaces, *subdm));
3705   } else {
3706     PetscCall(DMPlexCreateSubmesh_Uninterpolated(dm, vertexLabel, value, *subdm));
3707   }
3708   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_TRUE, *subdm));
3709   PetscFunctionReturn(PETSC_SUCCESS);
3710 }
3711 
3712 static PetscErrorCode DMPlexCreateCohesiveSubmesh_Uninterpolated(DM dm, PetscBool hasLagrange, const char label[], PetscInt value, DM subdm)
3713 {
3714   MPI_Comm        comm;
3715   DMLabel         subpointMap;
3716   IS              subvertexIS;
3717   const PetscInt *subVertices;
3718   PetscInt        numSubVertices, firstSubVertex, numSubCells, *subCells = NULL;
3719   PetscInt       *subface, maxConeSize, numSubFaces, firstSubFace, newFacePoint, nFV;
3720   PetscInt        c, f;
3721 
3722   PetscFunctionBegin;
3723   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
3724   /* Create subpointMap which marks the submesh */
3725   PetscCall(DMLabelCreate(PETSC_COMM_SELF, "subpoint_map", &subpointMap));
3726   PetscCall(DMPlexSetSubpointMap(subdm, subpointMap));
3727   PetscCall(DMPlexMarkCohesiveSubmesh_Uninterpolated(dm, hasLagrange, label, value, subpointMap, &numSubFaces, &nFV, &subCells, subdm));
3728   /* Setup chart */
3729   PetscCall(DMLabelGetStratumSize(subpointMap, 0, &numSubVertices));
3730   PetscCall(DMLabelGetStratumSize(subpointMap, 2, &numSubCells));
3731   PetscCall(DMPlexSetChart(subdm, 0, numSubCells + numSubFaces + numSubVertices));
3732   PetscCall(DMPlexSetVTKCellHeight(subdm, 1));
3733   /* Set cone sizes */
3734   firstSubVertex = numSubCells;
3735   firstSubFace   = numSubCells + numSubVertices;
3736   newFacePoint   = firstSubFace;
3737   PetscCall(DMLabelGetStratumIS(subpointMap, 0, &subvertexIS));
3738   if (subvertexIS) PetscCall(ISGetIndices(subvertexIS, &subVertices));
3739   for (c = 0; c < numSubCells; ++c) PetscCall(DMPlexSetConeSize(subdm, c, 1));
3740   for (f = firstSubFace; f < firstSubFace + numSubFaces; ++f) PetscCall(DMPlexSetConeSize(subdm, f, nFV));
3741   PetscCall(DMSetUp(subdm));
3742   PetscCall(DMLabelDestroy(&subpointMap));
3743   /* Create face cones */
3744   PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, NULL));
3745   PetscCall(DMGetWorkArray(subdm, maxConeSize, MPIU_INT, (void **)&subface));
3746   for (c = 0; c < numSubCells; ++c) {
3747     const PetscInt  cell    = subCells[c];
3748     const PetscInt  subcell = c;
3749     const PetscInt *cone, *cells;
3750     PetscBool       isHybrid = PETSC_FALSE;
3751     PetscInt        numCells, subVertex, p, v;
3752 
3753     PetscCall(DMPlexCellIsHybrid_Internal(dm, cell, &isHybrid));
3754     if (!isHybrid) continue;
3755     PetscCall(DMPlexGetCone(dm, cell, &cone));
3756     for (v = 0; v < nFV; ++v) {
3757       PetscCall(PetscFindInt(cone[v], numSubVertices, subVertices, &subVertex));
3758       subface[v] = firstSubVertex + subVertex;
3759     }
3760     PetscCall(DMPlexSetCone(subdm, newFacePoint, subface));
3761     PetscCall(DMPlexSetCone(subdm, subcell, &newFacePoint));
3762     PetscCall(DMPlexGetJoin(dm, nFV, cone, &numCells, &cells));
3763     /* Not true in parallel
3764     PetscCheck(numCells == 2,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive cells should separate two cells"); */
3765     for (p = 0; p < numCells; ++p) {
3766       PetscInt  negsubcell;
3767       PetscBool isHybrid = PETSC_FALSE;
3768 
3769       PetscCall(DMPlexCellIsHybrid_Internal(dm, cells[p], &isHybrid));
3770       if (isHybrid) continue;
3771       /* I know this is a crap search */
3772       for (negsubcell = 0; negsubcell < numSubCells; ++negsubcell) {
3773         if (subCells[negsubcell] == cells[p]) break;
3774       }
3775       PetscCheck(negsubcell != numSubCells, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not find negative face neighbor for cohesive cell %" PetscInt_FMT, cell);
3776       PetscCall(DMPlexSetCone(subdm, negsubcell, &newFacePoint));
3777     }
3778     PetscCall(DMPlexRestoreJoin(dm, nFV, cone, &numCells, &cells));
3779     ++newFacePoint;
3780   }
3781   PetscCall(DMRestoreWorkArray(subdm, maxConeSize, MPIU_INT, (void **)&subface));
3782   PetscCall(DMPlexSymmetrize(subdm));
3783   PetscCall(DMPlexStratify(subdm));
3784   /* Build coordinates */
3785   {
3786     PetscSection coordSection, subCoordSection;
3787     Vec          coordinates, subCoordinates;
3788     PetscScalar *coords, *subCoords;
3789     PetscInt     cdim, numComp, coordSize, v;
3790     const char  *name;
3791 
3792     PetscCall(DMGetCoordinateDim(dm, &cdim));
3793     PetscCall(DMGetCoordinateSection(dm, &coordSection));
3794     PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
3795     PetscCall(DMGetCoordinateSection(subdm, &subCoordSection));
3796     PetscCall(PetscSectionSetNumFields(subCoordSection, 1));
3797     PetscCall(PetscSectionGetFieldComponents(coordSection, 0, &numComp));
3798     PetscCall(PetscSectionSetFieldComponents(subCoordSection, 0, numComp));
3799     PetscCall(PetscSectionSetChart(subCoordSection, firstSubVertex, firstSubVertex + numSubVertices));
3800     for (v = 0; v < numSubVertices; ++v) {
3801       const PetscInt vertex    = subVertices[v];
3802       const PetscInt subvertex = firstSubVertex + v;
3803       PetscInt       dof;
3804 
3805       PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3806       PetscCall(PetscSectionSetDof(subCoordSection, subvertex, dof));
3807       PetscCall(PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof));
3808     }
3809     PetscCall(PetscSectionSetUp(subCoordSection));
3810     PetscCall(PetscSectionGetStorageSize(subCoordSection, &coordSize));
3811     PetscCall(VecCreate(PETSC_COMM_SELF, &subCoordinates));
3812     PetscCall(PetscObjectGetName((PetscObject)coordinates, &name));
3813     PetscCall(PetscObjectSetName((PetscObject)subCoordinates, name));
3814     PetscCall(VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE));
3815     PetscCall(VecSetBlockSize(subCoordinates, cdim));
3816     PetscCall(VecSetType(subCoordinates, VECSTANDARD));
3817     PetscCall(VecGetArray(coordinates, &coords));
3818     PetscCall(VecGetArray(subCoordinates, &subCoords));
3819     for (v = 0; v < numSubVertices; ++v) {
3820       const PetscInt vertex    = subVertices[v];
3821       const PetscInt subvertex = firstSubVertex + v;
3822       PetscInt       dof, off, sdof, soff, d;
3823 
3824       PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3825       PetscCall(PetscSectionGetOffset(coordSection, vertex, &off));
3826       PetscCall(PetscSectionGetDof(subCoordSection, subvertex, &sdof));
3827       PetscCall(PetscSectionGetOffset(subCoordSection, subvertex, &soff));
3828       PetscCheck(dof == sdof, comm, PETSC_ERR_PLIB, "Coordinate dimension %" PetscInt_FMT " on subvertex %" PetscInt_FMT ", vertex %" PetscInt_FMT " should be %" PetscInt_FMT, sdof, subvertex, vertex, dof);
3829       for (d = 0; d < dof; ++d) subCoords[soff + d] = coords[off + d];
3830     }
3831     PetscCall(VecRestoreArray(coordinates, &coords));
3832     PetscCall(VecRestoreArray(subCoordinates, &subCoords));
3833     PetscCall(DMSetCoordinatesLocal(subdm, subCoordinates));
3834     PetscCall(VecDestroy(&subCoordinates));
3835   }
3836   /* Build SF */
3837   CHKMEMQ;
3838   {
3839     PetscSF            sfPoint, sfPointSub;
3840     const PetscSFNode *remotePoints;
3841     PetscSFNode       *sremotePoints, *newLocalPoints, *newOwners;
3842     const PetscInt    *localPoints;
3843     PetscInt          *slocalPoints;
3844     PetscInt           numRoots, numLeaves, numSubRoots = numSubCells + numSubFaces + numSubVertices, numSubLeaves = 0, l, sl, ll, pStart, pEnd, p, vStart, vEnd;
3845     PetscMPIInt        rank;
3846 
3847     PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
3848     PetscCall(DMGetPointSF(dm, &sfPoint));
3849     PetscCall(DMGetPointSF(subdm, &sfPointSub));
3850     PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
3851     PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
3852     PetscCall(PetscSFGetGraph(sfPoint, &numRoots, &numLeaves, &localPoints, &remotePoints));
3853     if (numRoots >= 0) {
3854       /* Only vertices should be shared */
3855       PetscCall(PetscMalloc2(pEnd - pStart, &newLocalPoints, numRoots, &newOwners));
3856       for (p = 0; p < pEnd - pStart; ++p) {
3857         newLocalPoints[p].rank  = -2;
3858         newLocalPoints[p].index = -2;
3859       }
3860       /* Set subleaves */
3861       for (l = 0; l < numLeaves; ++l) {
3862         const PetscInt point    = localPoints[l];
3863         const PetscInt subPoint = DMPlexFilterPoint_Internal(point, firstSubVertex, numSubVertices, subVertices);
3864 
3865         PetscCheck(!(point < vStart) || !(point >= vEnd), PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Should not be mapping anything but vertices, %" PetscInt_FMT, point);
3866         if (subPoint < 0) continue;
3867         newLocalPoints[point - pStart].rank  = rank;
3868         newLocalPoints[point - pStart].index = subPoint;
3869         ++numSubLeaves;
3870       }
3871       /* Must put in owned subpoints */
3872       for (p = pStart; p < pEnd; ++p) {
3873         const PetscInt subPoint = DMPlexFilterPoint_Internal(p, firstSubVertex, numSubVertices, subVertices);
3874 
3875         if (subPoint < 0) {
3876           newOwners[p - pStart].rank  = -3;
3877           newOwners[p - pStart].index = -3;
3878         } else {
3879           newOwners[p - pStart].rank  = rank;
3880           newOwners[p - pStart].index = subPoint;
3881         }
3882       }
3883       PetscCall(PetscSFReduceBegin(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC));
3884       PetscCall(PetscSFReduceEnd(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC));
3885       PetscCall(PetscSFBcastBegin(sfPoint, MPIU_2INT, newOwners, newLocalPoints, MPI_REPLACE));
3886       PetscCall(PetscSFBcastEnd(sfPoint, MPIU_2INT, newOwners, newLocalPoints, MPI_REPLACE));
3887       PetscCall(PetscMalloc1(numSubLeaves, &slocalPoints));
3888       PetscCall(PetscMalloc1(numSubLeaves, &sremotePoints));
3889       for (l = 0, sl = 0, ll = 0; l < numLeaves; ++l) {
3890         const PetscInt point    = localPoints[l];
3891         const PetscInt subPoint = DMPlexFilterPoint_Internal(point, firstSubVertex, numSubVertices, subVertices);
3892 
3893         if (subPoint < 0) continue;
3894         if (newLocalPoints[point].rank == rank) {
3895           ++ll;
3896           continue;
3897         }
3898         slocalPoints[sl]        = subPoint;
3899         sremotePoints[sl].rank  = newLocalPoints[point].rank;
3900         sremotePoints[sl].index = newLocalPoints[point].index;
3901         PetscCheck(sremotePoints[sl].rank >= 0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote rank for local point %" PetscInt_FMT, point);
3902         PetscCheck(sremotePoints[sl].index >= 0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote subpoint for local point %" PetscInt_FMT, point);
3903         ++sl;
3904       }
3905       PetscCall(PetscFree2(newLocalPoints, newOwners));
3906       PetscCheck(sl + ll == numSubLeaves, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatch in number of subleaves %" PetscInt_FMT " + %" PetscInt_FMT " != %" PetscInt_FMT, sl, ll, numSubLeaves);
3907       PetscCall(PetscSFSetGraph(sfPointSub, numSubRoots, sl, slocalPoints, PETSC_OWN_POINTER, sremotePoints, PETSC_OWN_POINTER));
3908     }
3909   }
3910   CHKMEMQ;
3911   /* Cleanup */
3912   if (subvertexIS) PetscCall(ISRestoreIndices(subvertexIS, &subVertices));
3913   PetscCall(ISDestroy(&subvertexIS));
3914   PetscCall(PetscFree(subCells));
3915   PetscFunctionReturn(PETSC_SUCCESS);
3916 }
3917 
3918 static PetscErrorCode DMPlexCreateCohesiveSubmesh_Interpolated(DM dm, const char labelname[], PetscInt value, DM subdm)
3919 {
3920   DMLabel label = NULL;
3921 
3922   PetscFunctionBegin;
3923   if (labelname) PetscCall(DMGetLabel(dm, labelname, &label));
3924   PetscCall(DMPlexCreateSubmeshGeneric_Interpolated(dm, label, value, PETSC_FALSE, PETSC_TRUE, 1, subdm));
3925   PetscFunctionReturn(PETSC_SUCCESS);
3926 }
3927 
3928 /*@C
3929   DMPlexCreateCohesiveSubmesh - Extract from a mesh with cohesive cells the hypersurface defined by one face of the cells. Optionally, a label can be given to restrict the cells.
3930 
3931   Input Parameters:
3932 + dm          - The original mesh
3933 . hasLagrange - The mesh has Lagrange unknowns in the cohesive cells
3934 . label       - A label name, or `NULL`
3935 - value       - A label value
3936 
3937   Output Parameter:
3938 . subdm - The surface mesh
3939 
3940   Level: developer
3941 
3942   Note:
3943   This function produces a `DMLabel` mapping original points in the submesh to their depth. This can be obtained using `DMPlexGetSubpointMap()`.
3944 
3945 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetSubpointMap()`, `DMPlexCreateSubmesh()`
3946 @*/
3947 PetscErrorCode DMPlexCreateCohesiveSubmesh(DM dm, PetscBool hasLagrange, const char label[], PetscInt value, DM *subdm)
3948 {
3949   PetscInt dim, cdim, depth;
3950 
3951   PetscFunctionBegin;
3952   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3953   PetscAssertPointer(subdm, 5);
3954   PetscCall(DMGetDimension(dm, &dim));
3955   PetscCall(DMPlexGetDepth(dm, &depth));
3956   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), subdm));
3957   PetscCall(DMSetType(*subdm, DMPLEX));
3958   PetscCall(DMSetDimension(*subdm, dim - 1));
3959   PetscCall(DMGetCoordinateDim(dm, &cdim));
3960   PetscCall(DMSetCoordinateDim(*subdm, cdim));
3961   if (depth == dim) {
3962     PetscCall(DMPlexCreateCohesiveSubmesh_Interpolated(dm, label, value, *subdm));
3963   } else {
3964     PetscCall(DMPlexCreateCohesiveSubmesh_Uninterpolated(dm, hasLagrange, label, value, *subdm));
3965   }
3966   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_TRUE, *subdm));
3967   PetscFunctionReturn(PETSC_SUCCESS);
3968 }
3969 
3970 /*@
3971   DMPlexReorderCohesiveSupports - Ensure that face supports for cohesive end caps are ordered
3972 
3973   Not Collective
3974 
3975   Input Parameter:
3976 . dm - The `DM` containing cohesive cells
3977 
3978   Level: developer
3979 
3980   Note:
3981   For the negative size (first) face, the cohesive cell should be first in the support, and for
3982   the positive side (second) face, the cohesive cell should be second in the support.
3983 
3984 .seealso: `DMPlexConstructCohesiveCells()`, `DMPlexCreateCohesiveSubmesh()`
3985 @*/
3986 PetscErrorCode DMPlexReorderCohesiveSupports(DM dm)
3987 {
3988   PetscInt dim, cStart, cEnd;
3989 
3990   PetscFunctionBegin;
3991   PetscCall(DMGetDimension(dm, &dim));
3992   PetscCall(DMPlexGetTensorPrismBounds_Internal(dm, dim, &cStart, &cEnd));
3993   for (PetscInt c = cStart; c < cEnd; ++c) {
3994     const PetscInt *cone;
3995     PetscInt        coneSize;
3996 
3997     PetscCall(DMPlexGetConeSize(dm, c, &coneSize));
3998     PetscCall(DMPlexGetCone(dm, c, &cone));
3999     PetscCheck(coneSize >= 2, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Hybrid cell %" PetscInt_FMT " cone size %" PetscInt_FMT " must be >= 2", c, coneSize);
4000     for (PetscInt s = 0; s < 2; ++s) {
4001       const PetscInt *supp;
4002       PetscInt        suppSize, newsupp[2];
4003 
4004       PetscCall(DMPlexGetSupportSize(dm, cone[s], &suppSize));
4005       PetscCall(DMPlexGetSupport(dm, cone[s], &supp));
4006       if (suppSize == 2) {
4007         /* Reorder hybrid end cap support */
4008         if (supp[s] == c) {
4009           newsupp[0] = supp[1];
4010           newsupp[1] = supp[0];
4011         } else {
4012           newsupp[0] = supp[0];
4013           newsupp[1] = supp[1];
4014         }
4015         PetscCheck(newsupp[1 - s] == c, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Hybrid end cap %" PetscInt_FMT " support entry %" PetscInt_FMT " != %" PetscInt_FMT " hybrid cell", cone[s], newsupp[1 - s], c);
4016         PetscCall(DMPlexSetSupport(dm, cone[s], newsupp));
4017       }
4018     }
4019   }
4020   PetscFunctionReturn(PETSC_SUCCESS);
4021 }
4022 
4023 /*@
4024   DMPlexFilter - Extract a subset of mesh cells defined by a label as a separate mesh
4025 
4026   Input Parameters:
4027 + dm        - The original mesh
4028 . cellLabel - The `DMLabel` marking cells contained in the new mesh
4029 - value     - The label value to use
4030 
4031   Output Parameter:
4032 . subdm - The new mesh
4033 
4034   Level: developer
4035 
4036   Note:
4037   This function produces a `DMLabel` mapping original points in the submesh to their depth. This can be obtained using `DMPlexGetSubpointMap()`.
4038 
4039 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetSubpointMap()`, `DMGetLabel()`, `DMLabelSetValue()`, `DMPlexCreateSubmesh()`
4040 @*/
4041 PetscErrorCode DMPlexFilter(DM dm, DMLabel cellLabel, PetscInt value, DM *subdm)
4042 {
4043   PetscInt dim, overlap;
4044 
4045   PetscFunctionBegin;
4046   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4047   PetscAssertPointer(subdm, 4);
4048   PetscCall(DMGetDimension(dm, &dim));
4049   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), subdm));
4050   PetscCall(DMSetType(*subdm, DMPLEX));
4051   /* Extract submesh in place, could be empty on some procs, could have inconsistency if procs do not both extract a shared cell */
4052   PetscCall(DMPlexCreateSubmeshGeneric_Interpolated(dm, cellLabel, value, PETSC_FALSE, PETSC_FALSE, 0, *subdm));
4053   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_TRUE, *subdm));
4054   // It is possible to obtain a surface mesh where some faces are in SF
4055   //   We should either mark the mesh as having an overlap, or delete these from the SF
4056   PetscCall(DMPlexGetOverlap(dm, &overlap));
4057   if (!overlap) {
4058     PetscSF         sf;
4059     const PetscInt *leaves;
4060     PetscInt        cStart, cEnd, Nl;
4061     PetscBool       hasSubcell = PETSC_FALSE, ghasSubcell;
4062 
4063     PetscCall(DMPlexGetHeightStratum(*subdm, 0, &cStart, &cEnd));
4064     PetscCall(DMGetPointSF(*subdm, &sf));
4065     PetscCall(PetscSFGetGraph(sf, NULL, &Nl, &leaves, NULL));
4066     for (PetscInt l = 0; l < Nl; ++l) {
4067       const PetscInt point = leaves ? leaves[l] : l;
4068 
4069       if (point >= cStart && point < cEnd) {
4070         hasSubcell = PETSC_TRUE;
4071         break;
4072       }
4073     }
4074     PetscCall(MPIU_Allreduce(&hasSubcell, &ghasSubcell, 1, MPIU_BOOL, MPI_LOR, PetscObjectComm((PetscObject)dm)));
4075     if (ghasSubcell) PetscCall(DMPlexSetOverlap(*subdm, NULL, 1));
4076   }
4077   PetscFunctionReturn(PETSC_SUCCESS);
4078 }
4079 
4080 /*@
4081   DMPlexGetSubpointMap - Returns a `DMLabel` with point dimension as values
4082 
4083   Input Parameter:
4084 . dm - The submesh `DM`
4085 
4086   Output Parameter:
4087 . subpointMap - The `DMLabel` of all the points from the original mesh in this submesh, or `NULL` if this is not a submesh
4088 
4089   Level: developer
4090 
4091 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateSubmesh()`, `DMPlexGetSubpointIS()`
4092 @*/
4093 PetscErrorCode DMPlexGetSubpointMap(DM dm, DMLabel *subpointMap)
4094 {
4095   PetscFunctionBegin;
4096   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4097   PetscAssertPointer(subpointMap, 2);
4098   *subpointMap = ((DM_Plex *)dm->data)->subpointMap;
4099   PetscFunctionReturn(PETSC_SUCCESS);
4100 }
4101 
4102 /*@
4103   DMPlexSetSubpointMap - Sets the `DMLabel` with point dimension as values
4104 
4105   Input Parameters:
4106 + dm          - The submesh `DM`
4107 - subpointMap - The `DMLabel` of all the points from the original mesh in this submesh
4108 
4109   Level: developer
4110 
4111   Note:
4112   Should normally not be called by the user, since it is set in `DMPlexCreateSubmesh()`
4113 
4114 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateSubmesh()`, `DMPlexGetSubpointIS()`
4115 @*/
4116 PetscErrorCode DMPlexSetSubpointMap(DM dm, DMLabel subpointMap)
4117 {
4118   DM_Plex *mesh = (DM_Plex *)dm->data;
4119   DMLabel  tmp;
4120 
4121   PetscFunctionBegin;
4122   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4123   tmp               = mesh->subpointMap;
4124   mesh->subpointMap = subpointMap;
4125   PetscCall(PetscObjectReference((PetscObject)mesh->subpointMap));
4126   PetscCall(DMLabelDestroy(&tmp));
4127   PetscFunctionReturn(PETSC_SUCCESS);
4128 }
4129 
4130 static PetscErrorCode DMPlexCreateSubpointIS_Internal(DM dm, IS *subpointIS)
4131 {
4132   DMLabel  spmap;
4133   PetscInt depth, d;
4134 
4135   PetscFunctionBegin;
4136   PetscCall(DMPlexGetSubpointMap(dm, &spmap));
4137   PetscCall(DMPlexGetDepth(dm, &depth));
4138   if (spmap && depth >= 0) {
4139     DM_Plex  *mesh = (DM_Plex *)dm->data;
4140     PetscInt *points, *depths;
4141     PetscInt  pStart, pEnd, p, off;
4142 
4143     PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
4144     PetscCheck(!pStart, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Submeshes must start the point numbering at 0, not %" PetscInt_FMT, pStart);
4145     PetscCall(PetscMalloc1(pEnd, &points));
4146     PetscCall(DMGetWorkArray(dm, depth + 1, MPIU_INT, &depths));
4147     depths[0] = depth;
4148     depths[1] = 0;
4149     for (d = 2; d <= depth; ++d) depths[d] = depth + 1 - d;
4150     for (d = 0, off = 0; d <= depth; ++d) {
4151       const PetscInt dep = depths[d];
4152       PetscInt       depStart, depEnd, n;
4153 
4154       PetscCall(DMPlexGetDepthStratum(dm, dep, &depStart, &depEnd));
4155       PetscCall(DMLabelGetStratumSize(spmap, dep, &n));
4156       if (((d < 2) && (depth > 1)) || (d == 1)) { /* Only check vertices and cells for now since the map is broken for others */
4157         PetscCheck(n == depEnd - depStart, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "The number of mapped submesh points %" PetscInt_FMT " at depth %" PetscInt_FMT " should be %" PetscInt_FMT, n, dep, depEnd - depStart);
4158       } else {
4159         if (!n) {
4160           if (d == 0) {
4161             /* Missing cells */
4162             for (p = 0; p < depEnd - depStart; ++p, ++off) points[off] = -1;
4163           } else {
4164             /* Missing faces */
4165             for (p = 0; p < depEnd - depStart; ++p, ++off) points[off] = PETSC_MAX_INT;
4166           }
4167         }
4168       }
4169       if (n) {
4170         IS              is;
4171         const PetscInt *opoints;
4172 
4173         PetscCall(DMLabelGetStratumIS(spmap, dep, &is));
4174         PetscCall(ISGetIndices(is, &opoints));
4175         for (p = 0; p < n; ++p, ++off) points[off] = opoints[p];
4176         PetscCall(ISRestoreIndices(is, &opoints));
4177         PetscCall(ISDestroy(&is));
4178       }
4179     }
4180     PetscCall(DMRestoreWorkArray(dm, depth + 1, MPIU_INT, &depths));
4181     PetscCheck(off == pEnd, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "The number of mapped submesh points %" PetscInt_FMT " should be %" PetscInt_FMT, off, pEnd);
4182     PetscCall(ISCreateGeneral(PETSC_COMM_SELF, pEnd, points, PETSC_OWN_POINTER, subpointIS));
4183     PetscCall(PetscObjectStateGet((PetscObject)spmap, &mesh->subpointState));
4184   }
4185   PetscFunctionReturn(PETSC_SUCCESS);
4186 }
4187 
4188 /*@
4189   DMPlexGetSubpointIS - Returns an `IS` covering the entire subdm chart with the original points as data
4190 
4191   Input Parameter:
4192 . dm - The submesh `DM`
4193 
4194   Output Parameter:
4195 . subpointIS - The `IS` of all the points from the original mesh in this submesh, or `NULL` if this is not a submesh
4196 
4197   Level: developer
4198 
4199   Note:
4200   This `IS` is guaranteed to be sorted by the construction of the submesh
4201 
4202 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateSubmesh()`, `DMPlexGetSubpointMap()`
4203 @*/
4204 PetscErrorCode DMPlexGetSubpointIS(DM dm, IS *subpointIS)
4205 {
4206   DM_Plex         *mesh = (DM_Plex *)dm->data;
4207   DMLabel          spmap;
4208   PetscObjectState state;
4209 
4210   PetscFunctionBegin;
4211   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4212   PetscAssertPointer(subpointIS, 2);
4213   PetscCall(DMPlexGetSubpointMap(dm, &spmap));
4214   PetscCall(PetscObjectStateGet((PetscObject)spmap, &state));
4215   if (state != mesh->subpointState || !mesh->subpointIS) PetscCall(DMPlexCreateSubpointIS_Internal(dm, &mesh->subpointIS));
4216   *subpointIS = mesh->subpointIS;
4217   PetscFunctionReturn(PETSC_SUCCESS);
4218 }
4219 
4220 /*@
4221   DMGetEnclosureRelation - Get the relationship between `dmA` and `dmB`
4222 
4223   Input Parameters:
4224 + dmA - The first `DM`
4225 - dmB - The second `DM`
4226 
4227   Output Parameter:
4228 . rel - The relation of `dmA` to `dmB`
4229 
4230   Level: intermediate
4231 
4232 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMGetEnclosurePoint()`
4233 @*/
4234 PetscErrorCode DMGetEnclosureRelation(DM dmA, DM dmB, DMEnclosureType *rel)
4235 {
4236   DM       plexA, plexB, sdm;
4237   DMLabel  spmap;
4238   PetscInt pStartA, pEndA, pStartB, pEndB, NpA, NpB;
4239 
4240   PetscFunctionBegin;
4241   PetscAssertPointer(rel, 3);
4242   *rel = DM_ENC_NONE;
4243   if (!dmA || !dmB) PetscFunctionReturn(PETSC_SUCCESS);
4244   PetscValidHeaderSpecific(dmA, DM_CLASSID, 1);
4245   PetscValidHeaderSpecific(dmB, DM_CLASSID, 2);
4246   if (dmA == dmB) {
4247     *rel = DM_ENC_EQUALITY;
4248     PetscFunctionReturn(PETSC_SUCCESS);
4249   }
4250   PetscCall(DMConvert(dmA, DMPLEX, &plexA));
4251   PetscCall(DMConvert(dmB, DMPLEX, &plexB));
4252   PetscCall(DMPlexGetChart(plexA, &pStartA, &pEndA));
4253   PetscCall(DMPlexGetChart(plexB, &pStartB, &pEndB));
4254   /* Assumption 1: subDMs have smaller charts than the DMs that they originate from
4255     - The degenerate case of a subdomain which includes all of the domain on some process can be treated as equality */
4256   if ((pStartA == pStartB) && (pEndA == pEndB)) {
4257     *rel = DM_ENC_EQUALITY;
4258     goto end;
4259   }
4260   NpA = pEndA - pStartA;
4261   NpB = pEndB - pStartB;
4262   if (NpA == NpB) goto end;
4263   sdm = NpA > NpB ? plexB : plexA; /* The other is the original, enclosing dm */
4264   PetscCall(DMPlexGetSubpointMap(sdm, &spmap));
4265   if (!spmap) goto end;
4266   /* TODO Check the space mapped to by subpointMap is same size as dm */
4267   if (NpA > NpB) {
4268     *rel = DM_ENC_SUPERMESH;
4269   } else {
4270     *rel = DM_ENC_SUBMESH;
4271   }
4272 end:
4273   PetscCall(DMDestroy(&plexA));
4274   PetscCall(DMDestroy(&plexB));
4275   PetscFunctionReturn(PETSC_SUCCESS);
4276 }
4277 
4278 /*@
4279   DMGetEnclosurePoint - Get the point `pA` in `dmA` which corresponds to the point `pB` in `dmB`
4280 
4281   Input Parameters:
4282 + dmA   - The first `DM`
4283 . dmB   - The second `DM`
4284 . etype - The type of enclosure relation that `dmA` has to `dmB`
4285 - pB    - A point of `dmB`
4286 
4287   Output Parameter:
4288 . pA - The corresponding point of `dmA`
4289 
4290   Level: intermediate
4291 
4292 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMGetEnclosureRelation()`
4293 @*/
4294 PetscErrorCode DMGetEnclosurePoint(DM dmA, DM dmB, DMEnclosureType etype, PetscInt pB, PetscInt *pA)
4295 {
4296   DM              sdm;
4297   IS              subpointIS;
4298   const PetscInt *subpoints;
4299   PetscInt        numSubpoints;
4300 
4301   PetscFunctionBegin;
4302   /* TODO Cache the IS, making it look like an index */
4303   switch (etype) {
4304   case DM_ENC_SUPERMESH:
4305     sdm = dmB;
4306     PetscCall(DMPlexGetSubpointIS(sdm, &subpointIS));
4307     PetscCall(ISGetIndices(subpointIS, &subpoints));
4308     *pA = subpoints[pB];
4309     PetscCall(ISRestoreIndices(subpointIS, &subpoints));
4310     break;
4311   case DM_ENC_SUBMESH:
4312     sdm = dmA;
4313     PetscCall(DMPlexGetSubpointIS(sdm, &subpointIS));
4314     PetscCall(ISGetLocalSize(subpointIS, &numSubpoints));
4315     PetscCall(ISGetIndices(subpointIS, &subpoints));
4316     PetscCall(PetscFindInt(pB, numSubpoints, subpoints, pA));
4317     if (*pA < 0) {
4318       PetscCall(DMViewFromOptions(dmA, NULL, "-dm_enc_A_view"));
4319       PetscCall(DMViewFromOptions(dmB, NULL, "-dm_enc_B_view"));
4320       SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Point %" PetscInt_FMT " not found in submesh", pB);
4321     }
4322     PetscCall(ISRestoreIndices(subpointIS, &subpoints));
4323     break;
4324   case DM_ENC_EQUALITY:
4325   case DM_ENC_NONE:
4326     *pA = pB;
4327     break;
4328   case DM_ENC_UNKNOWN: {
4329     DMEnclosureType enc;
4330 
4331     PetscCall(DMGetEnclosureRelation(dmA, dmB, &enc));
4332     PetscCall(DMGetEnclosurePoint(dmA, dmB, enc, pB, pA));
4333   } break;
4334   default:
4335     SETERRQ(PetscObjectComm((PetscObject)dmA), PETSC_ERR_ARG_OUTOFRANGE, "Invalid enclosure type %d", (int)etype);
4336   }
4337   PetscFunctionReturn(PETSC_SUCCESS);
4338 }
4339