xref: /petsc/src/dm/impls/plex/plexsubmesh.c (revision 4ad8454beace47809662cdae21ee081016eaa39a)
1 #include <petsc/private/dmpleximpl.h>  /*I      "petscdmplex.h"    I*/
2 #include <petsc/private/dmlabelimpl.h> /*I      "petscdmlabel.h"   I*/
3 #include <petscsf.h>
4 
5 static PetscErrorCode DMPlexCellIsHybrid_Internal(DM dm, PetscInt p, PetscBool *isHybrid)
6 {
7   DMPolytopeType ct;
8 
9   PetscFunctionBegin;
10   PetscCall(DMPlexGetCellType(dm, p, &ct));
11   switch (ct) {
12   case DM_POLYTOPE_POINT_PRISM_TENSOR:
13   case DM_POLYTOPE_SEG_PRISM_TENSOR:
14   case DM_POLYTOPE_TRI_PRISM_TENSOR:
15   case DM_POLYTOPE_QUAD_PRISM_TENSOR:
16     *isHybrid = PETSC_TRUE;
17     break;
18   default:
19     *isHybrid = PETSC_FALSE;
20     break;
21   }
22   PetscFunctionReturn(PETSC_SUCCESS);
23 }
24 
25 static PetscErrorCode DMPlexGetTensorPrismBounds_Internal(DM dm, PetscInt dim, PetscInt *cStart, PetscInt *cEnd)
26 {
27   DMLabel ctLabel;
28 
29   PetscFunctionBegin;
30   if (cStart) *cStart = -1;
31   if (cEnd) *cEnd = -1;
32   PetscCall(DMPlexGetCellTypeLabel(dm, &ctLabel));
33   switch (dim) {
34   case 1:
35     PetscCall(DMLabelGetStratumBounds(ctLabel, DM_POLYTOPE_POINT_PRISM_TENSOR, cStart, cEnd));
36     break;
37   case 2:
38     PetscCall(DMLabelGetStratumBounds(ctLabel, DM_POLYTOPE_SEG_PRISM_TENSOR, cStart, cEnd));
39     break;
40   case 3:
41     PetscCall(DMLabelGetStratumBounds(ctLabel, DM_POLYTOPE_TRI_PRISM_TENSOR, cStart, cEnd));
42     if (*cStart < 0) PetscCall(DMLabelGetStratumBounds(ctLabel, DM_POLYTOPE_QUAD_PRISM_TENSOR, cStart, cEnd));
43     break;
44   default:
45     PetscFunctionReturn(PETSC_SUCCESS);
46   }
47   PetscFunctionReturn(PETSC_SUCCESS);
48 }
49 
50 PetscErrorCode DMPlexMarkBoundaryFaces_Internal(DM dm, PetscInt val, PetscInt cellHeight, DMLabel label, PetscBool missing_only)
51 {
52   PetscInt           depth, pStart, pEnd, fStart, fEnd, f, supportSize, nroots = -1, nleaves = -1, defval;
53   PetscSF            sf;
54   const PetscSFNode *iremote = NULL;
55   const PetscInt    *ilocal  = NULL;
56   PetscInt          *leafData;
57 
58   PetscFunctionBegin;
59   PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
60   PetscCall(DMPlexGetDepth(dm, &depth));
61   if (depth >= cellHeight + 1) {
62     PetscCall(DMPlexGetHeightStratum(dm, cellHeight + 1, &fStart, &fEnd));
63   } else {
64     /* Note DMPlexGetHeightStratum() returns fStart, fEnd = pStart, pEnd */
65     /* if height > depth, which is not what we want here.                */
66     fStart = 0;
67     fEnd   = 0;
68   }
69   PetscCall(DMLabelGetDefaultValue(label, &defval));
70   PetscCall(PetscCalloc1(pEnd - pStart, &leafData));
71   leafData = PetscSafePointerPlusOffset(leafData, -pStart);
72   PetscCall(DMGetPointSF(dm, &sf));
73   if (sf) PetscCall(PetscSFGetGraph(sf, &nroots, &nleaves, &ilocal, &iremote));
74   if (sf && nroots >= 0) {
75     PetscInt        cStart, cEnd, c, i;
76     PetscInt       *rootData, *rootData1, *cellOwners, hasTwoSupportCells = -2;
77     const PetscInt *support;
78     PetscMPIInt     rank;
79 
80     PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
81     PetscCall(DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd));
82     PetscCall(PetscCalloc3(pEnd - pStart, &rootData, pEnd - pStart, &rootData1, cEnd - cStart, &cellOwners));
83     rootData -= pStart;
84     rootData1 -= pStart;
85     for (c = cStart; c < cEnd; ++c) cellOwners[c - cStart] = (PetscInt)rank;
86     for (i = 0; i < nleaves; ++i) {
87       c = ilocal ? ilocal[i] : i;
88       if (c >= cStart && c < cEnd) cellOwners[c - cStart] = iremote[i].rank;
89     }
90     for (f = fStart; f < fEnd; ++f) {
91       PetscCall(DMPlexGetSupportSize(dm, f, &supportSize));
92       if (supportSize == 1) {
93         PetscCall(DMPlexGetSupport(dm, f, &support));
94         leafData[f] = cellOwners[support[0] - cStart];
95       } else {
96         /* TODO: When using DMForest, we could have a parent facet (a coarse facet)     */
97         /*       supportSize of which alone does not tell us if it is an interior       */
98         /*       facet or an exterior facet. Those facets can be identified by checking */
99         /*       if they are in the parent tree. We should probably skip those parent   */
100         /*       facets here, which will allow for including the following check, and   */
101         /*       see if they are exterior facets or not afterwards by checking if the   */
102         /*       children are exterior or not.                                          */
103         /* PetscCheck(supportSize == 2, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid facet support size (%" PetscInt_FMT ") on facet (%" PetscInt_FMT ")", supportSize, f); */
104         leafData[f] = hasTwoSupportCells; /* some negative PetscInt */
105       }
106       rootData[f]  = leafData[f];
107       rootData1[f] = leafData[f];
108     }
109     PetscCall(PetscSFReduceBegin(sf, MPIU_INT, leafData, rootData, MPI_MIN));
110     PetscCall(PetscSFReduceBegin(sf, MPIU_INT, leafData, rootData1, MPI_MAX));
111     PetscCall(PetscSFReduceEnd(sf, MPIU_INT, leafData, rootData, MPI_MIN));
112     PetscCall(PetscSFReduceEnd(sf, MPIU_INT, leafData, rootData1, MPI_MAX));
113     for (f = fStart; f < fEnd; ++f) {
114       /* Store global support size of f.                                                    */
115       /* Facet f is an interior facet if and only if one of the following two is satisfied: */
116       /* 1. supportSize is 2 on some rank.                                                  */
117       /* 2. supportSize is 1 on any rank that can see f, but f is on a partition boundary;  */
118       /*    i.e., rootData[f] < rootData1[f].                                               */
119       rootData[f] = (rootData[f] == hasTwoSupportCells || (rootData[f] < rootData1[f])) ? 2 : 1;
120       leafData[f] = rootData[f];
121     }
122     PetscCall(PetscSFBcastBegin(sf, MPIU_INT, rootData, leafData, MPI_REPLACE));
123     PetscCall(PetscSFBcastEnd(sf, MPIU_INT, rootData, leafData, MPI_REPLACE));
124     rootData += pStart;
125     rootData1 += pStart;
126     PetscCall(PetscFree3(rootData, rootData1, cellOwners));
127   } else {
128     for (f = fStart; f < fEnd; ++f) {
129       PetscCall(DMPlexGetSupportSize(dm, f, &supportSize));
130       leafData[f] = supportSize;
131     }
132   }
133   for (f = fStart; f < fEnd; ++f) {
134     if (leafData[f] == 1) {
135       if (val < 0) {
136         PetscInt *closure = NULL;
137         PetscInt  clSize, cl, cval;
138 
139         PetscAssert(!missing_only, PETSC_COMM_SELF, PETSC_ERR_SUP, "Not implemented");
140         PetscCall(DMPlexGetTransitiveClosure(dm, f, PETSC_TRUE, &clSize, &closure));
141         for (cl = 0; cl < clSize * 2; cl += 2) {
142           PetscCall(DMLabelGetValue(label, closure[cl], &cval));
143           if (cval < 0) continue;
144           PetscCall(DMLabelSetValue(label, f, cval));
145           break;
146         }
147         if (cl == clSize * 2) PetscCall(DMLabelSetValue(label, f, 1));
148         PetscCall(DMPlexRestoreTransitiveClosure(dm, f, PETSC_TRUE, &clSize, &closure));
149       } else {
150         if (missing_only) {
151           PetscInt fval;
152           PetscCall(DMLabelGetValue(label, f, &fval));
153           if (fval != defval) PetscCall(DMLabelClearValue(label, f, fval));
154           else PetscCall(DMLabelSetValue(label, f, val));
155         } else {
156           PetscCall(DMLabelSetValue(label, f, val));
157         }
158       }
159     } else {
160       /* TODO: See the above comment on DMForest */
161       /* PetscCheck(leafData[f] == 2, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid facet support size (%" PetscInt_FMT ") on facet (%" PetscInt_FMT ")", leafData[f], f); */
162     }
163   }
164   leafData = PetscSafePointerPlusOffset(leafData, pStart);
165   PetscCall(PetscFree(leafData));
166   PetscFunctionReturn(PETSC_SUCCESS);
167 }
168 
169 /*@
170   DMPlexMarkBoundaryFaces - Mark all faces on the boundary
171 
172   Collective
173 
174   Input Parameters:
175 + dm  - The original `DM`
176 - val - The marker value, or `PETSC_DETERMINE` to use some value in the closure (or 1 if none are found)
177 
178   Output Parameter:
179 . label - The `DMLabel` marking boundary faces with the given value
180 
181   Level: developer
182 
183   Note:
184   This function will use the point `PetscSF` from the input `DM` and the ownership of the support cells to exclude points on the partition boundary from being marked. If you also wish to mark the partition boundary, you can use `DMSetPointSF()` to temporarily set it to `NULL`, and then reset it to the original object after the call.
185 
186   In DMForest there can be facets support sizes of which alone can not determine whether they are on the boundary. Currently, this function is not guaranteed to produce the correct result in such case.
187 
188 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMLabelCreate()`, `DMCreateLabel()`
189 @*/
190 PetscErrorCode DMPlexMarkBoundaryFaces(DM dm, PetscInt val, DMLabel label)
191 {
192   DM                     plex;
193   DMPlexInterpolatedFlag flg;
194 
195   PetscFunctionBegin;
196   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
197   PetscCall(DMConvert(dm, DMPLEX, &plex));
198   PetscCall(DMPlexIsInterpolated(plex, &flg));
199   PetscCheck(flg == DMPLEX_INTERPOLATED_FULL, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM is not fully interpolated on this rank");
200   PetscCall(DMPlexMarkBoundaryFaces_Internal(plex, val, 0, label, PETSC_FALSE));
201   PetscCall(DMDestroy(&plex));
202   PetscFunctionReturn(PETSC_SUCCESS);
203 }
204 
205 static PetscErrorCode DMPlexLabelComplete_Internal(DM dm, DMLabel label, PetscBool completeCells)
206 {
207   IS              valueIS;
208   PetscSF         sfPoint;
209   const PetscInt *values;
210   PetscInt        numValues, v, cStart, cEnd, nroots;
211 
212   PetscFunctionBegin;
213   PetscCall(DMLabelGetNumValues(label, &numValues));
214   PetscCall(DMLabelGetValueIS(label, &valueIS));
215   PetscCall(DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd));
216   PetscCall(ISGetIndices(valueIS, &values));
217   for (v = 0; v < numValues; ++v) {
218     IS              pointIS;
219     const PetscInt *points;
220     PetscInt        numPoints, p;
221 
222     PetscCall(DMLabelGetStratumSize(label, values[v], &numPoints));
223     PetscCall(DMLabelGetStratumIS(label, values[v], &pointIS));
224     PetscCall(ISGetIndices(pointIS, &points));
225     for (p = 0; p < numPoints; ++p) {
226       PetscInt  q       = points[p];
227       PetscInt *closure = NULL;
228       PetscInt  closureSize, c;
229 
230       if (cStart <= q && q < cEnd && !completeCells) { /* skip cells */
231         continue;
232       }
233       PetscCall(DMPlexGetTransitiveClosure(dm, q, PETSC_TRUE, &closureSize, &closure));
234       for (c = 0; c < closureSize * 2; c += 2) PetscCall(DMLabelSetValue(label, closure[c], values[v]));
235       PetscCall(DMPlexRestoreTransitiveClosure(dm, q, PETSC_TRUE, &closureSize, &closure));
236     }
237     PetscCall(ISRestoreIndices(pointIS, &points));
238     PetscCall(ISDestroy(&pointIS));
239   }
240   PetscCall(ISRestoreIndices(valueIS, &values));
241   PetscCall(ISDestroy(&valueIS));
242   PetscCall(DMGetPointSF(dm, &sfPoint));
243   PetscCall(PetscSFGetGraph(sfPoint, &nroots, NULL, NULL, NULL));
244   if (nroots >= 0) {
245     DMLabel         lblRoots, lblLeaves;
246     IS              valueIS, pointIS;
247     const PetscInt *values;
248     PetscInt        numValues, v;
249 
250     /* Pull point contributions from remote leaves into local roots */
251     PetscCall(DMLabelGather(label, sfPoint, &lblLeaves));
252     PetscCall(DMLabelGetValueIS(lblLeaves, &valueIS));
253     PetscCall(ISGetLocalSize(valueIS, &numValues));
254     PetscCall(ISGetIndices(valueIS, &values));
255     for (v = 0; v < numValues; ++v) {
256       const PetscInt value = values[v];
257 
258       PetscCall(DMLabelGetStratumIS(lblLeaves, value, &pointIS));
259       PetscCall(DMLabelInsertIS(label, pointIS, value));
260       PetscCall(ISDestroy(&pointIS));
261     }
262     PetscCall(ISRestoreIndices(valueIS, &values));
263     PetscCall(ISDestroy(&valueIS));
264     PetscCall(DMLabelDestroy(&lblLeaves));
265     /* Push point contributions from roots into remote leaves */
266     PetscCall(DMLabelDistribute(label, sfPoint, &lblRoots));
267     PetscCall(DMLabelGetValueIS(lblRoots, &valueIS));
268     PetscCall(ISGetLocalSize(valueIS, &numValues));
269     PetscCall(ISGetIndices(valueIS, &values));
270     for (v = 0; v < numValues; ++v) {
271       const PetscInt value = values[v];
272 
273       PetscCall(DMLabelGetStratumIS(lblRoots, value, &pointIS));
274       PetscCall(DMLabelInsertIS(label, pointIS, value));
275       PetscCall(ISDestroy(&pointIS));
276     }
277     PetscCall(ISRestoreIndices(valueIS, &values));
278     PetscCall(ISDestroy(&valueIS));
279     PetscCall(DMLabelDestroy(&lblRoots));
280   }
281   PetscFunctionReturn(PETSC_SUCCESS);
282 }
283 
284 /*@
285   DMPlexLabelComplete - Starting with a label marking points on a surface, we add the transitive closure to the surface
286 
287   Input Parameters:
288 + dm    - The `DM`
289 - label - A `DMLabel` marking the surface points
290 
291   Output Parameter:
292 . label - A `DMLabel` marking all surface points in the transitive closure
293 
294   Level: developer
295 
296 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexLabelCohesiveComplete()`
297 @*/
298 PetscErrorCode DMPlexLabelComplete(DM dm, DMLabel label)
299 {
300   PetscFunctionBegin;
301   PetscCall(DMPlexLabelComplete_Internal(dm, label, PETSC_TRUE));
302   PetscFunctionReturn(PETSC_SUCCESS);
303 }
304 
305 /*@
306   DMPlexLabelAddCells - Starting with a label marking points on a surface, we add a cell for each point
307 
308   Input Parameters:
309 + dm    - The `DM`
310 - label - A `DMLabel` marking the surface points
311 
312   Output Parameter:
313 . label - A `DMLabel` incorporating cells
314 
315   Level: developer
316 
317   Note:
318   The cells allow FEM boundary conditions to be applied using the cell geometry
319 
320 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexLabelAddFaceCells()`, `DMPlexLabelComplete()`, `DMPlexLabelCohesiveComplete()`
321 @*/
322 PetscErrorCode DMPlexLabelAddCells(DM dm, DMLabel label)
323 {
324   IS              valueIS;
325   const PetscInt *values;
326   PetscInt        numValues, v, csStart, csEnd, chStart, chEnd;
327 
328   PetscFunctionBegin;
329   PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &csStart, &csEnd));
330   PetscCall(DMPlexGetHeightStratum(dm, 0, &chStart, &chEnd));
331   PetscCall(DMLabelGetNumValues(label, &numValues));
332   PetscCall(DMLabelGetValueIS(label, &valueIS));
333   PetscCall(ISGetIndices(valueIS, &values));
334   for (v = 0; v < numValues; ++v) {
335     IS              pointIS;
336     const PetscInt *points;
337     PetscInt        numPoints, p;
338 
339     PetscCall(DMLabelGetStratumSize(label, values[v], &numPoints));
340     PetscCall(DMLabelGetStratumIS(label, values[v], &pointIS));
341     PetscCall(ISGetIndices(pointIS, &points));
342     for (p = 0; p < numPoints; ++p) {
343       const PetscInt point   = points[p];
344       PetscInt      *closure = NULL;
345       PetscInt       closureSize, cl, h, cStart, cEnd;
346       DMPolytopeType ct;
347 
348       // If the point is a hybrid, allow hybrid cells
349       PetscCall(DMPlexGetCellType(dm, point, &ct));
350       PetscCall(DMPlexGetPointHeight(dm, point, &h));
351       if (DMPolytopeTypeIsHybrid(ct)) {
352         cStart = chStart;
353         cEnd   = chEnd;
354       } else {
355         cStart = csStart;
356         cEnd   = csEnd;
357       }
358 
359       PetscCall(DMPlexGetTransitiveClosure(dm, points[p], PETSC_FALSE, &closureSize, &closure));
360       for (cl = closureSize - 1; cl > 0; --cl) {
361         const PetscInt cell = closure[cl * 2];
362         if ((cell >= cStart) && (cell < cEnd)) {
363           PetscCall(DMLabelSetValue(label, cell, values[v]));
364           break;
365         }
366       }
367       PetscCall(DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_FALSE, &closureSize, &closure));
368     }
369     PetscCall(ISRestoreIndices(pointIS, &points));
370     PetscCall(ISDestroy(&pointIS));
371   }
372   PetscCall(ISRestoreIndices(valueIS, &values));
373   PetscCall(ISDestroy(&valueIS));
374   PetscFunctionReturn(PETSC_SUCCESS);
375 }
376 
377 /*@
378   DMPlexLabelAddFaceCells - Starting with a label marking faces on a surface, we add a cell for each face
379 
380   Input Parameters:
381 + dm    - The `DM`
382 - label - A `DMLabel` marking the surface points
383 
384   Output Parameter:
385 . label - A `DMLabel` incorporating cells
386 
387   Level: developer
388 
389   Note:
390   The cells allow FEM boundary conditions to be applied using the cell geometry
391 
392 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexLabelAddCells()`, `DMPlexLabelComplete()`, `DMPlexLabelCohesiveComplete()`
393 @*/
394 PetscErrorCode DMPlexLabelAddFaceCells(DM dm, DMLabel label)
395 {
396   IS              valueIS;
397   const PetscInt *values;
398   PetscInt        numValues, v, cStart, cEnd, fStart, fEnd;
399 
400   PetscFunctionBegin;
401   PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd));
402   PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
403   PetscCall(DMLabelGetNumValues(label, &numValues));
404   PetscCall(DMLabelGetValueIS(label, &valueIS));
405   PetscCall(ISGetIndices(valueIS, &values));
406   for (v = 0; v < numValues; ++v) {
407     IS              pointIS;
408     const PetscInt *points;
409     PetscInt        numPoints, p;
410 
411     PetscCall(DMLabelGetStratumSize(label, values[v], &numPoints));
412     PetscCall(DMLabelGetStratumIS(label, values[v], &pointIS));
413     PetscCall(ISGetIndices(pointIS, &points));
414     for (p = 0; p < numPoints; ++p) {
415       const PetscInt face    = points[p];
416       PetscInt      *closure = NULL;
417       PetscInt       closureSize, cl;
418 
419       if ((face < fStart) || (face >= fEnd)) continue;
420       PetscCall(DMPlexGetTransitiveClosure(dm, face, PETSC_FALSE, &closureSize, &closure));
421       for (cl = closureSize - 1; cl > 0; --cl) {
422         const PetscInt cell = closure[cl * 2];
423         if ((cell >= cStart) && (cell < cEnd)) {
424           PetscCall(DMLabelSetValue(label, cell, values[v]));
425           break;
426         }
427       }
428       PetscCall(DMPlexRestoreTransitiveClosure(dm, face, PETSC_FALSE, &closureSize, &closure));
429     }
430     PetscCall(ISRestoreIndices(pointIS, &points));
431     PetscCall(ISDestroy(&pointIS));
432   }
433   PetscCall(ISRestoreIndices(valueIS, &values));
434   PetscCall(ISDestroy(&valueIS));
435   PetscFunctionReturn(PETSC_SUCCESS);
436 }
437 
438 /*@
439   DMPlexLabelClearCells - Remove cells from a label
440 
441   Input Parameters:
442 + dm    - The `DM`
443 - label - A `DMLabel` marking surface points and their adjacent cells
444 
445   Output Parameter:
446 . label - A `DMLabel` without cells
447 
448   Level: developer
449 
450   Note:
451   This undoes `DMPlexLabelAddCells()` or `DMPlexLabelAddFaceCells()`
452 
453 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexLabelComplete()`, `DMPlexLabelCohesiveComplete()`, `DMPlexLabelAddCells()`
454 @*/
455 PetscErrorCode DMPlexLabelClearCells(DM dm, DMLabel label)
456 {
457   IS              valueIS;
458   const PetscInt *values;
459   PetscInt        numValues, v, cStart, cEnd;
460 
461   PetscFunctionBegin;
462   PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd));
463   PetscCall(DMLabelGetNumValues(label, &numValues));
464   PetscCall(DMLabelGetValueIS(label, &valueIS));
465   PetscCall(ISGetIndices(valueIS, &values));
466   for (v = 0; v < numValues; ++v) {
467     IS              pointIS;
468     const PetscInt *points;
469     PetscInt        numPoints, p;
470 
471     PetscCall(DMLabelGetStratumSize(label, values[v], &numPoints));
472     PetscCall(DMLabelGetStratumIS(label, values[v], &pointIS));
473     PetscCall(ISGetIndices(pointIS, &points));
474     for (p = 0; p < numPoints; ++p) {
475       PetscInt point = points[p];
476 
477       if (point >= cStart && point < cEnd) PetscCall(DMLabelClearValue(label, point, values[v]));
478     }
479     PetscCall(ISRestoreIndices(pointIS, &points));
480     PetscCall(ISDestroy(&pointIS));
481   }
482   PetscCall(ISRestoreIndices(valueIS, &values));
483   PetscCall(ISDestroy(&valueIS));
484   PetscFunctionReturn(PETSC_SUCCESS);
485 }
486 
487 /* take (oldEnd, added) pairs, ordered by height and convert them to (oldstart, newstart) pairs, ordered by ascending
488  * index (skipping first, which is (0,0)) */
489 static inline PetscErrorCode DMPlexShiftPointSetUp_Internal(PetscInt depth, PetscInt depthShift[])
490 {
491   PetscInt d, off = 0;
492 
493   PetscFunctionBegin;
494   /* sort by (oldend): yes this is an O(n^2) sort, we expect depth <= 3 */
495   for (d = 0; d < depth; d++) {
496     PetscInt firstd     = d;
497     PetscInt firstStart = depthShift[2 * d];
498     PetscInt e;
499 
500     for (e = d + 1; e <= depth; e++) {
501       if (depthShift[2 * e] < firstStart) {
502         firstd     = e;
503         firstStart = depthShift[2 * d];
504       }
505     }
506     if (firstd != d) {
507       PetscInt swap[2];
508 
509       e                     = firstd;
510       swap[0]               = depthShift[2 * d];
511       swap[1]               = depthShift[2 * d + 1];
512       depthShift[2 * d]     = depthShift[2 * e];
513       depthShift[2 * d + 1] = depthShift[2 * e + 1];
514       depthShift[2 * e]     = swap[0];
515       depthShift[2 * e + 1] = swap[1];
516     }
517   }
518   /* convert (oldstart, added) to (oldstart, newstart) */
519   for (d = 0; d <= depth; d++) {
520     off += depthShift[2 * d + 1];
521     depthShift[2 * d + 1] = depthShift[2 * d] + off;
522   }
523   PetscFunctionReturn(PETSC_SUCCESS);
524 }
525 
526 /* depthShift is a list of (old, new) pairs */
527 static inline PetscInt DMPlexShiftPoint_Internal(PetscInt p, PetscInt depth, PetscInt depthShift[])
528 {
529   PetscInt d;
530   PetscInt newOff = 0;
531 
532   for (d = 0; d <= depth; d++) {
533     if (p < depthShift[2 * d]) return p + newOff;
534     else newOff = depthShift[2 * d + 1] - depthShift[2 * d];
535   }
536   return p + newOff;
537 }
538 
539 /* depthShift is a list of (old, new) pairs */
540 static inline PetscInt DMPlexShiftPointInverse_Internal(PetscInt p, PetscInt depth, PetscInt depthShift[])
541 {
542   PetscInt d;
543   PetscInt newOff = 0;
544 
545   for (d = 0; d <= depth; d++) {
546     if (p < depthShift[2 * d + 1]) return p + newOff;
547     else newOff = depthShift[2 * d] - depthShift[2 * d + 1];
548   }
549   return p + newOff;
550 }
551 
552 static PetscErrorCode DMPlexShiftSizes_Internal(DM dm, PetscInt depthShift[], DM dmNew)
553 {
554   PetscInt depth = 0, d, pStart, pEnd, p;
555   DMLabel  depthLabel;
556 
557   PetscFunctionBegin;
558   PetscCall(DMPlexGetDepth(dm, &depth));
559   if (depth < 0) PetscFunctionReturn(PETSC_SUCCESS);
560   /* Step 1: Expand chart */
561   PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
562   pEnd = DMPlexShiftPoint_Internal(pEnd, depth, depthShift);
563   PetscCall(DMPlexSetChart(dmNew, pStart, pEnd));
564   PetscCall(DMCreateLabel(dmNew, "depth"));
565   PetscCall(DMPlexGetDepthLabel(dmNew, &depthLabel));
566   PetscCall(DMCreateLabel(dmNew, "celltype"));
567   /* Step 2: Set cone and support sizes */
568   for (d = 0; d <= depth; ++d) {
569     PetscInt pStartNew, pEndNew;
570     IS       pIS;
571 
572     PetscCall(DMPlexGetDepthStratum(dm, d, &pStart, &pEnd));
573     pStartNew = DMPlexShiftPoint_Internal(pStart, depth, depthShift);
574     pEndNew   = DMPlexShiftPoint_Internal(pEnd, depth, depthShift);
575     PetscCall(ISCreateStride(PETSC_COMM_SELF, pEndNew - pStartNew, pStartNew, 1, &pIS));
576     PetscCall(DMLabelSetStratumIS(depthLabel, d, pIS));
577     PetscCall(ISDestroy(&pIS));
578     for (p = pStart; p < pEnd; ++p) {
579       PetscInt       newp = DMPlexShiftPoint_Internal(p, depth, depthShift);
580       PetscInt       size;
581       DMPolytopeType ct;
582 
583       PetscCall(DMPlexGetConeSize(dm, p, &size));
584       PetscCall(DMPlexSetConeSize(dmNew, newp, size));
585       PetscCall(DMPlexGetSupportSize(dm, p, &size));
586       PetscCall(DMPlexSetSupportSize(dmNew, newp, size));
587       PetscCall(DMPlexGetCellType(dm, p, &ct));
588       PetscCall(DMPlexSetCellType(dmNew, newp, ct));
589     }
590   }
591   PetscFunctionReturn(PETSC_SUCCESS);
592 }
593 
594 static PetscErrorCode DMPlexShiftPoints_Internal(DM dm, PetscInt depthShift[], DM dmNew)
595 {
596   PetscInt *newpoints;
597   PetscInt  depth = 0, maxConeSize, maxSupportSize, maxConeSizeNew, maxSupportSizeNew, pStart, pEnd, p;
598 
599   PetscFunctionBegin;
600   PetscCall(DMPlexGetDepth(dm, &depth));
601   if (depth < 0) PetscFunctionReturn(PETSC_SUCCESS);
602   PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize));
603   PetscCall(DMPlexGetMaxSizes(dmNew, &maxConeSizeNew, &maxSupportSizeNew));
604   PetscCall(PetscMalloc1(PetscMax(PetscMax(maxConeSize, maxSupportSize), PetscMax(maxConeSizeNew, maxSupportSizeNew)), &newpoints));
605   /* Step 5: Set cones and supports */
606   PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
607   for (p = pStart; p < pEnd; ++p) {
608     const PetscInt *points = NULL, *orientations = NULL;
609     PetscInt        size, sizeNew, i, newp = DMPlexShiftPoint_Internal(p, depth, depthShift);
610 
611     PetscCall(DMPlexGetConeSize(dm, p, &size));
612     PetscCall(DMPlexGetCone(dm, p, &points));
613     PetscCall(DMPlexGetConeOrientation(dm, p, &orientations));
614     for (i = 0; i < size; ++i) newpoints[i] = DMPlexShiftPoint_Internal(points[i], depth, depthShift);
615     PetscCall(DMPlexSetCone(dmNew, newp, newpoints));
616     PetscCall(DMPlexSetConeOrientation(dmNew, newp, orientations));
617     PetscCall(DMPlexGetSupportSize(dm, p, &size));
618     PetscCall(DMPlexGetSupportSize(dmNew, newp, &sizeNew));
619     PetscCall(DMPlexGetSupport(dm, p, &points));
620     for (i = 0; i < size; ++i) newpoints[i] = DMPlexShiftPoint_Internal(points[i], depth, depthShift);
621     for (i = size; i < sizeNew; ++i) newpoints[i] = 0;
622     PetscCall(DMPlexSetSupport(dmNew, newp, newpoints));
623   }
624   PetscCall(PetscFree(newpoints));
625   PetscFunctionReturn(PETSC_SUCCESS);
626 }
627 
628 static PetscErrorCode DMPlexShiftCoordinates_Internal(DM dm, PetscInt depthShift[], DM dmNew)
629 {
630   PetscSection coordSection, newCoordSection;
631   Vec          coordinates, newCoordinates;
632   PetscScalar *coords, *newCoords;
633   PetscInt     coordSize, sStart, sEnd;
634   PetscInt     dim, depth = 0, cStart, cEnd, cStartNew, cEndNew, c, vStart, vEnd, vStartNew, vEndNew, v;
635   PetscBool    hasCells;
636 
637   PetscFunctionBegin;
638   PetscCall(DMGetCoordinateDim(dm, &dim));
639   PetscCall(DMSetCoordinateDim(dmNew, dim));
640   PetscCall(DMPlexGetDepth(dm, &depth));
641   /* Step 8: Convert coordinates */
642   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
643   PetscCall(DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd));
644   PetscCall(DMPlexGetDepthStratum(dmNew, 0, &vStartNew, &vEndNew));
645   PetscCall(DMPlexGetHeightStratum(dmNew, 0, &cStartNew, &cEndNew));
646   PetscCall(DMGetCoordinateSection(dm, &coordSection));
647   PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dm), &newCoordSection));
648   PetscCall(PetscSectionSetNumFields(newCoordSection, 1));
649   PetscCall(PetscSectionSetFieldComponents(newCoordSection, 0, dim));
650   PetscCall(PetscSectionGetChart(coordSection, &sStart, &sEnd));
651   hasCells = sStart == cStart ? PETSC_TRUE : PETSC_FALSE;
652   PetscCall(PetscSectionSetChart(newCoordSection, hasCells ? cStartNew : vStartNew, vEndNew));
653   if (hasCells) {
654     for (c = cStart; c < cEnd; ++c) {
655       PetscInt cNew = DMPlexShiftPoint_Internal(c, depth, depthShift), dof;
656 
657       PetscCall(PetscSectionGetDof(coordSection, c, &dof));
658       PetscCall(PetscSectionSetDof(newCoordSection, cNew, dof));
659       PetscCall(PetscSectionSetFieldDof(newCoordSection, cNew, 0, dof));
660     }
661   }
662   for (v = vStartNew; v < vEndNew; ++v) {
663     PetscCall(PetscSectionSetDof(newCoordSection, v, dim));
664     PetscCall(PetscSectionSetFieldDof(newCoordSection, v, 0, dim));
665   }
666   PetscCall(PetscSectionSetUp(newCoordSection));
667   PetscCall(DMSetCoordinateSection(dmNew, PETSC_DETERMINE, newCoordSection));
668   PetscCall(PetscSectionGetStorageSize(newCoordSection, &coordSize));
669   PetscCall(VecCreate(PETSC_COMM_SELF, &newCoordinates));
670   PetscCall(PetscObjectSetName((PetscObject)newCoordinates, "coordinates"));
671   PetscCall(VecSetSizes(newCoordinates, coordSize, PETSC_DETERMINE));
672   PetscCall(VecSetBlockSize(newCoordinates, dim));
673   PetscCall(VecSetType(newCoordinates, VECSTANDARD));
674   PetscCall(DMSetCoordinatesLocal(dmNew, newCoordinates));
675   PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
676   PetscCall(VecGetArray(coordinates, &coords));
677   PetscCall(VecGetArray(newCoordinates, &newCoords));
678   if (hasCells) {
679     for (c = cStart; c < cEnd; ++c) {
680       PetscInt cNew = DMPlexShiftPoint_Internal(c, depth, depthShift), dof, off, noff, d;
681 
682       PetscCall(PetscSectionGetDof(coordSection, c, &dof));
683       PetscCall(PetscSectionGetOffset(coordSection, c, &off));
684       PetscCall(PetscSectionGetOffset(newCoordSection, cNew, &noff));
685       for (d = 0; d < dof; ++d) newCoords[noff + d] = coords[off + d];
686     }
687   }
688   for (v = vStart; v < vEnd; ++v) {
689     PetscInt dof, off, noff, d;
690 
691     PetscCall(PetscSectionGetDof(coordSection, v, &dof));
692     PetscCall(PetscSectionGetOffset(coordSection, v, &off));
693     PetscCall(PetscSectionGetOffset(newCoordSection, DMPlexShiftPoint_Internal(v, depth, depthShift), &noff));
694     for (d = 0; d < dof; ++d) newCoords[noff + d] = coords[off + d];
695   }
696   PetscCall(VecRestoreArray(coordinates, &coords));
697   PetscCall(VecRestoreArray(newCoordinates, &newCoords));
698   PetscCall(VecDestroy(&newCoordinates));
699   PetscCall(PetscSectionDestroy(&newCoordSection));
700   PetscFunctionReturn(PETSC_SUCCESS);
701 }
702 
703 static PetscErrorCode DMPlexShiftSF_Single(DM dm, PetscInt depthShift[], PetscSF sf, PetscSF sfNew)
704 {
705   const PetscSFNode *remotePoints;
706   PetscSFNode       *gremotePoints;
707   const PetscInt    *localPoints;
708   PetscInt          *glocalPoints, *newLocation, *newRemoteLocation;
709   PetscInt           numRoots, numLeaves, l, pStart, pEnd, depth = 0, totShift = 0;
710 
711   PetscFunctionBegin;
712   PetscCall(DMPlexGetDepth(dm, &depth));
713   PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
714   PetscCall(PetscSFGetGraph(sf, &numRoots, &numLeaves, &localPoints, &remotePoints));
715   totShift = DMPlexShiftPoint_Internal(pEnd, depth, depthShift) - pEnd;
716   if (numRoots >= 0) {
717     PetscCall(PetscMalloc2(numRoots, &newLocation, pEnd - pStart, &newRemoteLocation));
718     for (l = 0; l < numRoots; ++l) newLocation[l] = DMPlexShiftPoint_Internal(l, depth, depthShift);
719     PetscCall(PetscSFBcastBegin(sf, MPIU_INT, newLocation, newRemoteLocation, MPI_REPLACE));
720     PetscCall(PetscSFBcastEnd(sf, MPIU_INT, newLocation, newRemoteLocation, MPI_REPLACE));
721     PetscCall(PetscMalloc1(numLeaves, &glocalPoints));
722     PetscCall(PetscMalloc1(numLeaves, &gremotePoints));
723     for (l = 0; l < numLeaves; ++l) {
724       glocalPoints[l]        = DMPlexShiftPoint_Internal(localPoints[l], depth, depthShift);
725       gremotePoints[l].rank  = remotePoints[l].rank;
726       gremotePoints[l].index = newRemoteLocation[localPoints[l]];
727     }
728     PetscCall(PetscFree2(newLocation, newRemoteLocation));
729     PetscCall(PetscSFSetGraph(sfNew, numRoots + totShift, numLeaves, glocalPoints, PETSC_OWN_POINTER, gremotePoints, PETSC_OWN_POINTER));
730   }
731   PetscFunctionReturn(PETSC_SUCCESS);
732 }
733 
734 static PetscErrorCode DMPlexShiftSF_Internal(DM dm, PetscInt depthShift[], DM dmNew)
735 {
736   PetscSF   sfPoint, sfPointNew;
737   PetscBool useNatural;
738 
739   PetscFunctionBegin;
740   /* Step 9: Convert pointSF */
741   PetscCall(DMGetPointSF(dm, &sfPoint));
742   PetscCall(DMGetPointSF(dmNew, &sfPointNew));
743   PetscCall(DMPlexShiftSF_Single(dm, depthShift, sfPoint, sfPointNew));
744   /* Step 9b: Convert naturalSF */
745   PetscCall(DMGetUseNatural(dm, &useNatural));
746   if (useNatural) {
747     PetscSF sfNat, sfNatNew;
748 
749     PetscCall(DMSetUseNatural(dmNew, useNatural));
750     PetscCall(DMGetNaturalSF(dm, &sfNat));
751     PetscCall(DMGetNaturalSF(dmNew, &sfNatNew));
752     PetscCall(DMPlexShiftSF_Single(dm, depthShift, sfNat, sfNatNew));
753   }
754   PetscFunctionReturn(PETSC_SUCCESS);
755 }
756 
757 static PetscErrorCode DMPlexShiftLabels_Internal(DM dm, PetscInt depthShift[], DM dmNew)
758 {
759   PetscInt depth = 0, numLabels, l;
760 
761   PetscFunctionBegin;
762   PetscCall(DMPlexGetDepth(dm, &depth));
763   /* Step 10: Convert labels */
764   PetscCall(DMGetNumLabels(dm, &numLabels));
765   for (l = 0; l < numLabels; ++l) {
766     DMLabel         label, newlabel;
767     const char     *lname;
768     PetscBool       isDepth, isDim;
769     IS              valueIS;
770     const PetscInt *values;
771     PetscInt        numValues, val;
772 
773     PetscCall(DMGetLabelName(dm, l, &lname));
774     PetscCall(PetscStrcmp(lname, "depth", &isDepth));
775     if (isDepth) continue;
776     PetscCall(PetscStrcmp(lname, "dim", &isDim));
777     if (isDim) continue;
778     PetscCall(DMCreateLabel(dmNew, lname));
779     PetscCall(DMGetLabel(dm, lname, &label));
780     PetscCall(DMGetLabel(dmNew, lname, &newlabel));
781     PetscCall(DMLabelGetDefaultValue(label, &val));
782     PetscCall(DMLabelSetDefaultValue(newlabel, val));
783     PetscCall(DMLabelGetValueIS(label, &valueIS));
784     PetscCall(ISGetLocalSize(valueIS, &numValues));
785     PetscCall(ISGetIndices(valueIS, &values));
786     for (val = 0; val < numValues; ++val) {
787       IS              pointIS;
788       const PetscInt *points;
789       PetscInt        numPoints, p;
790 
791       PetscCall(DMLabelGetStratumIS(label, values[val], &pointIS));
792       PetscCall(ISGetLocalSize(pointIS, &numPoints));
793       PetscCall(ISGetIndices(pointIS, &points));
794       for (p = 0; p < numPoints; ++p) {
795         const PetscInt newpoint = DMPlexShiftPoint_Internal(points[p], depth, depthShift);
796 
797         PetscCall(DMLabelSetValue(newlabel, newpoint, values[val]));
798       }
799       PetscCall(ISRestoreIndices(pointIS, &points));
800       PetscCall(ISDestroy(&pointIS));
801     }
802     PetscCall(ISRestoreIndices(valueIS, &values));
803     PetscCall(ISDestroy(&valueIS));
804   }
805   PetscFunctionReturn(PETSC_SUCCESS);
806 }
807 
808 static PetscErrorCode DMPlexCreateVTKLabel_Internal(DM dm, PetscBool createGhostLabel, DM dmNew)
809 {
810   PetscSF            sfPoint;
811   DMLabel            vtkLabel, ghostLabel = NULL;
812   const PetscSFNode *leafRemote;
813   const PetscInt    *leafLocal;
814   PetscInt           cellHeight, cStart, cEnd, c, fStart, fEnd, f, numLeaves, l;
815   PetscMPIInt        rank;
816 
817   PetscFunctionBegin;
818   /* Step 11: Make label for output (vtk) and to mark ghost points (ghost) */
819   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
820   PetscCall(DMGetPointSF(dm, &sfPoint));
821   PetscCall(DMPlexGetVTKCellHeight(dmNew, &cellHeight));
822   PetscCall(DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd));
823   PetscCall(PetscSFGetGraph(sfPoint, NULL, &numLeaves, &leafLocal, &leafRemote));
824   PetscCall(DMCreateLabel(dmNew, "vtk"));
825   PetscCall(DMGetLabel(dmNew, "vtk", &vtkLabel));
826   if (createGhostLabel) {
827     PetscCall(DMCreateLabel(dmNew, "ghost"));
828     PetscCall(DMGetLabel(dmNew, "ghost", &ghostLabel));
829   }
830   for (l = 0, c = cStart; l < numLeaves && c < cEnd; ++l, ++c) {
831     for (; c < leafLocal[l] && c < cEnd; ++c) PetscCall(DMLabelSetValue(vtkLabel, c, 1));
832     if (leafLocal[l] >= cEnd) break;
833     if (leafRemote[l].rank == rank) {
834       PetscCall(DMLabelSetValue(vtkLabel, c, 1));
835     } else if (ghostLabel) PetscCall(DMLabelSetValue(ghostLabel, c, 2));
836   }
837   for (; c < cEnd; ++c) PetscCall(DMLabelSetValue(vtkLabel, c, 1));
838   if (ghostLabel) {
839     PetscCall(DMPlexGetHeightStratum(dmNew, 1, &fStart, &fEnd));
840     for (f = fStart; f < fEnd; ++f) {
841       PetscInt numCells;
842 
843       PetscCall(DMPlexGetSupportSize(dmNew, f, &numCells));
844       if (numCells < 2) {
845         PetscCall(DMLabelSetValue(ghostLabel, f, 1));
846       } else {
847         const PetscInt *cells = NULL;
848         PetscInt        vA, vB;
849 
850         PetscCall(DMPlexGetSupport(dmNew, f, &cells));
851         PetscCall(DMLabelGetValue(vtkLabel, cells[0], &vA));
852         PetscCall(DMLabelGetValue(vtkLabel, cells[1], &vB));
853         if (vA != 1 && vB != 1) PetscCall(DMLabelSetValue(ghostLabel, f, 1));
854       }
855     }
856   }
857   PetscFunctionReturn(PETSC_SUCCESS);
858 }
859 
860 static PetscErrorCode DMPlexShiftTree_Internal(DM dm, PetscInt depthShift[], DM dmNew)
861 {
862   DM           refTree;
863   PetscSection pSec;
864   PetscInt    *parents, *childIDs;
865 
866   PetscFunctionBegin;
867   PetscCall(DMPlexGetReferenceTree(dm, &refTree));
868   PetscCall(DMPlexSetReferenceTree(dmNew, refTree));
869   PetscCall(DMPlexGetTree(dm, &pSec, &parents, &childIDs, NULL, NULL));
870   if (pSec) {
871     PetscInt     p, pStart, pEnd, *parentsShifted, pStartShifted, pEndShifted, depth;
872     PetscInt    *childIDsShifted;
873     PetscSection pSecShifted;
874 
875     PetscCall(PetscSectionGetChart(pSec, &pStart, &pEnd));
876     PetscCall(DMPlexGetDepth(dm, &depth));
877     pStartShifted = DMPlexShiftPoint_Internal(pStart, depth, depthShift);
878     pEndShifted   = DMPlexShiftPoint_Internal(pEnd, depth, depthShift);
879     PetscCall(PetscMalloc2(pEndShifted - pStartShifted, &parentsShifted, pEndShifted - pStartShifted, &childIDsShifted));
880     PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dmNew), &pSecShifted));
881     PetscCall(PetscSectionSetChart(pSecShifted, pStartShifted, pEndShifted));
882     for (p = pStartShifted; p < pEndShifted; p++) {
883       /* start off assuming no children */
884       PetscCall(PetscSectionSetDof(pSecShifted, p, 0));
885     }
886     for (p = pStart; p < pEnd; p++) {
887       PetscInt dof;
888       PetscInt pNew = DMPlexShiftPoint_Internal(p, depth, depthShift);
889 
890       PetscCall(PetscSectionGetDof(pSec, p, &dof));
891       PetscCall(PetscSectionSetDof(pSecShifted, pNew, dof));
892     }
893     PetscCall(PetscSectionSetUp(pSecShifted));
894     for (p = pStart; p < pEnd; p++) {
895       PetscInt dof;
896       PetscInt pNew = DMPlexShiftPoint_Internal(p, depth, depthShift);
897 
898       PetscCall(PetscSectionGetDof(pSec, p, &dof));
899       if (dof) {
900         PetscInt off, offNew;
901 
902         PetscCall(PetscSectionGetOffset(pSec, p, &off));
903         PetscCall(PetscSectionGetOffset(pSecShifted, pNew, &offNew));
904         parentsShifted[offNew]  = DMPlexShiftPoint_Internal(parents[off], depth, depthShift);
905         childIDsShifted[offNew] = childIDs[off];
906       }
907     }
908     PetscCall(DMPlexSetTree(dmNew, pSecShifted, parentsShifted, childIDsShifted));
909     PetscCall(PetscFree2(parentsShifted, childIDsShifted));
910     PetscCall(PetscSectionDestroy(&pSecShifted));
911   }
912   PetscFunctionReturn(PETSC_SUCCESS);
913 }
914 
915 static PetscErrorCode DMPlexConstructGhostCells_Internal(DM dm, DMLabel label, PetscInt *numGhostCells, DM gdm)
916 {
917   PetscSF          sf;
918   IS               valueIS;
919   const PetscInt  *values, *leaves;
920   PetscInt        *depthShift;
921   PetscInt         d, depth = 0, nleaves, loc, Ng, numFS, fs, fStart, fEnd, ghostCell, cEnd, c;
922   const PetscReal *maxCell, *Lstart, *L;
923 
924   PetscFunctionBegin;
925   PetscCall(DMGetPointSF(dm, &sf));
926   PetscCall(PetscSFGetGraph(sf, NULL, &nleaves, &leaves, NULL));
927   nleaves = PetscMax(0, nleaves);
928   PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
929   /* Count ghost cells */
930   PetscCall(DMLabelGetValueIS(label, &valueIS));
931   PetscCall(ISGetLocalSize(valueIS, &numFS));
932   PetscCall(ISGetIndices(valueIS, &values));
933   Ng = 0;
934   for (fs = 0; fs < numFS; ++fs) {
935     IS              faceIS;
936     const PetscInt *faces;
937     PetscInt        numFaces, f, numBdFaces = 0;
938 
939     PetscCall(DMLabelGetStratumIS(label, values[fs], &faceIS));
940     PetscCall(ISGetLocalSize(faceIS, &numFaces));
941     PetscCall(ISGetIndices(faceIS, &faces));
942     for (f = 0; f < numFaces; ++f) {
943       PetscInt numChildren;
944 
945       PetscCall(PetscFindInt(faces[f], nleaves, leaves, &loc));
946       PetscCall(DMPlexGetTreeChildren(dm, faces[f], &numChildren, NULL));
947       /* non-local and ancestors points don't get to register ghosts */
948       if (loc >= 0 || numChildren) continue;
949       if ((faces[f] >= fStart) && (faces[f] < fEnd)) ++numBdFaces;
950     }
951     Ng += numBdFaces;
952     PetscCall(ISRestoreIndices(faceIS, &faces));
953     PetscCall(ISDestroy(&faceIS));
954   }
955   PetscCall(DMPlexGetDepth(dm, &depth));
956   PetscCall(PetscMalloc1(2 * (depth + 1), &depthShift));
957   for (d = 0; d <= depth; d++) {
958     PetscInt dEnd;
959 
960     PetscCall(DMPlexGetDepthStratum(dm, d, NULL, &dEnd));
961     depthShift[2 * d]     = dEnd;
962     depthShift[2 * d + 1] = 0;
963   }
964   if (depth >= 0) depthShift[2 * depth + 1] = Ng;
965   PetscCall(DMPlexShiftPointSetUp_Internal(depth, depthShift));
966   PetscCall(DMPlexShiftSizes_Internal(dm, depthShift, gdm));
967   /* Step 3: Set cone/support sizes for new points */
968   PetscCall(DMPlexGetHeightStratum(dm, 0, NULL, &cEnd));
969   for (c = cEnd; c < cEnd + Ng; ++c) PetscCall(DMPlexSetConeSize(gdm, c, 1));
970   for (fs = 0; fs < numFS; ++fs) {
971     IS              faceIS;
972     const PetscInt *faces;
973     PetscInt        numFaces, f;
974 
975     PetscCall(DMLabelGetStratumIS(label, values[fs], &faceIS));
976     PetscCall(ISGetLocalSize(faceIS, &numFaces));
977     PetscCall(ISGetIndices(faceIS, &faces));
978     for (f = 0; f < numFaces; ++f) {
979       PetscInt size, numChildren;
980 
981       PetscCall(PetscFindInt(faces[f], nleaves, leaves, &loc));
982       PetscCall(DMPlexGetTreeChildren(dm, faces[f], &numChildren, NULL));
983       if (loc >= 0 || numChildren) continue;
984       if ((faces[f] < fStart) || (faces[f] >= fEnd)) continue;
985       PetscCall(DMPlexGetSupportSize(dm, faces[f], &size));
986       PetscCheck(size == 1, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM has boundary face %" PetscInt_FMT " with %" PetscInt_FMT " support cells", faces[f], size);
987       PetscCall(DMPlexSetSupportSize(gdm, faces[f] + Ng, 2));
988     }
989     PetscCall(ISRestoreIndices(faceIS, &faces));
990     PetscCall(ISDestroy(&faceIS));
991   }
992   /* Step 4: Setup ghosted DM */
993   PetscCall(DMSetUp(gdm));
994   PetscCall(DMPlexShiftPoints_Internal(dm, depthShift, gdm));
995   /* Step 6: Set cones and supports for new points */
996   ghostCell = cEnd;
997   for (fs = 0; fs < numFS; ++fs) {
998     IS              faceIS;
999     const PetscInt *faces;
1000     PetscInt        numFaces, f;
1001 
1002     PetscCall(DMLabelGetStratumIS(label, values[fs], &faceIS));
1003     PetscCall(ISGetLocalSize(faceIS, &numFaces));
1004     PetscCall(ISGetIndices(faceIS, &faces));
1005     for (f = 0; f < numFaces; ++f) {
1006       PetscInt newFace = faces[f] + Ng, numChildren;
1007 
1008       PetscCall(PetscFindInt(faces[f], nleaves, leaves, &loc));
1009       PetscCall(DMPlexGetTreeChildren(dm, faces[f], &numChildren, NULL));
1010       if (loc >= 0 || numChildren) continue;
1011       if ((faces[f] < fStart) || (faces[f] >= fEnd)) continue;
1012       PetscCall(DMPlexSetCone(gdm, ghostCell, &newFace));
1013       PetscCall(DMPlexInsertSupport(gdm, newFace, 1, ghostCell));
1014       ++ghostCell;
1015     }
1016     PetscCall(ISRestoreIndices(faceIS, &faces));
1017     PetscCall(ISDestroy(&faceIS));
1018   }
1019   PetscCall(ISRestoreIndices(valueIS, &values));
1020   PetscCall(ISDestroy(&valueIS));
1021   PetscCall(DMPlexShiftCoordinates_Internal(dm, depthShift, gdm));
1022   PetscCall(DMPlexShiftSF_Internal(dm, depthShift, gdm));
1023   PetscCall(DMPlexShiftLabels_Internal(dm, depthShift, gdm));
1024   PetscCall(DMPlexCreateVTKLabel_Internal(dm, PETSC_TRUE, gdm));
1025   PetscCall(DMPlexShiftTree_Internal(dm, depthShift, gdm));
1026   PetscCall(PetscFree(depthShift));
1027   for (c = cEnd; c < cEnd + Ng; ++c) PetscCall(DMPlexSetCellType(gdm, c, DM_POLYTOPE_FV_GHOST));
1028   /* Step 7: Periodicity */
1029   PetscCall(DMGetPeriodicity(dm, &maxCell, &Lstart, &L));
1030   PetscCall(DMSetPeriodicity(gdm, maxCell, Lstart, L));
1031   if (numGhostCells) *numGhostCells = Ng;
1032   PetscFunctionReturn(PETSC_SUCCESS);
1033 }
1034 
1035 /*@C
1036   DMPlexConstructGhostCells - Construct ghost cells which connect to every boundary face
1037 
1038   Collective
1039 
1040   Input Parameters:
1041 + dm        - The original `DM`
1042 - labelName - The label specifying the boundary faces, or "Face Sets" if this is `NULL`
1043 
1044   Output Parameters:
1045 + numGhostCells - The number of ghost cells added to the `DM`
1046 - dmGhosted     - The new `DM`
1047 
1048   Level: developer
1049 
1050   Note:
1051   If no label exists of that name, one will be created marking all boundary faces
1052 
1053 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMCreate()`
1054 @*/
1055 PetscErrorCode DMPlexConstructGhostCells(DM dm, const char labelName[], PetscInt *numGhostCells, DM *dmGhosted)
1056 {
1057   DM          gdm;
1058   DMLabel     label;
1059   const char *name = labelName ? labelName : "Face Sets";
1060   PetscInt    dim, Ng = 0;
1061   PetscBool   useCone, useClosure;
1062 
1063   PetscFunctionBegin;
1064   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1065   if (numGhostCells) PetscAssertPointer(numGhostCells, 3);
1066   PetscAssertPointer(dmGhosted, 4);
1067   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &gdm));
1068   PetscCall(DMSetType(gdm, DMPLEX));
1069   PetscCall(DMGetDimension(dm, &dim));
1070   PetscCall(DMSetDimension(gdm, dim));
1071   PetscCall(DMGetBasicAdjacency(dm, &useCone, &useClosure));
1072   PetscCall(DMSetBasicAdjacency(gdm, useCone, useClosure));
1073   PetscCall(DMGetLabel(dm, name, &label));
1074   if (!label) {
1075     /* Get label for boundary faces */
1076     PetscCall(DMCreateLabel(dm, name));
1077     PetscCall(DMGetLabel(dm, name, &label));
1078     PetscCall(DMPlexMarkBoundaryFaces(dm, 1, label));
1079   }
1080   PetscCall(DMPlexConstructGhostCells_Internal(dm, label, &Ng, gdm));
1081   PetscCall(DMCopyDisc(dm, gdm));
1082   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_TRUE, gdm));
1083   gdm->setfromoptionscalled = dm->setfromoptionscalled;
1084   if (numGhostCells) *numGhostCells = Ng;
1085   *dmGhosted = gdm;
1086   PetscFunctionReturn(PETSC_SUCCESS);
1087 }
1088 
1089 static PetscErrorCode DivideCells_Private(DM dm, DMLabel label, DMPlexPointQueue queue)
1090 {
1091   PetscInt dim, d, shift = 100, *pStart, *pEnd;
1092 
1093   PetscFunctionBegin;
1094   PetscCall(DMGetDimension(dm, &dim));
1095   PetscCall(PetscMalloc2(dim, &pStart, dim, &pEnd));
1096   for (d = 0; d < dim; ++d) PetscCall(DMPlexGetDepthStratum(dm, d, &pStart[d], &pEnd[d]));
1097   while (!DMPlexPointQueueEmpty(queue)) {
1098     PetscInt  cell    = -1;
1099     PetscInt *closure = NULL;
1100     PetscInt  closureSize, cl, cval;
1101 
1102     PetscCall(DMPlexPointQueueDequeue(queue, &cell));
1103     PetscCall(DMLabelGetValue(label, cell, &cval));
1104     PetscCall(DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
1105     /* Mark points in the cell closure that touch the fault */
1106     for (d = 0; d < dim; ++d) {
1107       for (cl = 0; cl < closureSize * 2; cl += 2) {
1108         const PetscInt clp = closure[cl];
1109         PetscInt       clval;
1110 
1111         if ((clp < pStart[d]) || (clp >= pEnd[d])) continue;
1112         PetscCall(DMLabelGetValue(label, clp, &clval));
1113         if (clval == -1) {
1114           const PetscInt *cone;
1115           PetscInt        coneSize, c;
1116 
1117           /* If a cone point touches the fault, then this point touches the fault */
1118           PetscCall(DMPlexGetCone(dm, clp, &cone));
1119           PetscCall(DMPlexGetConeSize(dm, clp, &coneSize));
1120           for (c = 0; c < coneSize; ++c) {
1121             PetscInt cpval;
1122 
1123             PetscCall(DMLabelGetValue(label, cone[c], &cpval));
1124             if (cpval != -1) {
1125               PetscInt dep;
1126 
1127               PetscCall(DMPlexGetPointDepth(dm, clp, &dep));
1128               clval = cval < 0 ? -(shift + dep) : shift + dep;
1129               PetscCall(DMLabelSetValue(label, clp, clval));
1130               break;
1131             }
1132           }
1133         }
1134         /* Mark neighbor cells through marked faces (these cells must also touch the fault) */
1135         if (d == dim - 1 && clval != -1) {
1136           const PetscInt *support;
1137           PetscInt        supportSize, s, nval;
1138 
1139           PetscCall(DMPlexGetSupport(dm, clp, &support));
1140           PetscCall(DMPlexGetSupportSize(dm, clp, &supportSize));
1141           for (s = 0; s < supportSize; ++s) {
1142             PetscCall(DMLabelGetValue(label, support[s], &nval));
1143             if (nval == -1) {
1144               PetscCall(DMLabelSetValue(label, support[s], clval < 0 ? clval - 1 : clval + 1));
1145               PetscCall(DMPlexPointQueueEnqueue(queue, support[s]));
1146             }
1147           }
1148         }
1149       }
1150     }
1151     PetscCall(DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
1152   }
1153   PetscCall(PetscFree2(pStart, pEnd));
1154   PetscFunctionReturn(PETSC_SUCCESS);
1155 }
1156 
1157 typedef struct {
1158   DM               dm;
1159   DMPlexPointQueue queue;
1160 } PointDivision;
1161 
1162 static PetscErrorCode divideCell(DMLabel label, PetscInt p, PetscInt val, void *ctx)
1163 {
1164   PointDivision  *div  = (PointDivision *)ctx;
1165   PetscInt        cval = val < 0 ? val - 1 : val + 1;
1166   const PetscInt *support;
1167   PetscInt        supportSize, s;
1168 
1169   PetscFunctionBegin;
1170   PetscCall(DMPlexGetSupport(div->dm, p, &support));
1171   PetscCall(DMPlexGetSupportSize(div->dm, p, &supportSize));
1172   for (s = 0; s < supportSize; ++s) {
1173     PetscCall(DMLabelSetValue(label, support[s], cval));
1174     PetscCall(DMPlexPointQueueEnqueue(div->queue, support[s]));
1175   }
1176   PetscFunctionReturn(PETSC_SUCCESS);
1177 }
1178 
1179 /* Mark cells by label propagation */
1180 static PetscErrorCode DMPlexLabelFaultHalo(DM dm, DMLabel faultLabel)
1181 {
1182   DMPlexPointQueue queue = NULL;
1183   PointDivision    div;
1184   PetscSF          pointSF;
1185   IS               pointIS;
1186   const PetscInt  *points;
1187   PetscBool        empty;
1188   PetscInt         dim, shift = 100, n, i;
1189 
1190   PetscFunctionBegin;
1191   PetscCall(DMGetDimension(dm, &dim));
1192   PetscCall(DMPlexPointQueueCreate(1024, &queue));
1193   div.dm    = dm;
1194   div.queue = queue;
1195   /* Enqueue cells on fault */
1196   PetscCall(DMLabelGetStratumIS(faultLabel, shift + dim, &pointIS));
1197   if (pointIS) {
1198     PetscCall(ISGetLocalSize(pointIS, &n));
1199     PetscCall(ISGetIndices(pointIS, &points));
1200     for (i = 0; i < n; ++i) PetscCall(DMPlexPointQueueEnqueue(queue, points[i]));
1201     PetscCall(ISRestoreIndices(pointIS, &points));
1202     PetscCall(ISDestroy(&pointIS));
1203   }
1204   PetscCall(DMLabelGetStratumIS(faultLabel, -(shift + dim), &pointIS));
1205   if (pointIS) {
1206     PetscCall(ISGetLocalSize(pointIS, &n));
1207     PetscCall(ISGetIndices(pointIS, &points));
1208     for (i = 0; i < n; ++i) PetscCall(DMPlexPointQueueEnqueue(queue, points[i]));
1209     PetscCall(ISRestoreIndices(pointIS, &points));
1210     PetscCall(ISDestroy(&pointIS));
1211   }
1212 
1213   PetscCall(DMGetPointSF(dm, &pointSF));
1214   PetscCall(DMLabelPropagateBegin(faultLabel, pointSF));
1215   /* While edge queue is not empty: */
1216   PetscCall(DMPlexPointQueueEmptyCollective((PetscObject)dm, queue, &empty));
1217   while (!empty) {
1218     PetscCall(DivideCells_Private(dm, faultLabel, queue));
1219     PetscCall(DMLabelPropagatePush(faultLabel, pointSF, divideCell, &div));
1220     PetscCall(DMPlexPointQueueEmptyCollective((PetscObject)dm, queue, &empty));
1221   }
1222   PetscCall(DMLabelPropagateEnd(faultLabel, pointSF));
1223   PetscCall(DMPlexPointQueueDestroy(&queue));
1224   PetscFunctionReturn(PETSC_SUCCESS);
1225 }
1226 
1227 /*
1228   We are adding three kinds of points here:
1229     Replicated:     Copies of points which exist in the mesh, such as vertices identified across a fault
1230     Non-replicated: Points which exist on the fault, but are not replicated
1231     Ghost:          These are shared fault faces which are not owned by this process. These do not produce hybrid cells and do not replicate
1232     Hybrid:         Entirely new points, such as cohesive cells
1233 
1234   When creating subsequent cohesive cells, we shift the old hybrid cells to the end of the numbering at
1235   each depth so that the new split/hybrid points can be inserted as a block.
1236 */
1237 static PetscErrorCode DMPlexConstructCohesiveCells_Internal(DM dm, DMLabel label, DMLabel splitLabel, DM sdm)
1238 {
1239   MPI_Comm         comm;
1240   IS               valueIS;
1241   PetscInt         numSP = 0; /* The number of depths for which we have replicated points */
1242   const PetscInt  *values;    /* List of depths for which we have replicated points */
1243   IS              *splitIS;
1244   IS              *unsplitIS;
1245   IS               ghostIS;
1246   PetscInt        *numSplitPoints;     /* The number of replicated points at each depth */
1247   PetscInt        *numUnsplitPoints;   /* The number of non-replicated points at each depth which still give rise to hybrid points */
1248   PetscInt        *numHybridPoints;    /* The number of new hybrid points at each depth */
1249   PetscInt        *numHybridPointsOld; /* The number of existing hybrid points at each depth */
1250   PetscInt         numGhostPoints;     /* The number of unowned, shared fault faces */
1251   const PetscInt **splitPoints;        /* Replicated points for each depth */
1252   const PetscInt **unsplitPoints;      /* Non-replicated points for each depth */
1253   const PetscInt  *ghostPoints;        /* Ghost fault faces */
1254   PetscSection     coordSection;
1255   Vec              coordinates;
1256   PetscScalar     *coords;
1257   PetscInt        *depthMax;   /* The first hybrid point at each depth in the original mesh */
1258   PetscInt        *depthEnd;   /* The point limit at each depth in the original mesh */
1259   PetscInt        *depthShift; /* Number of replicated+hybrid points at each depth */
1260   PetscInt        *pMaxNew;    /* The first replicated point at each depth in the new mesh, hybrids come after this */
1261   PetscInt        *coneNew, *coneONew, *supportNew;
1262   PetscInt         shift = 100, shift2 = 200, depth = 0, dep, dim, d, sp, maxConeSize, maxSupportSize, maxConeSizeNew, maxSupportSizeNew, numLabels, vStart, vEnd, pEnd, p, v;
1263 
1264   PetscFunctionBegin;
1265   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
1266   PetscCall(DMGetDimension(dm, &dim));
1267   PetscCall(DMPlexGetDepth(dm, &depth));
1268   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
1269   /* We do not want this label automatically computed, instead we compute it here */
1270   PetscCall(DMCreateLabel(sdm, "celltype"));
1271   /* Count split points and add cohesive cells */
1272   PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize));
1273   PetscCall(PetscMalloc5(depth + 1, &depthMax, depth + 1, &depthEnd, 2 * (depth + 1), &depthShift, depth + 1, &pMaxNew, depth + 1, &numHybridPointsOld));
1274   PetscCall(PetscMalloc7(depth + 1, &splitIS, depth + 1, &unsplitIS, depth + 1, &numSplitPoints, depth + 1, &numUnsplitPoints, depth + 1, &numHybridPoints, depth + 1, &splitPoints, depth + 1, &unsplitPoints));
1275   for (d = 0; d <= depth; ++d) {
1276     PetscCall(DMPlexGetDepthStratum(dm, d, NULL, &pMaxNew[d]));
1277     PetscCall(DMPlexGetTensorPrismBounds_Internal(dm, d, &depthMax[d], NULL));
1278     depthEnd[d]           = pMaxNew[d];
1279     depthMax[d]           = depthMax[d] < 0 ? depthEnd[d] : depthMax[d];
1280     numSplitPoints[d]     = 0;
1281     numUnsplitPoints[d]   = 0;
1282     numHybridPoints[d]    = 0;
1283     numHybridPointsOld[d] = depthMax[d] < 0 ? 0 : depthEnd[d] - depthMax[d];
1284     splitPoints[d]        = NULL;
1285     unsplitPoints[d]      = NULL;
1286     splitIS[d]            = NULL;
1287     unsplitIS[d]          = NULL;
1288     /* we are shifting the existing hybrid points with the stratum behind them, so
1289      * the split comes at the end of the normal points, i.e., at depthMax[d] */
1290     depthShift[2 * d]     = depthMax[d];
1291     depthShift[2 * d + 1] = 0;
1292   }
1293   numGhostPoints = 0;
1294   ghostPoints    = NULL;
1295   if (label) {
1296     PetscCall(DMLabelGetValueIS(label, &valueIS));
1297     PetscCall(ISGetLocalSize(valueIS, &numSP));
1298     PetscCall(ISGetIndices(valueIS, &values));
1299   }
1300   for (sp = 0; sp < numSP; ++sp) {
1301     const PetscInt dep = values[sp];
1302 
1303     if ((dep < 0) || (dep > depth)) continue;
1304     PetscCall(DMLabelGetStratumIS(label, dep, &splitIS[dep]));
1305     if (splitIS[dep]) {
1306       PetscCall(ISGetLocalSize(splitIS[dep], &numSplitPoints[dep]));
1307       PetscCall(ISGetIndices(splitIS[dep], &splitPoints[dep]));
1308     }
1309     PetscCall(DMLabelGetStratumIS(label, shift2 + dep, &unsplitIS[dep]));
1310     if (unsplitIS[dep]) {
1311       PetscCall(ISGetLocalSize(unsplitIS[dep], &numUnsplitPoints[dep]));
1312       PetscCall(ISGetIndices(unsplitIS[dep], &unsplitPoints[dep]));
1313     }
1314   }
1315   PetscCall(DMLabelGetStratumIS(label, shift2 + dim - 1, &ghostIS));
1316   if (ghostIS) {
1317     PetscCall(ISGetLocalSize(ghostIS, &numGhostPoints));
1318     PetscCall(ISGetIndices(ghostIS, &ghostPoints));
1319   }
1320   /* Calculate number of hybrid points */
1321   for (d = 1; d <= depth; ++d) numHybridPoints[d] = numSplitPoints[d - 1] + numUnsplitPoints[d - 1]; /* There is a hybrid cell/face/edge for every split face/edge/vertex   */
1322   for (d = 0; d <= depth; ++d) depthShift[2 * d + 1] = numSplitPoints[d] + numHybridPoints[d];
1323   PetscCall(DMPlexShiftPointSetUp_Internal(depth, depthShift));
1324   /* the end of the points in this stratum that come before the new points:
1325    * shifting pMaxNew[d] gets the new start of the next stratum, then count back the old hybrid points and the newly
1326    * added points */
1327   for (d = 0; d <= depth; ++d) pMaxNew[d] = DMPlexShiftPoint_Internal(pMaxNew[d], depth, depthShift) - (numHybridPointsOld[d] + numSplitPoints[d] + numHybridPoints[d]);
1328   PetscCall(DMPlexShiftSizes_Internal(dm, depthShift, sdm));
1329   /* Step 3: Set cone/support sizes for new points */
1330   for (dep = 0; dep <= depth; ++dep) {
1331     for (p = 0; p < numSplitPoints[dep]; ++p) {
1332       const PetscInt  oldp   = splitPoints[dep][p];
1333       const PetscInt  newp   = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*oldp + depthOffset[dep]*/;
1334       const PetscInt  splitp = p + pMaxNew[dep];
1335       const PetscInt *support;
1336       DMPolytopeType  ct;
1337       PetscInt        coneSize, supportSize, qf, qn, qp, e;
1338 
1339       PetscCall(DMPlexGetConeSize(dm, oldp, &coneSize));
1340       PetscCall(DMPlexSetConeSize(sdm, splitp, coneSize));
1341       PetscCall(DMPlexGetSupportSize(dm, oldp, &supportSize));
1342       PetscCall(DMPlexSetSupportSize(sdm, splitp, supportSize));
1343       PetscCall(DMPlexGetCellType(dm, oldp, &ct));
1344       PetscCall(DMPlexSetCellType(sdm, splitp, ct));
1345       if (dep == depth - 1) {
1346         const PetscInt hybcell = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1];
1347 
1348         /* Add cohesive cells, they are prisms */
1349         PetscCall(DMPlexSetConeSize(sdm, hybcell, 2 + coneSize));
1350         switch (coneSize) {
1351         case 2:
1352           PetscCall(DMPlexSetCellType(sdm, hybcell, DM_POLYTOPE_SEG_PRISM_TENSOR));
1353           break;
1354         case 3:
1355           PetscCall(DMPlexSetCellType(sdm, hybcell, DM_POLYTOPE_TRI_PRISM_TENSOR));
1356           break;
1357         case 4:
1358           PetscCall(DMPlexSetCellType(sdm, hybcell, DM_POLYTOPE_QUAD_PRISM_TENSOR));
1359           break;
1360         }
1361         /* Shared fault faces with only one support cell now have two with the cohesive cell */
1362         /*   TODO Check thaat oldp has rootdegree == 1 */
1363         if (supportSize == 1) {
1364           const PetscInt *support;
1365           PetscInt        val;
1366 
1367           PetscCall(DMPlexGetSupport(dm, oldp, &support));
1368           PetscCall(DMLabelGetValue(label, support[0], &val));
1369           if (val < 0) PetscCall(DMPlexSetSupportSize(sdm, splitp, 2));
1370           else PetscCall(DMPlexSetSupportSize(sdm, newp, 2));
1371         }
1372       } else if (dep == 0) {
1373         const PetscInt hybedge = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1];
1374 
1375         PetscCall(DMPlexGetSupport(dm, oldp, &support));
1376         for (e = 0, qn = 0, qp = 0, qf = 0; e < supportSize; ++e) {
1377           PetscInt val;
1378 
1379           PetscCall(DMLabelGetValue(label, support[e], &val));
1380           if (val == 1) ++qf;
1381           if ((val == 1) || (val == (shift + 1))) ++qn;
1382           if ((val == 1) || (val == -(shift + 1))) ++qp;
1383         }
1384         /* Split old vertex: Edges into original vertex and new cohesive edge */
1385         PetscCall(DMPlexSetSupportSize(sdm, newp, qn + 1));
1386         /* Split new vertex: Edges into split vertex and new cohesive edge */
1387         PetscCall(DMPlexSetSupportSize(sdm, splitp, qp + 1));
1388         /* Add hybrid edge */
1389         PetscCall(DMPlexSetConeSize(sdm, hybedge, 2));
1390         PetscCall(DMPlexSetSupportSize(sdm, hybedge, qf));
1391         PetscCall(DMPlexSetCellType(sdm, hybedge, DM_POLYTOPE_POINT_PRISM_TENSOR));
1392       } else if (dep == dim - 2) {
1393         const PetscInt hybface = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1];
1394 
1395         PetscCall(DMPlexGetSupport(dm, oldp, &support));
1396         for (e = 0, qn = 0, qp = 0, qf = 0; e < supportSize; ++e) {
1397           PetscInt val;
1398 
1399           PetscCall(DMLabelGetValue(label, support[e], &val));
1400           if (val == dim - 1) ++qf;
1401           if ((val == dim - 1) || (val == (shift + dim - 1))) ++qn;
1402           if ((val == dim - 1) || (val == -(shift + dim - 1))) ++qp;
1403         }
1404         /* Split old edge: Faces into original edge and cohesive face (positive side?) */
1405         PetscCall(DMPlexSetSupportSize(sdm, newp, qn + 1));
1406         /* Split new edge: Faces into split edge and cohesive face (negative side?) */
1407         PetscCall(DMPlexSetSupportSize(sdm, splitp, qp + 1));
1408         /* Add hybrid face */
1409         PetscCall(DMPlexSetConeSize(sdm, hybface, 4));
1410         PetscCall(DMPlexSetSupportSize(sdm, hybface, qf));
1411         PetscCall(DMPlexSetCellType(sdm, hybface, DM_POLYTOPE_SEG_PRISM_TENSOR));
1412       }
1413     }
1414   }
1415   for (dep = 0; dep <= depth; ++dep) {
1416     for (p = 0; p < numUnsplitPoints[dep]; ++p) {
1417       const PetscInt  oldp = unsplitPoints[dep][p];
1418       const PetscInt  newp = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*oldp + depthOffset[dep]*/;
1419       const PetscInt *support;
1420       PetscInt        coneSize, supportSize, qf, e, s;
1421 
1422       PetscCall(DMPlexGetConeSize(dm, oldp, &coneSize));
1423       PetscCall(DMPlexGetSupportSize(dm, oldp, &supportSize));
1424       PetscCall(DMPlexGetSupport(dm, oldp, &support));
1425       if (dep == 0) {
1426         const PetscInt hybedge = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1] + numSplitPoints[dep];
1427 
1428         /* Unsplit vertex: Edges into original vertex, split edges, and new cohesive edge twice */
1429         for (s = 0, qf = 0; s < supportSize; ++s, ++qf) {
1430           PetscCall(PetscFindInt(support[s], numSplitPoints[dep + 1], splitPoints[dep + 1], &e));
1431           if (e >= 0) ++qf;
1432         }
1433         PetscCall(DMPlexSetSupportSize(sdm, newp, qf + 2));
1434         /* Add hybrid edge */
1435         PetscCall(DMPlexSetConeSize(sdm, hybedge, 2));
1436         for (e = 0, qf = 0; e < supportSize; ++e) {
1437           PetscInt val;
1438 
1439           PetscCall(DMLabelGetValue(label, support[e], &val));
1440           /* Split and unsplit edges produce hybrid faces */
1441           if (val == 1) ++qf;
1442           if (val == (shift2 + 1)) ++qf;
1443         }
1444         PetscCall(DMPlexSetSupportSize(sdm, hybedge, qf));
1445         PetscCall(DMPlexSetCellType(sdm, hybedge, DM_POLYTOPE_POINT_PRISM_TENSOR));
1446       } else if (dep == dim - 2) {
1447         const PetscInt hybface = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1] + numSplitPoints[dep];
1448         PetscInt       val;
1449 
1450         for (e = 0, qf = 0; e < supportSize; ++e) {
1451           PetscCall(DMLabelGetValue(label, support[e], &val));
1452           if (val == dim - 1) qf += 2;
1453           else ++qf;
1454         }
1455         /* Unsplit edge: Faces into original edge, split face, and cohesive face twice */
1456         PetscCall(DMPlexSetSupportSize(sdm, newp, qf + 2));
1457         /* Add hybrid face */
1458         for (e = 0, qf = 0; e < supportSize; ++e) {
1459           PetscCall(DMLabelGetValue(label, support[e], &val));
1460           if (val == dim - 1) ++qf;
1461         }
1462         PetscCall(DMPlexSetConeSize(sdm, hybface, 4));
1463         PetscCall(DMPlexSetSupportSize(sdm, hybface, qf));
1464         PetscCall(DMPlexSetCellType(sdm, hybface, DM_POLYTOPE_SEG_PRISM_TENSOR));
1465       }
1466     }
1467   }
1468   /* Step 4: Setup split DM */
1469   PetscCall(DMSetUp(sdm));
1470   PetscCall(DMPlexShiftPoints_Internal(dm, depthShift, sdm));
1471   PetscCall(DMPlexGetMaxSizes(sdm, &maxConeSizeNew, &maxSupportSizeNew));
1472   PetscCall(PetscMalloc3(PetscMax(maxConeSize, maxConeSizeNew) * 3, &coneNew, PetscMax(maxConeSize, maxConeSizeNew) * 3, &coneONew, PetscMax(maxSupportSize, maxSupportSizeNew), &supportNew));
1473   /* Step 6: Set cones and supports for new points */
1474   for (dep = 0; dep <= depth; ++dep) {
1475     for (p = 0; p < numSplitPoints[dep]; ++p) {
1476       const PetscInt  oldp   = splitPoints[dep][p];
1477       const PetscInt  newp   = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*oldp + depthOffset[dep]*/;
1478       const PetscInt  splitp = p + pMaxNew[dep];
1479       const PetscInt *cone, *support, *ornt;
1480       DMPolytopeType  ct;
1481       PetscInt        coneSize, supportSize, q, qf, qn, qp, v, e, s;
1482 
1483       PetscCall(DMPlexGetCellType(dm, oldp, &ct));
1484       PetscCall(DMPlexGetConeSize(dm, oldp, &coneSize));
1485       PetscCall(DMPlexGetCone(dm, oldp, &cone));
1486       PetscCall(DMPlexGetConeOrientation(dm, oldp, &ornt));
1487       PetscCall(DMPlexGetSupportSize(dm, oldp, &supportSize));
1488       PetscCall(DMPlexGetSupport(dm, oldp, &support));
1489       if (dep == depth - 1) {
1490         PetscBool       hasUnsplit = PETSC_FALSE;
1491         const PetscInt  hybcell    = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1];
1492         const PetscInt *supportF;
1493 
1494         coneONew[0] = coneONew[1] = -1000;
1495         /* Split face:       copy in old face to new face to start */
1496         PetscCall(DMPlexGetSupport(sdm, newp, &supportF));
1497         PetscCall(DMPlexSetSupport(sdm, splitp, supportF));
1498         /* Split old face:   old vertices/edges in cone so no change */
1499         /* Split new face:   new vertices/edges in cone */
1500         for (q = 0; q < coneSize; ++q) {
1501           PetscCall(PetscFindInt(cone[q], numSplitPoints[dep - 1], splitPoints[dep - 1], &v));
1502           if (v < 0) {
1503             PetscCall(PetscFindInt(cone[q], numUnsplitPoints[dep - 1], unsplitPoints[dep - 1], &v));
1504             PetscCheck(v >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %" PetscInt_FMT " in split or unsplit points of depth %" PetscInt_FMT, cone[q], dep - 1);
1505             coneNew[2 + q] = DMPlexShiftPoint_Internal(cone[q], depth, depthShift) /*cone[q] + depthOffset[dep-1]*/;
1506             hasUnsplit     = PETSC_TRUE;
1507           } else {
1508             coneNew[2 + q] = v + pMaxNew[dep - 1];
1509             if (dep > 1) {
1510               const PetscInt *econe;
1511               PetscInt        econeSize, r, vs, vu;
1512 
1513               PetscCall(DMPlexGetConeSize(dm, cone[q], &econeSize));
1514               PetscCall(DMPlexGetCone(dm, cone[q], &econe));
1515               for (r = 0; r < econeSize; ++r) {
1516                 PetscCall(PetscFindInt(econe[r], numSplitPoints[dep - 2], splitPoints[dep - 2], &vs));
1517                 PetscCall(PetscFindInt(econe[r], numUnsplitPoints[dep - 2], unsplitPoints[dep - 2], &vu));
1518                 if (vs >= 0) continue;
1519                 PetscCheck(vu >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %" PetscInt_FMT " in split or unsplit points of depth %" PetscInt_FMT, econe[r], dep - 2);
1520                 hasUnsplit = PETSC_TRUE;
1521               }
1522             }
1523           }
1524         }
1525         PetscCall(DMPlexSetCone(sdm, splitp, &coneNew[2]));
1526         PetscCall(DMPlexSetConeOrientation(sdm, splitp, ornt));
1527         /* Face support */
1528         PetscInt vals[2];
1529 
1530         PetscCall(DMLabelGetValue(label, support[0], &vals[0]));
1531         if (supportSize > 1) PetscCall(DMLabelGetValue(label, support[1], &vals[1]));
1532         else vals[1] = -vals[0];
1533         PetscCheck(vals[0] * vals[1] < 0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid support labels %" PetscInt_FMT " %" PetscInt_FMT, vals[0], vals[1]);
1534 
1535         for (s = 0; s < 2; ++s) {
1536           if (s >= supportSize) {
1537             if (vals[s] < 0) {
1538               /* Ghost old face:   Replace negative side cell with cohesive cell */
1539               PetscCall(DMPlexInsertSupport(sdm, newp, s, hybcell));
1540             } else {
1541               /* Ghost new face:   Replace positive side cell with cohesive cell */
1542               PetscCall(DMPlexInsertSupport(sdm, splitp, s, hybcell));
1543             }
1544           } else {
1545             if (vals[s] < 0) {
1546               /* Split old face:   Replace negative side cell with cohesive cell */
1547               PetscCall(DMPlexInsertSupport(sdm, newp, s, hybcell));
1548             } else {
1549               /* Split new face:   Replace positive side cell with cohesive cell */
1550               PetscCall(DMPlexInsertSupport(sdm, splitp, s, hybcell));
1551             }
1552           }
1553         }
1554         /* Get orientation for cohesive face using the positive side cell */
1555         {
1556           const PetscInt *ncone, *nconeO;
1557           PetscInt        nconeSize, nc, ocell;
1558           PetscBool       flip = PETSC_FALSE;
1559 
1560           if (supportSize > 1) {
1561             ocell = vals[0] < 0 ? support[1] : support[0];
1562           } else {
1563             ocell = support[0];
1564             flip  = vals[0] < 0 ? PETSC_TRUE : PETSC_FALSE;
1565           }
1566           PetscCall(DMPlexGetConeSize(dm, ocell, &nconeSize));
1567           PetscCall(DMPlexGetCone(dm, ocell, &ncone));
1568           PetscCall(DMPlexGetConeOrientation(dm, ocell, &nconeO));
1569           for (nc = 0; nc < nconeSize; ++nc) {
1570             if (ncone[nc] == oldp) {
1571               coneONew[0] = flip ? -(nconeO[nc] + 1) : nconeO[nc];
1572               break;
1573             }
1574           }
1575           PetscCheck(nc < nconeSize, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate face %" PetscInt_FMT " in neighboring cell %" PetscInt_FMT, oldp, ocell);
1576         }
1577         /* Cohesive cell:    Old and new split face, then new cohesive faces */
1578         {
1579           const PetscInt No = DMPolytopeTypeGetNumArrangements(ct) / 2;
1580           PetscCheck((coneONew[0] >= -No) && (coneONew[0] < No), PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid %s orientation %" PetscInt_FMT, DMPolytopeTypes[ct], coneONew[0]);
1581         }
1582         const PetscInt *arr = DMPolytopeTypeGetArrangement(ct, coneONew[0]);
1583 
1584         coneNew[0]  = newp; /* Extracted negative side orientation above */
1585         coneNew[1]  = splitp;
1586         coneONew[1] = coneONew[0];
1587         for (q = 0; q < coneSize; ++q) {
1588           /* Hybrid faces must follow order from oriented end face */
1589           const PetscInt qa = arr[q * 2 + 0];
1590           const PetscInt qo = arr[q * 2 + 1];
1591           DMPolytopeType ft = dep == 2 ? DM_POLYTOPE_SEGMENT : DM_POLYTOPE_POINT;
1592 
1593           PetscCall(PetscFindInt(cone[qa], numSplitPoints[dep - 1], splitPoints[dep - 1], &v));
1594           if (v < 0) {
1595             PetscCall(PetscFindInt(cone[qa], numUnsplitPoints[dep - 1], unsplitPoints[dep - 1], &v));
1596             coneNew[2 + q] = v + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep - 1];
1597           } else {
1598             coneNew[2 + q] = v + pMaxNew[dep] + numSplitPoints[dep];
1599           }
1600           coneONew[2 + q] = DMPolytopeTypeComposeOrientation(ft, qo, ornt[qa]);
1601         }
1602         PetscCall(DMPlexSetCone(sdm, hybcell, coneNew));
1603         PetscCall(DMPlexSetConeOrientation(sdm, hybcell, coneONew));
1604         /* Label the hybrid cells on the boundary of the split */
1605         if (hasUnsplit) PetscCall(DMLabelSetValue(label, -hybcell, dim));
1606       } else if (dep == 0) {
1607         const PetscInt hybedge = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1];
1608 
1609         /* Split old vertex: Edges in old split faces and new cohesive edge */
1610         for (e = 0, qn = 0; e < supportSize; ++e) {
1611           PetscInt val;
1612 
1613           PetscCall(DMLabelGetValue(label, support[e], &val));
1614           if ((val == 1) || (val == (shift + 1))) supportNew[qn++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1615         }
1616         supportNew[qn] = hybedge;
1617         PetscCall(DMPlexSetSupport(sdm, newp, supportNew));
1618         /* Split new vertex: Edges in new split faces and new cohesive edge */
1619         for (e = 0, qp = 0; e < supportSize; ++e) {
1620           PetscInt val, edge;
1621 
1622           PetscCall(DMLabelGetValue(label, support[e], &val));
1623           if (val == 1) {
1624             PetscCall(PetscFindInt(support[e], numSplitPoints[dep + 1], splitPoints[dep + 1], &edge));
1625             PetscCheck(edge >= 0, comm, PETSC_ERR_ARG_WRONG, "Edge %" PetscInt_FMT " is not a split edge", support[e]);
1626             supportNew[qp++] = edge + pMaxNew[dep + 1];
1627           } else if (val == -(shift + 1)) {
1628             supportNew[qp++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1629           }
1630         }
1631         supportNew[qp] = hybedge;
1632         PetscCall(DMPlexSetSupport(sdm, splitp, supportNew));
1633         /* Hybrid edge:    Old and new split vertex */
1634         coneNew[0] = newp;
1635         coneNew[1] = splitp;
1636         PetscCall(DMPlexSetCone(sdm, hybedge, coneNew));
1637         for (e = 0, qf = 0; e < supportSize; ++e) {
1638           PetscInt val, edge;
1639 
1640           PetscCall(DMLabelGetValue(label, support[e], &val));
1641           if (val == 1) {
1642             PetscCall(PetscFindInt(support[e], numSplitPoints[dep + 1], splitPoints[dep + 1], &edge));
1643             PetscCheck(edge >= 0, comm, PETSC_ERR_ARG_WRONG, "Edge %" PetscInt_FMT " is not a split edge", support[e]);
1644             supportNew[qf++] = edge + pMaxNew[dep + 2] + numSplitPoints[dep + 2];
1645           }
1646         }
1647         PetscCall(DMPlexSetSupport(sdm, hybedge, supportNew));
1648       } else if (dep == dim - 2) {
1649         const PetscInt hybface = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1];
1650 
1651         /* Split old edge:   old vertices in cone so no change */
1652         /* Split new edge:   new vertices in cone */
1653         for (q = 0; q < coneSize; ++q) {
1654           PetscCall(PetscFindInt(cone[q], numSplitPoints[dep - 1], splitPoints[dep - 1], &v));
1655           if (v < 0) {
1656             PetscCall(PetscFindInt(cone[q], numUnsplitPoints[dep - 1], unsplitPoints[dep - 1], &v));
1657             PetscCheck(v >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %" PetscInt_FMT " in split or unsplit points of depth %" PetscInt_FMT, cone[q], dep - 1);
1658             coneNew[q] = DMPlexShiftPoint_Internal(cone[q], depth, depthShift) /*cone[q] + depthOffset[dep-1]*/;
1659           } else {
1660             coneNew[q] = v + pMaxNew[dep - 1];
1661           }
1662         }
1663         PetscCall(DMPlexSetCone(sdm, splitp, coneNew));
1664         /* Split old edge: Faces in positive side cells and old split faces */
1665         for (e = 0, q = 0; e < supportSize; ++e) {
1666           PetscInt val;
1667 
1668           PetscCall(DMLabelGetValue(label, support[e], &val));
1669           if (val == dim - 1) {
1670             supportNew[q++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1671           } else if (val == (shift + dim - 1)) {
1672             supportNew[q++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1673           }
1674         }
1675         supportNew[q++] = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1];
1676         PetscCall(DMPlexSetSupport(sdm, newp, supportNew));
1677         /* Split new edge: Faces in negative side cells and new split faces */
1678         for (e = 0, q = 0; e < supportSize; ++e) {
1679           PetscInt val, face;
1680 
1681           PetscCall(DMLabelGetValue(label, support[e], &val));
1682           if (val == dim - 1) {
1683             PetscCall(PetscFindInt(support[e], numSplitPoints[dep + 1], splitPoints[dep + 1], &face));
1684             PetscCheck(face >= 0, comm, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " is not a split face", support[e]);
1685             supportNew[q++] = face + pMaxNew[dep + 1];
1686           } else if (val == -(shift + dim - 1)) {
1687             supportNew[q++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1688           }
1689         }
1690         supportNew[q++] = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1];
1691         PetscCall(DMPlexSetSupport(sdm, splitp, supportNew));
1692         /* Hybrid face */
1693         coneNew[0] = newp;
1694         coneNew[1] = splitp;
1695         for (v = 0; v < coneSize; ++v) {
1696           PetscInt vertex;
1697           PetscCall(PetscFindInt(cone[v], numSplitPoints[dep - 1], splitPoints[dep - 1], &vertex));
1698           if (vertex < 0) {
1699             PetscCall(PetscFindInt(cone[v], numUnsplitPoints[dep - 1], unsplitPoints[dep - 1], &vertex));
1700             PetscCheck(vertex >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %" PetscInt_FMT " in split or unsplit points of depth %" PetscInt_FMT, cone[v], dep - 1);
1701             coneNew[2 + v] = vertex + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep - 1];
1702           } else {
1703             coneNew[2 + v] = vertex + pMaxNew[dep] + numSplitPoints[dep];
1704           }
1705         }
1706         PetscCall(DMPlexSetCone(sdm, hybface, coneNew));
1707         for (e = 0, qf = 0; e < supportSize; ++e) {
1708           PetscInt val, face;
1709 
1710           PetscCall(DMLabelGetValue(label, support[e], &val));
1711           if (val == dim - 1) {
1712             PetscCall(PetscFindInt(support[e], numSplitPoints[dep + 1], splitPoints[dep + 1], &face));
1713             PetscCheck(face >= 0, comm, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " is not a split face", support[e]);
1714             supportNew[qf++] = face + pMaxNew[dep + 2] + numSplitPoints[dep + 2];
1715           }
1716         }
1717         PetscCall(DMPlexSetSupport(sdm, hybface, supportNew));
1718       }
1719     }
1720   }
1721   for (dep = 0; dep <= depth; ++dep) {
1722     for (p = 0; p < numUnsplitPoints[dep]; ++p) {
1723       const PetscInt  oldp = unsplitPoints[dep][p];
1724       const PetscInt  newp = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*oldp + depthOffset[dep]*/;
1725       const PetscInt *cone, *support;
1726       PetscInt        coneSize, supportSize, supportSizeNew, q, qf, e, f, s;
1727 
1728       PetscCall(DMPlexGetConeSize(dm, oldp, &coneSize));
1729       PetscCall(DMPlexGetCone(dm, oldp, &cone));
1730       PetscCall(DMPlexGetSupportSize(dm, oldp, &supportSize));
1731       PetscCall(DMPlexGetSupport(dm, oldp, &support));
1732       if (dep == 0) {
1733         const PetscInt hybedge = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1] + numSplitPoints[dep];
1734 
1735         /* Unsplit vertex */
1736         PetscCall(DMPlexGetSupportSize(sdm, newp, &supportSizeNew));
1737         for (s = 0, q = 0; s < supportSize; ++s) {
1738           supportNew[q++] = DMPlexShiftPoint_Internal(support[s], depth, depthShift) /*support[s] + depthOffset[dep+1]*/;
1739           PetscCall(PetscFindInt(support[s], numSplitPoints[dep + 1], splitPoints[dep + 1], &e));
1740           if (e >= 0) supportNew[q++] = e + pMaxNew[dep + 1];
1741         }
1742         supportNew[q++] = hybedge;
1743         supportNew[q++] = hybedge;
1744         PetscCheck(q == supportSizeNew, comm, PETSC_ERR_ARG_WRONG, "Support size %" PetscInt_FMT " != %" PetscInt_FMT " for vertex %" PetscInt_FMT, q, supportSizeNew, newp);
1745         PetscCall(DMPlexSetSupport(sdm, newp, supportNew));
1746         /* Hybrid edge */
1747         coneNew[0] = newp;
1748         coneNew[1] = newp;
1749         PetscCall(DMPlexSetCone(sdm, hybedge, coneNew));
1750         for (e = 0, qf = 0; e < supportSize; ++e) {
1751           PetscInt val, edge;
1752 
1753           PetscCall(DMLabelGetValue(label, support[e], &val));
1754           if (val == 1) {
1755             PetscCall(PetscFindInt(support[e], numSplitPoints[dep + 1], splitPoints[dep + 1], &edge));
1756             PetscCheck(edge >= 0, comm, PETSC_ERR_ARG_WRONG, "Edge %" PetscInt_FMT " is not a split edge", support[e]);
1757             supportNew[qf++] = edge + pMaxNew[dep + 2] + numSplitPoints[dep + 2];
1758           } else if (val == (shift2 + 1)) {
1759             PetscCall(PetscFindInt(support[e], numUnsplitPoints[dep + 1], unsplitPoints[dep + 1], &edge));
1760             PetscCheck(edge >= 0, comm, PETSC_ERR_ARG_WRONG, "Edge %" PetscInt_FMT " is not a unsplit edge", support[e]);
1761             supportNew[qf++] = edge + pMaxNew[dep + 2] + numSplitPoints[dep + 2] + numSplitPoints[dep + 1];
1762           }
1763         }
1764         PetscCall(DMPlexSetSupport(sdm, hybedge, supportNew));
1765       } else if (dep == dim - 2) {
1766         const PetscInt hybface = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1] + numSplitPoints[dep];
1767 
1768         /* Unsplit edge: Faces into original edge, split face, and hybrid face twice */
1769         for (f = 0, qf = 0; f < supportSize; ++f) {
1770           PetscInt val, face;
1771 
1772           PetscCall(DMLabelGetValue(label, support[f], &val));
1773           if (val == dim - 1) {
1774             PetscCall(PetscFindInt(support[f], numSplitPoints[dep + 1], splitPoints[dep + 1], &face));
1775             PetscCheck(face >= 0, comm, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " is not a split face", support[f]);
1776             supportNew[qf++] = DMPlexShiftPoint_Internal(support[f], depth, depthShift) /*support[f] + depthOffset[dep+1]*/;
1777             supportNew[qf++] = face + pMaxNew[dep + 1];
1778           } else {
1779             supportNew[qf++] = DMPlexShiftPoint_Internal(support[f], depth, depthShift) /*support[f] + depthOffset[dep+1]*/;
1780           }
1781         }
1782         supportNew[qf++] = hybface;
1783         supportNew[qf++] = hybface;
1784         PetscCall(DMPlexGetSupportSize(sdm, newp, &supportSizeNew));
1785         PetscCheck(qf == supportSizeNew, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Support size for unsplit edge %" PetscInt_FMT " is %" PetscInt_FMT " != %" PetscInt_FMT, newp, qf, supportSizeNew);
1786         PetscCall(DMPlexSetSupport(sdm, newp, supportNew));
1787         /* Add hybrid face */
1788         coneNew[0] = newp;
1789         coneNew[1] = newp;
1790         PetscCall(PetscFindInt(cone[0], numUnsplitPoints[dep - 1], unsplitPoints[dep - 1], &v));
1791         PetscCheck(v >= 0, comm, PETSC_ERR_ARG_WRONG, "Vertex %" PetscInt_FMT " is not an unsplit vertex", cone[0]);
1792         coneNew[2] = v + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep - 1];
1793         PetscCall(PetscFindInt(cone[1], numUnsplitPoints[dep - 1], unsplitPoints[dep - 1], &v));
1794         PetscCheck(v >= 0, comm, PETSC_ERR_ARG_WRONG, "Vertex %" PetscInt_FMT " is not an unsplit vertex", cone[1]);
1795         coneNew[3] = v + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep - 1];
1796         PetscCall(DMPlexSetCone(sdm, hybface, coneNew));
1797         for (f = 0, qf = 0; f < supportSize; ++f) {
1798           PetscInt val, face;
1799 
1800           PetscCall(DMLabelGetValue(label, support[f], &val));
1801           if (val == dim - 1) {
1802             PetscCall(PetscFindInt(support[f], numSplitPoints[dep + 1], splitPoints[dep + 1], &face));
1803             supportNew[qf++] = face + pMaxNew[dep + 2] + numSplitPoints[dep + 2];
1804           }
1805         }
1806         PetscCall(DMPlexGetSupportSize(sdm, hybface, &supportSizeNew));
1807         PetscCheck(qf == supportSizeNew, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Support size for hybrid face %" PetscInt_FMT " is %" PetscInt_FMT " != %" PetscInt_FMT, hybface, qf, supportSizeNew);
1808         PetscCall(DMPlexSetSupport(sdm, hybface, supportNew));
1809       }
1810     }
1811   }
1812   /* Step 6b: Replace split points in negative side cones */
1813   for (sp = 0; sp < numSP; ++sp) {
1814     PetscInt        dep = values[sp];
1815     IS              pIS;
1816     PetscInt        numPoints;
1817     const PetscInt *points;
1818 
1819     if (dep >= 0) continue;
1820     PetscCall(DMLabelGetStratumIS(label, dep, &pIS));
1821     if (!pIS) continue;
1822     dep = -dep - shift;
1823     PetscCall(ISGetLocalSize(pIS, &numPoints));
1824     PetscCall(ISGetIndices(pIS, &points));
1825     for (p = 0; p < numPoints; ++p) {
1826       const PetscInt  oldp = points[p];
1827       const PetscInt  newp = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*depthOffset[dep] + oldp*/;
1828       const PetscInt *cone;
1829       PetscInt        coneSize, c;
1830       /* PetscBool       replaced = PETSC_FALSE; */
1831 
1832       /* Negative edge: replace split vertex */
1833       /* Negative cell: replace split face */
1834       PetscCall(DMPlexGetConeSize(sdm, newp, &coneSize));
1835       PetscCall(DMPlexGetCone(sdm, newp, &cone));
1836       for (c = 0; c < coneSize; ++c) {
1837         const PetscInt coldp = DMPlexShiftPointInverse_Internal(cone[c], depth, depthShift);
1838         PetscInt       csplitp, cp, val;
1839 
1840         PetscCall(DMLabelGetValue(label, coldp, &val));
1841         if (val == dep - 1) {
1842           PetscCall(PetscFindInt(coldp, numSplitPoints[dep - 1], splitPoints[dep - 1], &cp));
1843           PetscCheck(cp >= 0, comm, PETSC_ERR_ARG_WRONG, "Point %" PetscInt_FMT " is not a split point of dimension %" PetscInt_FMT, oldp, dep - 1);
1844           csplitp = pMaxNew[dep - 1] + cp;
1845           PetscCall(DMPlexInsertCone(sdm, newp, c, csplitp));
1846           /* replaced = PETSC_TRUE; */
1847         }
1848       }
1849       /* Cells with only a vertex or edge on the submesh have no replacement */
1850       /* PetscCheck(replaced,comm, PETSC_ERR_ARG_WRONG, "The cone of point %d does not contain split points", oldp); */
1851     }
1852     PetscCall(ISRestoreIndices(pIS, &points));
1853     PetscCall(ISDestroy(&pIS));
1854   }
1855   PetscCall(DMPlexReorderCohesiveSupports(sdm));
1856   /* Step 7: Coordinates */
1857   PetscCall(DMPlexShiftCoordinates_Internal(dm, depthShift, sdm));
1858   PetscCall(DMGetCoordinateSection(sdm, &coordSection));
1859   PetscCall(DMGetCoordinatesLocal(sdm, &coordinates));
1860   PetscCall(VecGetArray(coordinates, &coords));
1861   for (v = 0; v < (numSplitPoints ? numSplitPoints[0] : 0); ++v) {
1862     const PetscInt newp   = DMPlexShiftPoint_Internal(splitPoints[0][v], depth, depthShift) /*depthOffset[0] + splitPoints[0][v]*/;
1863     const PetscInt splitp = pMaxNew[0] + v;
1864     PetscInt       dof, off, soff, d;
1865 
1866     PetscCall(PetscSectionGetDof(coordSection, newp, &dof));
1867     PetscCall(PetscSectionGetOffset(coordSection, newp, &off));
1868     PetscCall(PetscSectionGetOffset(coordSection, splitp, &soff));
1869     for (d = 0; d < dof; ++d) coords[soff + d] = coords[off + d];
1870   }
1871   PetscCall(VecRestoreArray(coordinates, &coords));
1872   /* Step 8: SF, if I can figure this out we can split the mesh in parallel */
1873   PetscCall(DMPlexShiftSF_Internal(dm, depthShift, sdm));
1874   /*   TODO We need to associate the ghost points with the correct replica */
1875   /* Step 9: Labels */
1876   PetscCall(DMPlexShiftLabels_Internal(dm, depthShift, sdm));
1877   PetscCall(DMPlexCreateVTKLabel_Internal(dm, PETSC_FALSE, sdm));
1878   PetscCall(DMGetNumLabels(sdm, &numLabels));
1879   for (dep = 0; dep <= depth; ++dep) {
1880     for (p = 0; p < numSplitPoints[dep]; ++p) {
1881       const PetscInt newp   = DMPlexShiftPoint_Internal(splitPoints[dep][p], depth, depthShift) /*depthOffset[dep] + splitPoints[dep][p]*/;
1882       const PetscInt splitp = pMaxNew[dep] + p;
1883       PetscInt       l;
1884 
1885       if (splitLabel) {
1886         const PetscInt val = 100 + dep;
1887 
1888         PetscCall(DMLabelSetValue(splitLabel, newp, val));
1889         PetscCall(DMLabelSetValue(splitLabel, splitp, -val));
1890       }
1891       for (l = 0; l < numLabels; ++l) {
1892         DMLabel     mlabel;
1893         const char *lname;
1894         PetscInt    val;
1895         PetscBool   isDepth;
1896 
1897         PetscCall(DMGetLabelName(sdm, l, &lname));
1898         PetscCall(PetscStrcmp(lname, "depth", &isDepth));
1899         if (isDepth) continue;
1900         PetscCall(DMGetLabel(sdm, lname, &mlabel));
1901         PetscCall(DMLabelGetValue(mlabel, newp, &val));
1902         if (val >= 0) PetscCall(DMLabelSetValue(mlabel, splitp, val));
1903       }
1904     }
1905   }
1906   for (sp = 0; sp < numSP; ++sp) {
1907     const PetscInt dep = values[sp];
1908 
1909     if ((dep < 0) || (dep > depth)) continue;
1910     if (splitIS[dep]) PetscCall(ISRestoreIndices(splitIS[dep], &splitPoints[dep]));
1911     PetscCall(ISDestroy(&splitIS[dep]));
1912     if (unsplitIS[dep]) PetscCall(ISRestoreIndices(unsplitIS[dep], &unsplitPoints[dep]));
1913     PetscCall(ISDestroy(&unsplitIS[dep]));
1914   }
1915   if (ghostIS) PetscCall(ISRestoreIndices(ghostIS, &ghostPoints));
1916   PetscCall(ISDestroy(&ghostIS));
1917   if (label) {
1918     PetscCall(ISRestoreIndices(valueIS, &values));
1919     PetscCall(ISDestroy(&valueIS));
1920   }
1921   for (d = 0; d <= depth; ++d) {
1922     PetscCall(DMPlexGetDepthStratum(sdm, d, NULL, &pEnd));
1923     pMaxNew[d] = pEnd - numHybridPoints[d] - numHybridPointsOld[d];
1924   }
1925   PetscCall(PetscFree3(coneNew, coneONew, supportNew));
1926   PetscCall(PetscFree5(depthMax, depthEnd, depthShift, pMaxNew, numHybridPointsOld));
1927   PetscCall(PetscFree7(splitIS, unsplitIS, numSplitPoints, numUnsplitPoints, numHybridPoints, splitPoints, unsplitPoints));
1928   PetscFunctionReturn(PETSC_SUCCESS);
1929 }
1930 
1931 /*@C
1932   DMPlexConstructCohesiveCells - Construct cohesive cells which split the face along an internal interface
1933 
1934   Collective
1935 
1936   Input Parameters:
1937 + dm    - The original `DM`
1938 - label - The `DMLabel` specifying the boundary faces (this could be auto-generated)
1939 
1940   Output Parameters:
1941 + splitLabel - The `DMLabel` containing the split points, or `NULL` if no output is desired
1942 - dmSplit    - The new `DM`
1943 
1944   Level: developer
1945 
1946 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMCreate()`, `DMPlexLabelCohesiveComplete()`
1947 @*/
1948 PetscErrorCode DMPlexConstructCohesiveCells(DM dm, DMLabel label, DMLabel splitLabel, DM *dmSplit)
1949 {
1950   DM       sdm;
1951   PetscInt dim;
1952 
1953   PetscFunctionBegin;
1954   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1955   PetscAssertPointer(dmSplit, 4);
1956   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &sdm));
1957   PetscCall(DMSetType(sdm, DMPLEX));
1958   PetscCall(DMGetDimension(dm, &dim));
1959   PetscCall(DMSetDimension(sdm, dim));
1960   switch (dim) {
1961   case 2:
1962   case 3:
1963     PetscCall(DMPlexConstructCohesiveCells_Internal(dm, label, splitLabel, sdm));
1964     break;
1965   default:
1966     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cannot construct cohesive cells for dimension %" PetscInt_FMT, dim);
1967   }
1968   *dmSplit = sdm;
1969   PetscFunctionReturn(PETSC_SUCCESS);
1970 }
1971 
1972 /* Returns the side of the surface for a given cell with a face on the surface */
1973 static PetscErrorCode GetSurfaceSide_Static(DM dm, DM subdm, PetscInt numSubpoints, const PetscInt *subpoints, PetscInt cell, PetscInt face, PetscBool *pos)
1974 {
1975   const PetscInt *cone, *ornt;
1976   PetscInt        dim, coneSize, c;
1977 
1978   PetscFunctionBegin;
1979   *pos = PETSC_TRUE;
1980   PetscCall(DMGetDimension(dm, &dim));
1981   PetscCall(DMPlexGetConeSize(dm, cell, &coneSize));
1982   PetscCall(DMPlexGetCone(dm, cell, &cone));
1983   PetscCall(DMPlexGetConeOrientation(dm, cell, &ornt));
1984   for (c = 0; c < coneSize; ++c) {
1985     if (cone[c] == face) {
1986       PetscInt o = ornt[c];
1987 
1988       if (subdm) {
1989         const PetscInt *subcone, *subornt;
1990         PetscInt        subpoint, subface, subconeSize, sc;
1991 
1992         PetscCall(PetscFindInt(cell, numSubpoints, subpoints, &subpoint));
1993         PetscCall(PetscFindInt(face, numSubpoints, subpoints, &subface));
1994         PetscCall(DMPlexGetConeSize(subdm, subpoint, &subconeSize));
1995         PetscCall(DMPlexGetCone(subdm, subpoint, &subcone));
1996         PetscCall(DMPlexGetConeOrientation(subdm, subpoint, &subornt));
1997         for (sc = 0; sc < subconeSize; ++sc) {
1998           if (subcone[sc] == subface) {
1999             o = subornt[0];
2000             break;
2001           }
2002         }
2003         PetscCheck(sc < subconeSize, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not find subpoint %" PetscInt_FMT " (%" PetscInt_FMT ") in cone for subpoint %" PetscInt_FMT " (%" PetscInt_FMT ")", subface, face, subpoint, cell);
2004       }
2005       if (o >= 0) *pos = PETSC_TRUE;
2006       else *pos = PETSC_FALSE;
2007       break;
2008     }
2009   }
2010   PetscCheck(c != coneSize, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Cell %" PetscInt_FMT " in split face %" PetscInt_FMT " support does not have it in the cone", cell, face);
2011   PetscFunctionReturn(PETSC_SUCCESS);
2012 }
2013 
2014 static PetscErrorCode CheckFaultEdge_Private(DM dm, DMLabel label)
2015 {
2016   IS              facePosIS, faceNegIS, dimIS;
2017   const PetscInt *points;
2018   PetscInt        dim, numPoints, p, shift = 100, shift2 = 200;
2019 
2020   PetscFunctionBegin;
2021   PetscCall(DMGetDimension(dm, &dim));
2022   /* If any faces touching the fault divide cells on either side, split them */
2023   PetscCall(DMLabelGetStratumIS(label, shift + dim - 1, &facePosIS));
2024   PetscCall(DMLabelGetStratumIS(label, -(shift + dim - 1), &faceNegIS));
2025   if (!facePosIS || !faceNegIS) {
2026     PetscCall(ISDestroy(&facePosIS));
2027     PetscCall(ISDestroy(&faceNegIS));
2028     PetscFunctionReturn(PETSC_SUCCESS);
2029   }
2030   PetscCall(ISExpand(facePosIS, faceNegIS, &dimIS));
2031   PetscCall(ISDestroy(&facePosIS));
2032   PetscCall(ISDestroy(&faceNegIS));
2033   PetscCall(ISGetLocalSize(dimIS, &numPoints));
2034   PetscCall(ISGetIndices(dimIS, &points));
2035   for (p = 0; p < numPoints; ++p) {
2036     const PetscInt  point = points[p];
2037     const PetscInt *support;
2038     PetscInt        supportSize, valA, valB;
2039 
2040     PetscCall(DMPlexGetSupportSize(dm, point, &supportSize));
2041     if (supportSize != 2) continue;
2042     PetscCall(DMPlexGetSupport(dm, point, &support));
2043     PetscCall(DMLabelGetValue(label, support[0], &valA));
2044     PetscCall(DMLabelGetValue(label, support[1], &valB));
2045     if ((valA == -1) || (valB == -1)) continue;
2046     if (valA * valB > 0) continue;
2047     /* Check that this face is not incident on only unsplit faces, meaning has at least one split face */
2048     {
2049       PetscInt *closure = NULL;
2050       PetscBool split   = PETSC_FALSE;
2051       PetscInt  closureSize, cl;
2052 
2053       PetscCall(DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
2054       for (cl = 0; cl < closureSize * 2; cl += 2) {
2055         PetscCall(DMLabelGetValue(label, closure[cl], &valA));
2056         if ((valA >= 0) && (valA <= dim)) {
2057           split = PETSC_TRUE;
2058           break;
2059         }
2060       }
2061       PetscCall(DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
2062       if (!split) continue;
2063     }
2064     /* Split the face */
2065     PetscCall(DMLabelGetValue(label, point, &valA));
2066     PetscCall(DMLabelClearValue(label, point, valA));
2067     PetscCall(DMLabelSetValue(label, point, dim - 1));
2068     /* Label its closure:
2069       unmarked: label as unsplit
2070       incident: relabel as split
2071       split:    do nothing
2072     */
2073     {
2074       PetscInt *closure = NULL;
2075       PetscInt  closureSize, cl, dep;
2076 
2077       PetscCall(DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
2078       for (cl = 0; cl < closureSize * 2; cl += 2) {
2079         PetscCall(DMLabelGetValue(label, closure[cl], &valA));
2080         if (valA == -1) { /* Mark as unsplit */
2081           PetscCall(DMPlexGetPointDepth(dm, closure[cl], &dep));
2082           PetscCall(DMLabelSetValue(label, closure[cl], shift2 + dep));
2083         } else if (((valA >= shift) && (valA < shift2)) || ((valA <= -shift) && (valA > -shift2))) {
2084           PetscCall(DMPlexGetPointDepth(dm, closure[cl], &dep));
2085           PetscCall(DMLabelClearValue(label, closure[cl], valA));
2086           PetscCall(DMLabelSetValue(label, closure[cl], dep));
2087         }
2088       }
2089       PetscCall(DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
2090     }
2091   }
2092   PetscCall(ISRestoreIndices(dimIS, &points));
2093   PetscCall(ISDestroy(&dimIS));
2094   PetscFunctionReturn(PETSC_SUCCESS);
2095 }
2096 
2097 /*@
2098   DMPlexLabelCohesiveComplete - Starting with a label marking points on an internal surface, we add all other mesh pieces
2099   to complete the surface
2100 
2101   Input Parameters:
2102 + dm     - The `DM`
2103 . label  - A `DMLabel` marking the surface
2104 . blabel - A `DMLabel` marking the vertices on the boundary which will not be duplicated, or `NULL` to find them automatically
2105 . bvalue - Value of `DMLabel` marking the vertices on the boundary
2106 . flip   - Flag to flip the submesh normal and replace points on the other side
2107 - subdm  - The `DM` associated with the label, or `NULL`
2108 
2109   Output Parameter:
2110 . label - A `DMLabel` marking all surface points
2111 
2112   Level: developer
2113 
2114   Note:
2115   The vertices in blabel are called "unsplit" in the terminology from hybrid cell creation.
2116 
2117 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexConstructCohesiveCells()`, `DMPlexLabelComplete()`
2118 @*/
2119 PetscErrorCode DMPlexLabelCohesiveComplete(DM dm, DMLabel label, DMLabel blabel, PetscInt bvalue, PetscBool flip, DM subdm)
2120 {
2121   DMLabel         depthLabel;
2122   IS              dimIS, subpointIS = NULL;
2123   const PetscInt *points, *subpoints;
2124   const PetscInt  rev   = flip ? -1 : 1;
2125   PetscInt        shift = 100, shift2 = 200, shift3 = 300, dim, depth, numPoints, numSubpoints, p, val;
2126 
2127   PetscFunctionBegin;
2128   PetscCall(DMPlexGetDepth(dm, &depth));
2129   PetscCall(DMGetDimension(dm, &dim));
2130   PetscCall(DMPlexGetDepthLabel(dm, &depthLabel));
2131   if (subdm) {
2132     PetscCall(DMPlexGetSubpointIS(subdm, &subpointIS));
2133     if (subpointIS) {
2134       PetscCall(ISGetLocalSize(subpointIS, &numSubpoints));
2135       PetscCall(ISGetIndices(subpointIS, &subpoints));
2136     }
2137   }
2138   /* Mark cell on the fault, and its faces which touch the fault: cell orientation for face gives the side of the fault */
2139   PetscCall(DMLabelGetStratumIS(label, dim - 1, &dimIS));
2140   if (!dimIS) goto divide;
2141   PetscCall(ISGetLocalSize(dimIS, &numPoints));
2142   PetscCall(ISGetIndices(dimIS, &points));
2143   for (p = 0; p < numPoints; ++p) { /* Loop over fault faces */
2144     const PetscInt *support;
2145     PetscInt        supportSize, s;
2146 
2147     PetscCall(DMPlexGetSupportSize(dm, points[p], &supportSize));
2148 #if 0
2149     if (supportSize != 2) {
2150       const PetscInt *lp;
2151       PetscInt        Nlp, pind;
2152 
2153       /* Check that for a cell with a single support face, that face is in the SF */
2154       /*   THis check only works for the remote side. We would need root side information */
2155       PetscCall(PetscSFGetGraph(dm->sf, NULL, &Nlp, &lp, NULL));
2156       PetscCall(PetscFindInt(points[p], Nlp, lp, &pind));
2157       PetscCheck(pind >= 0,PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Split face %" PetscInt_FMT " has %" PetscInt_FMT " != 2 supports, and the face is not shared with another process", points[p], supportSize);
2158     }
2159 #endif
2160     PetscCall(DMPlexGetSupport(dm, points[p], &support));
2161     for (s = 0; s < supportSize; ++s) {
2162       const PetscInt *cone;
2163       PetscInt        coneSize, c;
2164       PetscBool       pos;
2165 
2166       PetscCall(GetSurfaceSide_Static(dm, subdm, numSubpoints, subpoints, support[s], points[p], &pos));
2167       if (pos) PetscCall(DMLabelSetValue(label, support[s], rev * (shift + dim)));
2168       else PetscCall(DMLabelSetValue(label, support[s], -rev * (shift + dim)));
2169       if (rev < 0) pos = !pos ? PETSC_TRUE : PETSC_FALSE;
2170       /* Put faces touching the fault in the label */
2171       PetscCall(DMPlexGetConeSize(dm, support[s], &coneSize));
2172       PetscCall(DMPlexGetCone(dm, support[s], &cone));
2173       for (c = 0; c < coneSize; ++c) {
2174         const PetscInt point = cone[c];
2175 
2176         PetscCall(DMLabelGetValue(label, point, &val));
2177         if (val == -1) {
2178           PetscInt *closure = NULL;
2179           PetscInt  closureSize, cl;
2180 
2181           PetscCall(DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
2182           for (cl = 0; cl < closureSize * 2; cl += 2) {
2183             const PetscInt clp  = closure[cl];
2184             PetscInt       bval = -1;
2185 
2186             PetscCall(DMLabelGetValue(label, clp, &val));
2187             if (blabel) PetscCall(DMLabelGetValue(blabel, clp, &bval));
2188             if ((val >= 0) && (val < dim - 1) && (bval < 0)) {
2189               PetscCall(DMLabelSetValue(label, point, pos == PETSC_TRUE ? shift + dim - 1 : -(shift + dim - 1)));
2190               break;
2191             }
2192           }
2193           PetscCall(DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
2194         }
2195       }
2196     }
2197   }
2198   PetscCall(ISRestoreIndices(dimIS, &points));
2199   PetscCall(ISDestroy(&dimIS));
2200   /* Mark boundary points as unsplit */
2201   if (blabel) {
2202     IS bdIS;
2203 
2204     PetscCall(DMLabelGetStratumIS(blabel, bvalue, &bdIS));
2205     PetscCall(ISGetLocalSize(bdIS, &numPoints));
2206     PetscCall(ISGetIndices(bdIS, &points));
2207     for (p = 0; p < numPoints; ++p) {
2208       const PetscInt point = points[p];
2209       PetscInt       val, bval;
2210 
2211       PetscCall(DMLabelGetValue(blabel, point, &bval));
2212       if (bval >= 0) {
2213         PetscCall(DMLabelGetValue(label, point, &val));
2214         if ((val < 0) || (val > dim)) {
2215           /* This could be a point added from splitting a vertex on an adjacent fault, otherwise its just wrong */
2216           PetscCall(DMLabelClearValue(blabel, point, bval));
2217         }
2218       }
2219     }
2220     for (p = 0; p < numPoints; ++p) {
2221       const PetscInt point = points[p];
2222       PetscInt       val, bval;
2223 
2224       PetscCall(DMLabelGetValue(blabel, point, &bval));
2225       if (bval >= 0) {
2226         const PetscInt *cone, *support;
2227         PetscInt        coneSize, supportSize, s, valA, valB, valE;
2228 
2229         /* Mark as unsplit */
2230         PetscCall(DMLabelGetValue(label, point, &val));
2231         PetscCheck(val >= 0 && val <= dim, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %" PetscInt_FMT " has label value %" PetscInt_FMT ", should be part of the fault", point, val);
2232         PetscCall(DMLabelClearValue(label, point, val));
2233         PetscCall(DMLabelSetValue(label, point, shift2 + val));
2234         /* Check for cross-edge
2235              A cross-edge has endpoints which are both on the boundary of the surface, but the edge itself is not. */
2236         if (val != 0) continue;
2237         PetscCall(DMPlexGetSupport(dm, point, &support));
2238         PetscCall(DMPlexGetSupportSize(dm, point, &supportSize));
2239         for (s = 0; s < supportSize; ++s) {
2240           PetscCall(DMPlexGetCone(dm, support[s], &cone));
2241           PetscCall(DMPlexGetConeSize(dm, support[s], &coneSize));
2242           PetscCheck(coneSize == 2, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Edge %" PetscInt_FMT " has %" PetscInt_FMT " vertices != 2", support[s], coneSize);
2243           PetscCall(DMLabelGetValue(blabel, cone[0], &valA));
2244           PetscCall(DMLabelGetValue(blabel, cone[1], &valB));
2245           PetscCall(DMLabelGetValue(blabel, support[s], &valE));
2246           if ((valE < 0) && (valA >= 0) && (valB >= 0) && (cone[0] != cone[1])) PetscCall(DMLabelSetValue(blabel, support[s], 2));
2247         }
2248       }
2249     }
2250     PetscCall(ISRestoreIndices(bdIS, &points));
2251     PetscCall(ISDestroy(&bdIS));
2252   }
2253   /* Mark ghost fault cells */
2254   {
2255     PetscSF         sf;
2256     const PetscInt *leaves;
2257     PetscInt        Nl, l;
2258 
2259     PetscCall(DMGetPointSF(dm, &sf));
2260     PetscCall(PetscSFGetGraph(sf, NULL, &Nl, &leaves, NULL));
2261     PetscCall(DMLabelGetStratumIS(label, dim - 1, &dimIS));
2262     if (!dimIS) goto divide;
2263     PetscCall(ISGetLocalSize(dimIS, &numPoints));
2264     PetscCall(ISGetIndices(dimIS, &points));
2265     if (Nl > 0) {
2266       for (p = 0; p < numPoints; ++p) {
2267         const PetscInt point = points[p];
2268         PetscInt       val;
2269 
2270         PetscCall(PetscFindInt(point, Nl, leaves, &l));
2271         if (l >= 0) {
2272           PetscInt *closure = NULL;
2273           PetscInt  closureSize, cl;
2274 
2275           PetscCall(DMLabelGetValue(label, point, &val));
2276           PetscCheck((val == dim - 1) || (val == shift2 + dim - 1), PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %" PetscInt_FMT " has label value %" PetscInt_FMT ", should be a fault face", point, val);
2277           PetscCall(DMLabelClearValue(label, point, val));
2278           PetscCall(DMLabelSetValue(label, point, shift3 + val));
2279           PetscCall(DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
2280           for (cl = 2; cl < closureSize * 2; cl += 2) {
2281             const PetscInt clp = closure[cl];
2282 
2283             PetscCall(DMLabelGetValue(label, clp, &val));
2284             PetscCheck(val != -1, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %" PetscInt_FMT " is missing from label, but is in the closure of a fault face", point);
2285             PetscCall(DMLabelClearValue(label, clp, val));
2286             PetscCall(DMLabelSetValue(label, clp, shift3 + val));
2287           }
2288           PetscCall(DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
2289         }
2290       }
2291     }
2292     PetscCall(ISRestoreIndices(dimIS, &points));
2293     PetscCall(ISDestroy(&dimIS));
2294   }
2295 divide:
2296   if (subpointIS) PetscCall(ISRestoreIndices(subpointIS, &subpoints));
2297   PetscCall(DMPlexLabelFaultHalo(dm, label));
2298   PetscCall(CheckFaultEdge_Private(dm, label));
2299   PetscFunctionReturn(PETSC_SUCCESS);
2300 }
2301 
2302 /* Check that no cell have all vertices on the fault */
2303 static PetscErrorCode DMPlexCheckValidSubmesh_Private(DM dm, DMLabel label, DM subdm)
2304 {
2305   IS              subpointIS;
2306   const PetscInt *dmpoints;
2307   PetscInt        defaultValue, cStart, cEnd, c, vStart, vEnd;
2308 
2309   PetscFunctionBegin;
2310   if (!label) PetscFunctionReturn(PETSC_SUCCESS);
2311   PetscCall(DMLabelGetDefaultValue(label, &defaultValue));
2312   PetscCall(DMPlexGetSubpointIS(subdm, &subpointIS));
2313   if (!subpointIS) PetscFunctionReturn(PETSC_SUCCESS);
2314   PetscCall(DMPlexGetHeightStratum(subdm, 0, &cStart, &cEnd));
2315   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
2316   PetscCall(ISGetIndices(subpointIS, &dmpoints));
2317   for (c = cStart; c < cEnd; ++c) {
2318     PetscBool invalidCell = PETSC_TRUE;
2319     PetscInt *closure     = NULL;
2320     PetscInt  closureSize, cl;
2321 
2322     PetscCall(DMPlexGetTransitiveClosure(dm, dmpoints[c], PETSC_TRUE, &closureSize, &closure));
2323     for (cl = 0; cl < closureSize * 2; cl += 2) {
2324       PetscInt value = 0;
2325 
2326       if ((closure[cl] < vStart) || (closure[cl] >= vEnd)) continue;
2327       PetscCall(DMLabelGetValue(label, closure[cl], &value));
2328       if (value == defaultValue) {
2329         invalidCell = PETSC_FALSE;
2330         break;
2331       }
2332     }
2333     PetscCall(DMPlexRestoreTransitiveClosure(dm, dmpoints[c], PETSC_TRUE, &closureSize, &closure));
2334     if (invalidCell) {
2335       PetscCall(ISRestoreIndices(subpointIS, &dmpoints));
2336       PetscCall(ISDestroy(&subpointIS));
2337       PetscCall(DMDestroy(&subdm));
2338       SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Ambiguous submesh. Cell %" PetscInt_FMT " has all of its vertices on the submesh.", dmpoints[c]);
2339     }
2340   }
2341   PetscCall(ISRestoreIndices(subpointIS, &dmpoints));
2342   PetscFunctionReturn(PETSC_SUCCESS);
2343 }
2344 
2345 /*@
2346   DMPlexCreateHybridMesh - Create a mesh with hybrid cells along an internal interface
2347 
2348   Collective
2349 
2350   Input Parameters:
2351 + dm      - The original `DM`
2352 . label   - The label specifying the interface vertices
2353 . bdlabel - The optional label specifying the interface boundary vertices
2354 - bdvalue - Value of optional label specifying the interface boundary vertices
2355 
2356   Output Parameters:
2357 + hybridLabel - The label fully marking the interface, or `NULL` if no output is desired
2358 . splitLabel  - The label containing the split points, or `NULL` if no output is desired
2359 . dmInterface - The new interface `DM`, or `NULL`
2360 - dmHybrid    - The new `DM` with cohesive cells
2361 
2362   Level: developer
2363 
2364   Note:
2365   The hybridLabel indicates what parts of the original mesh impinged on the division surface. For points
2366   directly on the division surface, they are labeled with their dimension, so an edge 7 on the division surface would be
2367   7 (1) in hybridLabel. For points that impinge from the positive side, they are labeled with 100+dim, so an edge 6 with
2368   one vertex 3 on the surface would be 6 (101) and 3 (0) in hybridLabel. If an edge 9 from the negative side of the
2369   surface also hits vertex 3, it would be 9 (-101) in hybridLabel.
2370 
2371   The splitLabel indicates what points in the new hybrid mesh were the result of splitting points in the original
2372   mesh. The label value is $\pm 100+dim$ for each point. For example, if two edges 10 and 14 in the hybrid resulting from
2373   splitting an edge in the original mesh, you would have 10 (101) and 14 (-101) in the splitLabel.
2374 
2375   The dmInterface is a `DM` built from the original division surface. It has a label which can be retrieved using
2376   `DMPlexGetSubpointMap()` which maps each point back to the point in the surface of the original mesh.
2377 
2378 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexConstructCohesiveCells()`, `DMPlexLabelCohesiveComplete()`, `DMPlexGetSubpointMap()`, `DMCreate()`
2379 @*/
2380 PetscErrorCode DMPlexCreateHybridMesh(DM dm, DMLabel label, DMLabel bdlabel, PetscInt bdvalue, DMLabel *hybridLabel, DMLabel *splitLabel, DM *dmInterface, DM *dmHybrid)
2381 {
2382   DM       idm;
2383   DMLabel  subpointMap, hlabel, slabel = NULL;
2384   PetscInt dim;
2385 
2386   PetscFunctionBegin;
2387   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2388   if (label) PetscAssertPointer(label, 2);
2389   if (bdlabel) PetscAssertPointer(bdlabel, 3);
2390   if (hybridLabel) PetscAssertPointer(hybridLabel, 5);
2391   if (splitLabel) PetscAssertPointer(splitLabel, 6);
2392   if (dmInterface) PetscAssertPointer(dmInterface, 7);
2393   PetscAssertPointer(dmHybrid, 8);
2394   PetscCall(DMGetDimension(dm, &dim));
2395   PetscCall(DMPlexCreateSubmesh(dm, label, 1, PETSC_FALSE, &idm));
2396   PetscCall(DMPlexCheckValidSubmesh_Private(dm, label, idm));
2397   PetscCall(DMPlexOrient(idm));
2398   PetscCall(DMPlexGetSubpointMap(idm, &subpointMap));
2399   PetscCall(DMLabelDuplicate(subpointMap, &hlabel));
2400   PetscCall(DMLabelClearStratum(hlabel, dim));
2401   if (splitLabel) {
2402     const char *name;
2403     char        sname[PETSC_MAX_PATH_LEN];
2404 
2405     PetscCall(PetscObjectGetName((PetscObject)hlabel, &name));
2406     PetscCall(PetscStrncpy(sname, name, sizeof(sname)));
2407     PetscCall(PetscStrlcat(sname, " split", sizeof(sname)));
2408     PetscCall(DMLabelCreate(PETSC_COMM_SELF, sname, &slabel));
2409   }
2410   PetscCall(DMPlexLabelCohesiveComplete(dm, hlabel, bdlabel, bdvalue, PETSC_FALSE, idm));
2411   if (dmInterface) {
2412     *dmInterface = idm;
2413   } else PetscCall(DMDestroy(&idm));
2414   PetscCall(DMPlexConstructCohesiveCells(dm, hlabel, slabel, dmHybrid));
2415   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_TRUE, *dmHybrid));
2416   if (hybridLabel) *hybridLabel = hlabel;
2417   else PetscCall(DMLabelDestroy(&hlabel));
2418   if (splitLabel) *splitLabel = slabel;
2419   {
2420     DM      cdm;
2421     DMLabel ctLabel;
2422 
2423     /* We need to somehow share the celltype label with the coordinate dm */
2424     PetscCall(DMGetCoordinateDM(*dmHybrid, &cdm));
2425     PetscCall(DMPlexGetCellTypeLabel(*dmHybrid, &ctLabel));
2426     PetscCall(DMSetLabel(cdm, ctLabel));
2427   }
2428   PetscFunctionReturn(PETSC_SUCCESS);
2429 }
2430 
2431 /* Here we need the explicit assumption that:
2432 
2433      For any marked cell, the marked vertices constitute a single face
2434 */
2435 static PetscErrorCode DMPlexMarkSubmesh_Uninterpolated(DM dm, DMLabel vertexLabel, PetscInt value, DMLabel subpointMap, PetscInt *numFaces, PetscInt *nFV, DM subdm)
2436 {
2437   IS              subvertexIS = NULL;
2438   const PetscInt *subvertices;
2439   PetscInt       *pStart, *pEnd, pSize;
2440   PetscInt        depth, dim, d, numSubVerticesInitial = 0, v;
2441 
2442   PetscFunctionBegin;
2443   *numFaces = 0;
2444   *nFV      = 0;
2445   PetscCall(DMPlexGetDepth(dm, &depth));
2446   PetscCall(DMGetDimension(dm, &dim));
2447   pSize = PetscMax(depth, dim) + 1;
2448   PetscCall(PetscMalloc2(pSize, &pStart, pSize, &pEnd));
2449   for (d = 0; d <= depth; ++d) PetscCall(DMPlexGetSimplexOrBoxCells(dm, depth - d, &pStart[d], &pEnd[d]));
2450   /* Loop over initial vertices and mark all faces in the collective star() */
2451   if (vertexLabel) PetscCall(DMLabelGetStratumIS(vertexLabel, value, &subvertexIS));
2452   if (subvertexIS) {
2453     PetscCall(ISGetSize(subvertexIS, &numSubVerticesInitial));
2454     PetscCall(ISGetIndices(subvertexIS, &subvertices));
2455   }
2456   for (v = 0; v < numSubVerticesInitial; ++v) {
2457     const PetscInt vertex = subvertices[v];
2458     PetscInt      *star   = NULL;
2459     PetscInt       starSize, s, numCells = 0, c;
2460 
2461     PetscCall(DMPlexGetTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star));
2462     for (s = 0; s < starSize * 2; s += 2) {
2463       const PetscInt point = star[s];
2464       if ((point >= pStart[depth]) && (point < pEnd[depth])) star[numCells++] = point;
2465     }
2466     for (c = 0; c < numCells; ++c) {
2467       const PetscInt cell    = star[c];
2468       PetscInt      *closure = NULL;
2469       PetscInt       closureSize, cl;
2470       PetscInt       cellLoc, numCorners = 0, faceSize = 0;
2471 
2472       PetscCall(DMLabelGetValue(subpointMap, cell, &cellLoc));
2473       if (cellLoc == 2) continue;
2474       PetscCheck(cellLoc < 0, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Cell %" PetscInt_FMT " has dimension %" PetscInt_FMT " in the surface label", cell, cellLoc);
2475       PetscCall(DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
2476       for (cl = 0; cl < closureSize * 2; cl += 2) {
2477         const PetscInt point = closure[cl];
2478         PetscInt       vertexLoc;
2479 
2480         if ((point >= pStart[0]) && (point < pEnd[0])) {
2481           ++numCorners;
2482           PetscCall(DMLabelGetValue(vertexLabel, point, &vertexLoc));
2483           if (vertexLoc == value) closure[faceSize++] = point;
2484         }
2485       }
2486       if (!*nFV) PetscCall(DMPlexGetNumFaceVertices(dm, dim, numCorners, nFV));
2487       PetscCheck(faceSize <= *nFV, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Invalid submesh: Too many vertices %" PetscInt_FMT " of an element on the surface", faceSize);
2488       if (faceSize == *nFV) {
2489         const PetscInt *cells = NULL;
2490         PetscInt        numCells, nc;
2491 
2492         ++(*numFaces);
2493         for (cl = 0; cl < faceSize; ++cl) PetscCall(DMLabelSetValue(subpointMap, closure[cl], 0));
2494         PetscCall(DMPlexGetJoin(dm, faceSize, closure, &numCells, &cells));
2495         for (nc = 0; nc < numCells; ++nc) PetscCall(DMLabelSetValue(subpointMap, cells[nc], 2));
2496         PetscCall(DMPlexRestoreJoin(dm, faceSize, closure, &numCells, &cells));
2497       }
2498       PetscCall(DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
2499     }
2500     PetscCall(DMPlexRestoreTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star));
2501   }
2502   if (subvertexIS) PetscCall(ISRestoreIndices(subvertexIS, &subvertices));
2503   PetscCall(ISDestroy(&subvertexIS));
2504   PetscCall(PetscFree2(pStart, pEnd));
2505   PetscFunctionReturn(PETSC_SUCCESS);
2506 }
2507 
2508 static PetscErrorCode DMPlexMarkSubmesh_Interpolated(DM dm, DMLabel vertexLabel, PetscInt value, PetscBool markedFaces, DMLabel subpointMap, DM subdm)
2509 {
2510   IS              subvertexIS = NULL;
2511   const PetscInt *subvertices;
2512   PetscInt       *pStart, *pEnd;
2513   PetscInt        dim, d, numSubVerticesInitial = 0, v;
2514 
2515   PetscFunctionBegin;
2516   PetscCall(DMGetDimension(dm, &dim));
2517   PetscCall(PetscMalloc2(dim + 1, &pStart, dim + 1, &pEnd));
2518   for (d = 0; d <= dim; ++d) PetscCall(DMPlexGetSimplexOrBoxCells(dm, dim - d, &pStart[d], &pEnd[d]));
2519   /* Loop over initial vertices and mark all faces in the collective star() */
2520   if (vertexLabel) {
2521     PetscCall(DMLabelGetStratumIS(vertexLabel, value, &subvertexIS));
2522     if (subvertexIS) {
2523       PetscCall(ISGetSize(subvertexIS, &numSubVerticesInitial));
2524       PetscCall(ISGetIndices(subvertexIS, &subvertices));
2525     }
2526   }
2527   for (v = 0; v < numSubVerticesInitial; ++v) {
2528     const PetscInt vertex = subvertices[v];
2529     PetscInt      *star   = NULL;
2530     PetscInt       starSize, s, numFaces = 0, f;
2531 
2532     PetscCall(DMPlexGetTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star));
2533     for (s = 0; s < starSize * 2; s += 2) {
2534       const PetscInt point = star[s];
2535       PetscInt       faceLoc;
2536 
2537       if ((point >= pStart[dim - 1]) && (point < pEnd[dim - 1])) {
2538         if (markedFaces) {
2539           PetscCall(DMLabelGetValue(vertexLabel, point, &faceLoc));
2540           if (faceLoc < 0) continue;
2541         }
2542         star[numFaces++] = point;
2543       }
2544     }
2545     for (f = 0; f < numFaces; ++f) {
2546       const PetscInt face    = star[f];
2547       PetscInt      *closure = NULL;
2548       PetscInt       closureSize, c;
2549       PetscInt       faceLoc;
2550 
2551       PetscCall(DMLabelGetValue(subpointMap, face, &faceLoc));
2552       if (faceLoc == dim - 1) continue;
2553       PetscCheck(faceLoc < 0, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Face %" PetscInt_FMT " has dimension %" PetscInt_FMT " in the surface label", face, faceLoc);
2554       PetscCall(DMPlexGetTransitiveClosure(dm, face, PETSC_TRUE, &closureSize, &closure));
2555       for (c = 0; c < closureSize * 2; c += 2) {
2556         const PetscInt point = closure[c];
2557         PetscInt       vertexLoc;
2558 
2559         if ((point >= pStart[0]) && (point < pEnd[0])) {
2560           PetscCall(DMLabelGetValue(vertexLabel, point, &vertexLoc));
2561           if (vertexLoc != value) break;
2562         }
2563       }
2564       if (c == closureSize * 2) {
2565         const PetscInt *support;
2566         PetscInt        supportSize, s;
2567 
2568         for (c = 0; c < closureSize * 2; c += 2) {
2569           const PetscInt point = closure[c];
2570 
2571           for (d = 0; d < dim; ++d) {
2572             if ((point >= pStart[d]) && (point < pEnd[d])) {
2573               PetscCall(DMLabelSetValue(subpointMap, point, d));
2574               break;
2575             }
2576           }
2577         }
2578         PetscCall(DMPlexGetSupportSize(dm, face, &supportSize));
2579         PetscCall(DMPlexGetSupport(dm, face, &support));
2580         for (s = 0; s < supportSize; ++s) PetscCall(DMLabelSetValue(subpointMap, support[s], dim));
2581       }
2582       PetscCall(DMPlexRestoreTransitiveClosure(dm, face, PETSC_TRUE, &closureSize, &closure));
2583     }
2584     PetscCall(DMPlexRestoreTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star));
2585   }
2586   if (subvertexIS) PetscCall(ISRestoreIndices(subvertexIS, &subvertices));
2587   PetscCall(ISDestroy(&subvertexIS));
2588   PetscCall(PetscFree2(pStart, pEnd));
2589   PetscFunctionReturn(PETSC_SUCCESS);
2590 }
2591 
2592 static PetscErrorCode DMPlexMarkCohesiveSubmesh_Uninterpolated(DM dm, PetscBool hasLagrange, const char labelname[], PetscInt value, DMLabel subpointMap, PetscInt *numFaces, PetscInt *nFV, PetscInt *subCells[], DM subdm)
2593 {
2594   DMLabel         label = NULL;
2595   const PetscInt *cone;
2596   PetscInt        dim, cMax, cEnd, c, subc = 0, p, coneSize = -1;
2597 
2598   PetscFunctionBegin;
2599   *numFaces = 0;
2600   *nFV      = 0;
2601   if (labelname) PetscCall(DMGetLabel(dm, labelname, &label));
2602   *subCells = NULL;
2603   PetscCall(DMGetDimension(dm, &dim));
2604   PetscCall(DMPlexGetTensorPrismBounds_Internal(dm, dim, &cMax, &cEnd));
2605   if (cMax < 0) PetscFunctionReturn(PETSC_SUCCESS);
2606   if (label) {
2607     for (c = cMax; c < cEnd; ++c) {
2608       PetscInt val;
2609 
2610       PetscCall(DMLabelGetValue(label, c, &val));
2611       if (val == value) {
2612         ++(*numFaces);
2613         PetscCall(DMPlexGetConeSize(dm, c, &coneSize));
2614       }
2615     }
2616   } else {
2617     *numFaces = cEnd - cMax;
2618     PetscCall(DMPlexGetConeSize(dm, cMax, &coneSize));
2619   }
2620   PetscCall(PetscMalloc1(*numFaces * 2, subCells));
2621   if (!*numFaces) PetscFunctionReturn(PETSC_SUCCESS);
2622   *nFV = hasLagrange ? coneSize / 3 : coneSize / 2;
2623   for (c = cMax; c < cEnd; ++c) {
2624     const PetscInt *cells;
2625     PetscInt        numCells;
2626 
2627     if (label) {
2628       PetscInt val;
2629 
2630       PetscCall(DMLabelGetValue(label, c, &val));
2631       if (val != value) continue;
2632     }
2633     PetscCall(DMPlexGetCone(dm, c, &cone));
2634     for (p = 0; p < *nFV; ++p) PetscCall(DMLabelSetValue(subpointMap, cone[p], 0));
2635     /* Negative face */
2636     PetscCall(DMPlexGetJoin(dm, *nFV, cone, &numCells, &cells));
2637     /* Not true in parallel
2638     PetscCheck(numCells == 2,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive cells should separate two cells"); */
2639     for (p = 0; p < numCells; ++p) {
2640       PetscCall(DMLabelSetValue(subpointMap, cells[p], 2));
2641       (*subCells)[subc++] = cells[p];
2642     }
2643     PetscCall(DMPlexRestoreJoin(dm, *nFV, cone, &numCells, &cells));
2644     /* Positive face is not included */
2645   }
2646   PetscFunctionReturn(PETSC_SUCCESS);
2647 }
2648 
2649 static PetscErrorCode DMPlexMarkCohesiveSubmesh_Interpolated(DM dm, DMLabel label, PetscInt value, DMLabel subpointMap, DM subdm)
2650 {
2651   PetscInt *pStart, *pEnd;
2652   PetscInt  dim, cMax, cEnd, c, d;
2653 
2654   PetscFunctionBegin;
2655   PetscCall(DMGetDimension(dm, &dim));
2656   PetscCall(DMPlexGetTensorPrismBounds_Internal(dm, dim, &cMax, &cEnd));
2657   if (cMax < 0) PetscFunctionReturn(PETSC_SUCCESS);
2658   PetscCall(PetscMalloc2(dim + 1, &pStart, dim + 1, &pEnd));
2659   for (d = 0; d <= dim; ++d) PetscCall(DMPlexGetDepthStratum(dm, d, &pStart[d], &pEnd[d]));
2660   for (c = cMax; c < cEnd; ++c) {
2661     const PetscInt *cone;
2662     PetscInt       *closure = NULL;
2663     PetscInt        fconeSize, coneSize, closureSize, cl, val;
2664 
2665     if (label) {
2666       PetscCall(DMLabelGetValue(label, c, &val));
2667       if (val != value) continue;
2668     }
2669     PetscCall(DMPlexGetConeSize(dm, c, &coneSize));
2670     PetscCall(DMPlexGetCone(dm, c, &cone));
2671     PetscCall(DMPlexGetConeSize(dm, cone[0], &fconeSize));
2672     PetscCheck(coneSize == (fconeSize ? fconeSize : 1) + 2, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive cells should separate two cells");
2673     /* Negative face */
2674     PetscCall(DMPlexGetTransitiveClosure(dm, cone[0], PETSC_TRUE, &closureSize, &closure));
2675     for (cl = 0; cl < closureSize * 2; cl += 2) {
2676       const PetscInt point = closure[cl];
2677 
2678       for (d = 0; d <= dim; ++d) {
2679         if ((point >= pStart[d]) && (point < pEnd[d])) {
2680           PetscCall(DMLabelSetValue(subpointMap, point, d));
2681           break;
2682         }
2683       }
2684     }
2685     PetscCall(DMPlexRestoreTransitiveClosure(dm, cone[0], PETSC_TRUE, &closureSize, &closure));
2686     /* Cells -- positive face is not included */
2687     for (cl = 0; cl < 1; ++cl) {
2688       const PetscInt *support;
2689       PetscInt        supportSize, s;
2690 
2691       PetscCall(DMPlexGetSupportSize(dm, cone[cl], &supportSize));
2692       /* PetscCheck(supportSize == 2,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive faces should separate two cells"); */
2693       PetscCall(DMPlexGetSupport(dm, cone[cl], &support));
2694       for (s = 0; s < supportSize; ++s) PetscCall(DMLabelSetValue(subpointMap, support[s], dim));
2695     }
2696   }
2697   PetscCall(PetscFree2(pStart, pEnd));
2698   PetscFunctionReturn(PETSC_SUCCESS);
2699 }
2700 
2701 static PetscErrorCode DMPlexGetFaceOrientation(DM dm, PetscInt cell, PetscInt numCorners, PetscInt indices[], PetscInt oppositeVertex, PetscInt origVertices[], PetscInt faceVertices[], PetscBool *posOriented)
2702 {
2703   MPI_Comm       comm;
2704   PetscBool      posOrient = PETSC_FALSE;
2705   const PetscInt debug     = 0;
2706   PetscInt       cellDim, faceSize, f;
2707 
2708   PetscFunctionBegin;
2709   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
2710   PetscCall(DMGetDimension(dm, &cellDim));
2711   if (debug) PetscCall(PetscPrintf(comm, "cellDim: %" PetscInt_FMT " numCorners: %" PetscInt_FMT "\n", cellDim, numCorners));
2712 
2713   if (cellDim == 1 && numCorners == 2) {
2714     /* Triangle */
2715     faceSize  = numCorners - 1;
2716     posOrient = !(oppositeVertex % 2) ? PETSC_TRUE : PETSC_FALSE;
2717   } else if (cellDim == 2 && numCorners == 3) {
2718     /* Triangle */
2719     faceSize  = numCorners - 1;
2720     posOrient = !(oppositeVertex % 2) ? PETSC_TRUE : PETSC_FALSE;
2721   } else if (cellDim == 3 && numCorners == 4) {
2722     /* Tetrahedron */
2723     faceSize  = numCorners - 1;
2724     posOrient = (oppositeVertex % 2) ? PETSC_TRUE : PETSC_FALSE;
2725   } else if (cellDim == 1 && numCorners == 3) {
2726     /* Quadratic line */
2727     faceSize  = 1;
2728     posOrient = PETSC_TRUE;
2729   } else if (cellDim == 2 && numCorners == 4) {
2730     /* Quads */
2731     faceSize = 2;
2732     if ((indices[1] > indices[0]) && (indices[1] - indices[0] == 1)) {
2733       posOrient = PETSC_TRUE;
2734     } else if ((indices[0] == 3) && (indices[1] == 0)) {
2735       posOrient = PETSC_TRUE;
2736     } else {
2737       if (((indices[0] > indices[1]) && (indices[0] - indices[1] == 1)) || ((indices[0] == 0) && (indices[1] == 3))) {
2738         posOrient = PETSC_FALSE;
2739       } else SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid quad crossedge");
2740     }
2741   } else if (cellDim == 2 && numCorners == 6) {
2742     /* Quadratic triangle (I hate this) */
2743     /* Edges are determined by the first 2 vertices (corners of edges) */
2744     const PetscInt faceSizeTri = 3;
2745     PetscInt       sortedIndices[3], i, iFace;
2746     PetscBool      found                    = PETSC_FALSE;
2747     PetscInt       faceVerticesTriSorted[9] = {
2748       0, 3, 4, /* bottom */
2749       1, 4, 5, /* right */
2750       2, 3, 5, /* left */
2751     };
2752     PetscInt faceVerticesTri[9] = {
2753       0, 3, 4, /* bottom */
2754       1, 4, 5, /* right */
2755       2, 5, 3, /* left */
2756     };
2757 
2758     for (i = 0; i < faceSizeTri; ++i) sortedIndices[i] = indices[i];
2759     PetscCall(PetscSortInt(faceSizeTri, sortedIndices));
2760     for (iFace = 0; iFace < 3; ++iFace) {
2761       const PetscInt ii = iFace * faceSizeTri;
2762       PetscInt       fVertex, cVertex;
2763 
2764       if ((sortedIndices[0] == faceVerticesTriSorted[ii + 0]) && (sortedIndices[1] == faceVerticesTriSorted[ii + 1])) {
2765         for (fVertex = 0; fVertex < faceSizeTri; ++fVertex) {
2766           for (cVertex = 0; cVertex < faceSizeTri; ++cVertex) {
2767             if (indices[cVertex] == faceVerticesTri[ii + fVertex]) {
2768               faceVertices[fVertex] = origVertices[cVertex];
2769               break;
2770             }
2771           }
2772         }
2773         found = PETSC_TRUE;
2774         break;
2775       }
2776     }
2777     PetscCheck(found, comm, PETSC_ERR_ARG_WRONG, "Invalid tri crossface");
2778     if (posOriented) *posOriented = PETSC_TRUE;
2779     PetscFunctionReturn(PETSC_SUCCESS);
2780   } else if (cellDim == 2 && numCorners == 9) {
2781     /* Quadratic quad (I hate this) */
2782     /* Edges are determined by the first 2 vertices (corners of edges) */
2783     const PetscInt faceSizeQuad = 3;
2784     PetscInt       sortedIndices[3], i, iFace;
2785     PetscBool      found                      = PETSC_FALSE;
2786     PetscInt       faceVerticesQuadSorted[12] = {
2787       0, 1, 4, /* bottom */
2788       1, 2, 5, /* right */
2789       2, 3, 6, /* top */
2790       0, 3, 7, /* left */
2791     };
2792     PetscInt faceVerticesQuad[12] = {
2793       0, 1, 4, /* bottom */
2794       1, 2, 5, /* right */
2795       2, 3, 6, /* top */
2796       3, 0, 7, /* left */
2797     };
2798 
2799     for (i = 0; i < faceSizeQuad; ++i) sortedIndices[i] = indices[i];
2800     PetscCall(PetscSortInt(faceSizeQuad, sortedIndices));
2801     for (iFace = 0; iFace < 4; ++iFace) {
2802       const PetscInt ii = iFace * faceSizeQuad;
2803       PetscInt       fVertex, cVertex;
2804 
2805       if ((sortedIndices[0] == faceVerticesQuadSorted[ii + 0]) && (sortedIndices[1] == faceVerticesQuadSorted[ii + 1])) {
2806         for (fVertex = 0; fVertex < faceSizeQuad; ++fVertex) {
2807           for (cVertex = 0; cVertex < faceSizeQuad; ++cVertex) {
2808             if (indices[cVertex] == faceVerticesQuad[ii + fVertex]) {
2809               faceVertices[fVertex] = origVertices[cVertex];
2810               break;
2811             }
2812           }
2813         }
2814         found = PETSC_TRUE;
2815         break;
2816       }
2817     }
2818     PetscCheck(found, comm, PETSC_ERR_ARG_WRONG, "Invalid quad crossface");
2819     if (posOriented) *posOriented = PETSC_TRUE;
2820     PetscFunctionReturn(PETSC_SUCCESS);
2821   } else if (cellDim == 3 && numCorners == 8) {
2822     /* Hexes
2823        A hex is two oriented quads with the normal of the first
2824        pointing up at the second.
2825 
2826           7---6
2827          /|  /|
2828         4---5 |
2829         | 1-|-2
2830         |/  |/
2831         0---3
2832 
2833         Faces are determined by the first 4 vertices (corners of faces) */
2834     const PetscInt faceSizeHex = 4;
2835     PetscInt       sortedIndices[4], i, iFace;
2836     PetscBool      found                     = PETSC_FALSE;
2837     PetscInt       faceVerticesHexSorted[24] = {
2838       0, 1, 2, 3, /* bottom */
2839       4, 5, 6, 7, /* top */
2840       0, 3, 4, 5, /* front */
2841       2, 3, 5, 6, /* right */
2842       1, 2, 6, 7, /* back */
2843       0, 1, 4, 7, /* left */
2844     };
2845     PetscInt faceVerticesHex[24] = {
2846       1, 2, 3, 0, /* bottom */
2847       4, 5, 6, 7, /* top */
2848       0, 3, 5, 4, /* front */
2849       3, 2, 6, 5, /* right */
2850       2, 1, 7, 6, /* back */
2851       1, 0, 4, 7, /* left */
2852     };
2853 
2854     for (i = 0; i < faceSizeHex; ++i) sortedIndices[i] = indices[i];
2855     PetscCall(PetscSortInt(faceSizeHex, sortedIndices));
2856     for (iFace = 0; iFace < 6; ++iFace) {
2857       const PetscInt ii = iFace * faceSizeHex;
2858       PetscInt       fVertex, cVertex;
2859 
2860       if ((sortedIndices[0] == faceVerticesHexSorted[ii + 0]) && (sortedIndices[1] == faceVerticesHexSorted[ii + 1]) && (sortedIndices[2] == faceVerticesHexSorted[ii + 2]) && (sortedIndices[3] == faceVerticesHexSorted[ii + 3])) {
2861         for (fVertex = 0; fVertex < faceSizeHex; ++fVertex) {
2862           for (cVertex = 0; cVertex < faceSizeHex; ++cVertex) {
2863             if (indices[cVertex] == faceVerticesHex[ii + fVertex]) {
2864               faceVertices[fVertex] = origVertices[cVertex];
2865               break;
2866             }
2867           }
2868         }
2869         found = PETSC_TRUE;
2870         break;
2871       }
2872     }
2873     PetscCheck(found, comm, PETSC_ERR_ARG_WRONG, "Invalid hex crossface");
2874     if (posOriented) *posOriented = PETSC_TRUE;
2875     PetscFunctionReturn(PETSC_SUCCESS);
2876   } else if (cellDim == 3 && numCorners == 10) {
2877     /* Quadratic tet */
2878     /* Faces are determined by the first 3 vertices (corners of faces) */
2879     const PetscInt faceSizeTet = 6;
2880     PetscInt       sortedIndices[6], i, iFace;
2881     PetscBool      found                     = PETSC_FALSE;
2882     PetscInt       faceVerticesTetSorted[24] = {
2883       0, 1, 2, 6, 7, 8, /* bottom */
2884       0, 3, 4, 6, 7, 9, /* front */
2885       1, 4, 5, 7, 8, 9, /* right */
2886       2, 3, 5, 6, 8, 9, /* left */
2887     };
2888     PetscInt faceVerticesTet[24] = {
2889       0, 1, 2, 6, 7, 8, /* bottom */
2890       0, 4, 3, 6, 7, 9, /* front */
2891       1, 5, 4, 7, 8, 9, /* right */
2892       2, 3, 5, 8, 6, 9, /* left */
2893     };
2894 
2895     for (i = 0; i < faceSizeTet; ++i) sortedIndices[i] = indices[i];
2896     PetscCall(PetscSortInt(faceSizeTet, sortedIndices));
2897     for (iFace = 0; iFace < 4; ++iFace) {
2898       const PetscInt ii = iFace * faceSizeTet;
2899       PetscInt       fVertex, cVertex;
2900 
2901       if ((sortedIndices[0] == faceVerticesTetSorted[ii + 0]) && (sortedIndices[1] == faceVerticesTetSorted[ii + 1]) && (sortedIndices[2] == faceVerticesTetSorted[ii + 2]) && (sortedIndices[3] == faceVerticesTetSorted[ii + 3])) {
2902         for (fVertex = 0; fVertex < faceSizeTet; ++fVertex) {
2903           for (cVertex = 0; cVertex < faceSizeTet; ++cVertex) {
2904             if (indices[cVertex] == faceVerticesTet[ii + fVertex]) {
2905               faceVertices[fVertex] = origVertices[cVertex];
2906               break;
2907             }
2908           }
2909         }
2910         found = PETSC_TRUE;
2911         break;
2912       }
2913     }
2914     PetscCheck(found, comm, PETSC_ERR_ARG_WRONG, "Invalid tet crossface");
2915     if (posOriented) *posOriented = PETSC_TRUE;
2916     PetscFunctionReturn(PETSC_SUCCESS);
2917   } else if (cellDim == 3 && numCorners == 27) {
2918     /* Quadratic hexes (I hate this)
2919        A hex is two oriented quads with the normal of the first
2920        pointing up at the second.
2921 
2922          7---6
2923         /|  /|
2924        4---5 |
2925        | 3-|-2
2926        |/  |/
2927        0---1
2928 
2929        Faces are determined by the first 4 vertices (corners of faces) */
2930     const PetscInt faceSizeQuadHex = 9;
2931     PetscInt       sortedIndices[9], i, iFace;
2932     PetscBool      found                         = PETSC_FALSE;
2933     PetscInt       faceVerticesQuadHexSorted[54] = {
2934       0, 1, 2, 3, 8,  9,  10, 11, 24, /* bottom */
2935       4, 5, 6, 7, 12, 13, 14, 15, 25, /* top */
2936       0, 1, 4, 5, 8,  12, 16, 17, 22, /* front */
2937       1, 2, 5, 6, 9,  13, 17, 18, 21, /* right */
2938       2, 3, 6, 7, 10, 14, 18, 19, 23, /* back */
2939       0, 3, 4, 7, 11, 15, 16, 19, 20, /* left */
2940     };
2941     PetscInt faceVerticesQuadHex[54] = {
2942       3, 2, 1, 0, 10, 9,  8,  11, 24, /* bottom */
2943       4, 5, 6, 7, 12, 13, 14, 15, 25, /* top */
2944       0, 1, 5, 4, 8,  17, 12, 16, 22, /* front */
2945       1, 2, 6, 5, 9,  18, 13, 17, 21, /* right */
2946       2, 3, 7, 6, 10, 19, 14, 18, 23, /* back */
2947       3, 0, 4, 7, 11, 16, 15, 19, 20  /* left */
2948     };
2949 
2950     for (i = 0; i < faceSizeQuadHex; ++i) sortedIndices[i] = indices[i];
2951     PetscCall(PetscSortInt(faceSizeQuadHex, sortedIndices));
2952     for (iFace = 0; iFace < 6; ++iFace) {
2953       const PetscInt ii = iFace * faceSizeQuadHex;
2954       PetscInt       fVertex, cVertex;
2955 
2956       if ((sortedIndices[0] == faceVerticesQuadHexSorted[ii + 0]) && (sortedIndices[1] == faceVerticesQuadHexSorted[ii + 1]) && (sortedIndices[2] == faceVerticesQuadHexSorted[ii + 2]) && (sortedIndices[3] == faceVerticesQuadHexSorted[ii + 3])) {
2957         for (fVertex = 0; fVertex < faceSizeQuadHex; ++fVertex) {
2958           for (cVertex = 0; cVertex < faceSizeQuadHex; ++cVertex) {
2959             if (indices[cVertex] == faceVerticesQuadHex[ii + fVertex]) {
2960               faceVertices[fVertex] = origVertices[cVertex];
2961               break;
2962             }
2963           }
2964         }
2965         found = PETSC_TRUE;
2966         break;
2967       }
2968     }
2969     PetscCheck(found, comm, PETSC_ERR_ARG_WRONG, "Invalid hex crossface");
2970     if (posOriented) *posOriented = PETSC_TRUE;
2971     PetscFunctionReturn(PETSC_SUCCESS);
2972   } else SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Unknown cell type for faceOrientation().");
2973   if (!posOrient) {
2974     if (debug) PetscCall(PetscPrintf(comm, "  Reversing initial face orientation\n"));
2975     for (f = 0; f < faceSize; ++f) faceVertices[f] = origVertices[faceSize - 1 - f];
2976   } else {
2977     if (debug) PetscCall(PetscPrintf(comm, "  Keeping initial face orientation\n"));
2978     for (f = 0; f < faceSize; ++f) faceVertices[f] = origVertices[f];
2979   }
2980   if (posOriented) *posOriented = posOrient;
2981   PetscFunctionReturn(PETSC_SUCCESS);
2982 }
2983 
2984 /*@
2985   DMPlexGetOrientedFace - Given a cell and a face, as a set of vertices, return the oriented face, as a set of vertices,
2986   in faceVertices. The orientation is such that the face normal points out of the cell
2987 
2988   Not Collective
2989 
2990   Input Parameters:
2991 + dm           - The original mesh
2992 . cell         - The cell mesh point
2993 . faceSize     - The number of vertices on the face
2994 . face         - The face vertices
2995 . numCorners   - The number of vertices on the cell
2996 . indices      - Local numbering of face vertices in cell cone
2997 - origVertices - Original face vertices
2998 
2999   Output Parameters:
3000 + faceVertices - The face vertices properly oriented
3001 - posOriented  - `PETSC_TRUE` if the face was oriented with outward normal
3002 
3003   Level: developer
3004 
3005 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetCone()`
3006 @*/
3007 PetscErrorCode DMPlexGetOrientedFace(DM dm, PetscInt cell, PetscInt faceSize, const PetscInt face[], PetscInt numCorners, PetscInt indices[], PetscInt origVertices[], PetscInt faceVertices[], PetscBool *posOriented)
3008 {
3009   const PetscInt *cone = NULL;
3010   PetscInt        coneSize, v, f, v2;
3011   PetscInt        oppositeVertex = -1;
3012 
3013   PetscFunctionBegin;
3014   PetscCall(DMPlexGetConeSize(dm, cell, &coneSize));
3015   PetscCall(DMPlexGetCone(dm, cell, &cone));
3016   for (v = 0, v2 = 0; v < coneSize; ++v) {
3017     PetscBool found = PETSC_FALSE;
3018 
3019     for (f = 0; f < faceSize; ++f) {
3020       if (face[f] == cone[v]) {
3021         found = PETSC_TRUE;
3022         break;
3023       }
3024     }
3025     if (found) {
3026       indices[v2]      = v;
3027       origVertices[v2] = cone[v];
3028       ++v2;
3029     } else {
3030       oppositeVertex = v;
3031     }
3032   }
3033   PetscCall(DMPlexGetFaceOrientation(dm, cell, numCorners, indices, oppositeVertex, origVertices, faceVertices, posOriented));
3034   PetscFunctionReturn(PETSC_SUCCESS);
3035 }
3036 
3037 /*
3038   DMPlexInsertFace_Internal - Puts a face into the mesh
3039 
3040   Not Collective
3041 
3042   Input Parameters:
3043   + dm              - The `DMPLEX`
3044   . numFaceVertex   - The number of vertices in the face
3045   . faceVertices    - The vertices in the face for `dm`
3046   . subfaceVertices - The vertices in the face for subdm
3047   . numCorners      - The number of vertices in the `cell`
3048   . cell            - A cell in `dm` containing the face
3049   . subcell         - A cell in subdm containing the face
3050   . firstFace       - First face in the mesh
3051   - newFacePoint    - Next face in the mesh
3052 
3053   Output Parameter:
3054   . newFacePoint - Contains next face point number on input, updated on output
3055 
3056   Level: developer
3057 */
3058 static PetscErrorCode DMPlexInsertFace_Internal(DM dm, DM subdm, PetscInt numFaceVertices, const PetscInt faceVertices[], const PetscInt subfaceVertices[], PetscInt numCorners, PetscInt cell, PetscInt subcell, PetscInt firstFace, PetscInt *newFacePoint)
3059 {
3060   MPI_Comm        comm;
3061   DM_Plex        *submesh = (DM_Plex *)subdm->data;
3062   const PetscInt *faces;
3063   PetscInt        numFaces, coneSize;
3064 
3065   PetscFunctionBegin;
3066   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
3067   PetscCall(DMPlexGetConeSize(subdm, subcell, &coneSize));
3068   PetscCheck(coneSize == 1, comm, PETSC_ERR_ARG_OUTOFRANGE, "Cone size of cell %" PetscInt_FMT " is %" PetscInt_FMT " != 1", cell, coneSize);
3069 #if 0
3070   /* Cannot use this because support() has not been constructed yet */
3071   PetscCall(DMPlexGetJoin(subdm, numFaceVertices, subfaceVertices, &numFaces, &faces));
3072 #else
3073   {
3074     PetscInt f;
3075 
3076     numFaces = 0;
3077     PetscCall(DMGetWorkArray(subdm, 1, MPIU_INT, (void **)&faces));
3078     for (f = firstFace; f < *newFacePoint; ++f) {
3079       PetscInt dof, off, d;
3080 
3081       PetscCall(PetscSectionGetDof(submesh->coneSection, f, &dof));
3082       PetscCall(PetscSectionGetOffset(submesh->coneSection, f, &off));
3083       /* Yes, I know this is quadratic, but I expect the sizes to be <5 */
3084       for (d = 0; d < dof; ++d) {
3085         const PetscInt p = submesh->cones[off + d];
3086         PetscInt       v;
3087 
3088         for (v = 0; v < numFaceVertices; ++v) {
3089           if (subfaceVertices[v] == p) break;
3090         }
3091         if (v == numFaceVertices) break;
3092       }
3093       if (d == dof) {
3094         numFaces               = 1;
3095         ((PetscInt *)faces)[0] = f;
3096       }
3097     }
3098   }
3099 #endif
3100   PetscCheck(numFaces <= 1, comm, PETSC_ERR_ARG_WRONG, "Vertex set had %" PetscInt_FMT " faces, not one", numFaces);
3101   if (numFaces == 1) {
3102     /* Add the other cell neighbor for this face */
3103     PetscCall(DMPlexSetCone(subdm, subcell, faces));
3104   } else {
3105     PetscInt *indices, *origVertices, *orientedVertices, *orientedSubVertices, v, ov;
3106     PetscBool posOriented;
3107 
3108     PetscCall(DMGetWorkArray(subdm, 4 * numFaceVertices * sizeof(PetscInt), MPIU_INT, &orientedVertices));
3109     origVertices        = &orientedVertices[numFaceVertices];
3110     indices             = &orientedVertices[numFaceVertices * 2];
3111     orientedSubVertices = &orientedVertices[numFaceVertices * 3];
3112     PetscCall(DMPlexGetOrientedFace(dm, cell, numFaceVertices, faceVertices, numCorners, indices, origVertices, orientedVertices, &posOriented));
3113     /* TODO: I know that routine should return a permutation, not the indices */
3114     for (v = 0; v < numFaceVertices; ++v) {
3115       const PetscInt vertex = faceVertices[v], subvertex = subfaceVertices[v];
3116       for (ov = 0; ov < numFaceVertices; ++ov) {
3117         if (orientedVertices[ov] == vertex) {
3118           orientedSubVertices[ov] = subvertex;
3119           break;
3120         }
3121       }
3122       PetscCheck(ov != numFaceVertices, comm, PETSC_ERR_PLIB, "Could not find face vertex %" PetscInt_FMT " in orientated set", vertex);
3123     }
3124     PetscCall(DMPlexSetCone(subdm, *newFacePoint, orientedSubVertices));
3125     PetscCall(DMPlexSetCone(subdm, subcell, newFacePoint));
3126     PetscCall(DMRestoreWorkArray(subdm, 4 * numFaceVertices * sizeof(PetscInt), MPIU_INT, &orientedVertices));
3127     ++(*newFacePoint);
3128   }
3129 #if 0
3130   PetscCall(DMPlexRestoreJoin(subdm, numFaceVertices, subfaceVertices, &numFaces, &faces));
3131 #else
3132   PetscCall(DMRestoreWorkArray(subdm, 1, MPIU_INT, (void **)&faces));
3133 #endif
3134   PetscFunctionReturn(PETSC_SUCCESS);
3135 }
3136 
3137 static PetscErrorCode DMPlexCreateSubmesh_Uninterpolated(DM dm, DMLabel vertexLabel, PetscInt value, DM subdm)
3138 {
3139   MPI_Comm        comm;
3140   DMLabel         subpointMap;
3141   IS              subvertexIS, subcellIS;
3142   const PetscInt *subVertices, *subCells;
3143   PetscInt        numSubVertices, firstSubVertex, numSubCells;
3144   PetscInt       *subface, maxConeSize, numSubFaces = 0, firstSubFace, newFacePoint, nFV = 0;
3145   PetscInt        vStart, vEnd, c, f;
3146 
3147   PetscFunctionBegin;
3148   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
3149   /* Create subpointMap which marks the submesh */
3150   PetscCall(DMLabelCreate(PETSC_COMM_SELF, "subpoint_map", &subpointMap));
3151   PetscCall(DMPlexSetSubpointMap(subdm, subpointMap));
3152   if (vertexLabel) PetscCall(DMPlexMarkSubmesh_Uninterpolated(dm, vertexLabel, value, subpointMap, &numSubFaces, &nFV, subdm));
3153   /* Setup chart */
3154   PetscCall(DMLabelGetStratumSize(subpointMap, 0, &numSubVertices));
3155   PetscCall(DMLabelGetStratumSize(subpointMap, 2, &numSubCells));
3156   PetscCall(DMPlexSetChart(subdm, 0, numSubCells + numSubFaces + numSubVertices));
3157   PetscCall(DMPlexSetVTKCellHeight(subdm, 1));
3158   /* Set cone sizes */
3159   firstSubVertex = numSubCells;
3160   firstSubFace   = numSubCells + numSubVertices;
3161   newFacePoint   = firstSubFace;
3162   PetscCall(DMLabelGetStratumIS(subpointMap, 0, &subvertexIS));
3163   if (subvertexIS) PetscCall(ISGetIndices(subvertexIS, &subVertices));
3164   PetscCall(DMLabelGetStratumIS(subpointMap, 2, &subcellIS));
3165   if (subcellIS) PetscCall(ISGetIndices(subcellIS, &subCells));
3166   for (c = 0; c < numSubCells; ++c) PetscCall(DMPlexSetConeSize(subdm, c, 1));
3167   for (f = firstSubFace; f < firstSubFace + numSubFaces; ++f) PetscCall(DMPlexSetConeSize(subdm, f, nFV));
3168   PetscCall(DMSetUp(subdm));
3169   PetscCall(DMLabelDestroy(&subpointMap));
3170   /* Create face cones */
3171   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
3172   PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, NULL));
3173   PetscCall(DMGetWorkArray(subdm, maxConeSize, MPIU_INT, (void **)&subface));
3174   for (c = 0; c < numSubCells; ++c) {
3175     const PetscInt cell    = subCells[c];
3176     const PetscInt subcell = c;
3177     PetscInt      *closure = NULL;
3178     PetscInt       closureSize, cl, numCorners = 0, faceSize = 0;
3179 
3180     PetscCall(DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
3181     for (cl = 0; cl < closureSize * 2; cl += 2) {
3182       const PetscInt point = closure[cl];
3183       PetscInt       subVertex;
3184 
3185       if ((point >= vStart) && (point < vEnd)) {
3186         ++numCorners;
3187         PetscCall(PetscFindInt(point, numSubVertices, subVertices, &subVertex));
3188         if (subVertex >= 0) {
3189           closure[faceSize] = point;
3190           subface[faceSize] = firstSubVertex + subVertex;
3191           ++faceSize;
3192         }
3193       }
3194     }
3195     PetscCheck(faceSize <= nFV, comm, PETSC_ERR_ARG_WRONG, "Invalid submesh: Too many vertices %" PetscInt_FMT " of an element on the surface", faceSize);
3196     if (faceSize == nFV) PetscCall(DMPlexInsertFace_Internal(dm, subdm, faceSize, closure, subface, numCorners, cell, subcell, firstSubFace, &newFacePoint));
3197     PetscCall(DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
3198   }
3199   PetscCall(DMRestoreWorkArray(subdm, maxConeSize, MPIU_INT, (void **)&subface));
3200   PetscCall(DMPlexSymmetrize(subdm));
3201   PetscCall(DMPlexStratify(subdm));
3202   /* Build coordinates */
3203   {
3204     PetscSection coordSection, subCoordSection;
3205     Vec          coordinates, subCoordinates;
3206     PetscScalar *coords, *subCoords;
3207     PetscInt     numComp, coordSize, v;
3208     const char  *name;
3209 
3210     PetscCall(DMGetCoordinateSection(dm, &coordSection));
3211     PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
3212     PetscCall(DMGetCoordinateSection(subdm, &subCoordSection));
3213     PetscCall(PetscSectionSetNumFields(subCoordSection, 1));
3214     PetscCall(PetscSectionGetFieldComponents(coordSection, 0, &numComp));
3215     PetscCall(PetscSectionSetFieldComponents(subCoordSection, 0, numComp));
3216     PetscCall(PetscSectionSetChart(subCoordSection, firstSubVertex, firstSubVertex + numSubVertices));
3217     for (v = 0; v < numSubVertices; ++v) {
3218       const PetscInt vertex    = subVertices[v];
3219       const PetscInt subvertex = firstSubVertex + v;
3220       PetscInt       dof;
3221 
3222       PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3223       PetscCall(PetscSectionSetDof(subCoordSection, subvertex, dof));
3224       PetscCall(PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof));
3225     }
3226     PetscCall(PetscSectionSetUp(subCoordSection));
3227     PetscCall(PetscSectionGetStorageSize(subCoordSection, &coordSize));
3228     PetscCall(VecCreate(PETSC_COMM_SELF, &subCoordinates));
3229     PetscCall(PetscObjectGetName((PetscObject)coordinates, &name));
3230     PetscCall(PetscObjectSetName((PetscObject)subCoordinates, name));
3231     PetscCall(VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE));
3232     PetscCall(VecSetType(subCoordinates, VECSTANDARD));
3233     if (coordSize) {
3234       PetscCall(VecGetArray(coordinates, &coords));
3235       PetscCall(VecGetArray(subCoordinates, &subCoords));
3236       for (v = 0; v < numSubVertices; ++v) {
3237         const PetscInt vertex    = subVertices[v];
3238         const PetscInt subvertex = firstSubVertex + v;
3239         PetscInt       dof, off, sdof, soff, d;
3240 
3241         PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3242         PetscCall(PetscSectionGetOffset(coordSection, vertex, &off));
3243         PetscCall(PetscSectionGetDof(subCoordSection, subvertex, &sdof));
3244         PetscCall(PetscSectionGetOffset(subCoordSection, subvertex, &soff));
3245         PetscCheck(dof == sdof, comm, PETSC_ERR_PLIB, "Coordinate dimension %" PetscInt_FMT " on subvertex %" PetscInt_FMT ", vertex %" PetscInt_FMT " should be %" PetscInt_FMT, sdof, subvertex, vertex, dof);
3246         for (d = 0; d < dof; ++d) subCoords[soff + d] = coords[off + d];
3247       }
3248       PetscCall(VecRestoreArray(coordinates, &coords));
3249       PetscCall(VecRestoreArray(subCoordinates, &subCoords));
3250     }
3251     PetscCall(DMSetCoordinatesLocal(subdm, subCoordinates));
3252     PetscCall(VecDestroy(&subCoordinates));
3253   }
3254   /* Cleanup */
3255   if (subvertexIS) PetscCall(ISRestoreIndices(subvertexIS, &subVertices));
3256   PetscCall(ISDestroy(&subvertexIS));
3257   if (subcellIS) PetscCall(ISRestoreIndices(subcellIS, &subCells));
3258   PetscCall(ISDestroy(&subcellIS));
3259   PetscFunctionReturn(PETSC_SUCCESS);
3260 }
3261 
3262 /* TODO: Fix this to properly propagate up error conditions it may find */
3263 static inline PetscInt DMPlexFilterPoint_Internal(PetscInt point, PetscInt firstSubPoint, PetscInt numSubPoints, const PetscInt subPoints[])
3264 {
3265   PetscInt       subPoint;
3266   PetscErrorCode ierr;
3267 
3268   ierr = PetscFindInt(point, numSubPoints, subPoints, &subPoint);
3269   if (ierr) return -1;
3270   return subPoint < 0 ? subPoint : firstSubPoint + subPoint;
3271 }
3272 
3273 /* TODO: Fix this to properly propagate up error conditions it may find */
3274 static inline PetscInt DMPlexFilterPointPerm_Internal(PetscInt point, PetscInt firstSubPoint, PetscInt numSubPoints, const PetscInt subPoints[], const PetscInt subIndices[])
3275 {
3276   PetscInt       subPoint;
3277   PetscErrorCode ierr;
3278 
3279   ierr = PetscFindInt(point, numSubPoints, subPoints, &subPoint);
3280   if (ierr) return -1;
3281   return subPoint < 0 ? subPoint : firstSubPoint + (subIndices ? subIndices[subPoint] : subPoint);
3282 }
3283 
3284 static PetscErrorCode DMPlexFilterLabels_Internal(DM dm, const PetscInt numSubPoints[], const PetscInt *subpoints[], const PetscInt firstSubPoint[], DM subdm)
3285 {
3286   DMLabel  depthLabel;
3287   PetscInt Nl, l, d;
3288 
3289   PetscFunctionBegin;
3290   // Reset depth label for fast lookup
3291   PetscCall(DMPlexGetDepthLabel(dm, &depthLabel));
3292   PetscCall(DMLabelMakeAllInvalid_Internal(depthLabel));
3293   PetscCall(DMGetNumLabels(dm, &Nl));
3294   for (l = 0; l < Nl; ++l) {
3295     DMLabel         label, newlabel;
3296     const char     *lname;
3297     PetscBool       isDepth, isDim, isCelltype, isVTK;
3298     IS              valueIS;
3299     const PetscInt *values;
3300     PetscInt        Nv, v;
3301 
3302     PetscCall(DMGetLabelName(dm, l, &lname));
3303     PetscCall(PetscStrcmp(lname, "depth", &isDepth));
3304     PetscCall(PetscStrcmp(lname, "dim", &isDim));
3305     PetscCall(PetscStrcmp(lname, "celltype", &isCelltype));
3306     PetscCall(PetscStrcmp(lname, "vtk", &isVTK));
3307     if (isDepth || isDim || isCelltype || isVTK) continue;
3308     PetscCall(DMCreateLabel(subdm, lname));
3309     PetscCall(DMGetLabel(dm, lname, &label));
3310     PetscCall(DMGetLabel(subdm, lname, &newlabel));
3311     PetscCall(DMLabelGetDefaultValue(label, &v));
3312     PetscCall(DMLabelSetDefaultValue(newlabel, v));
3313     PetscCall(DMLabelGetValueIS(label, &valueIS));
3314     PetscCall(ISGetLocalSize(valueIS, &Nv));
3315     PetscCall(ISGetIndices(valueIS, &values));
3316     for (v = 0; v < Nv; ++v) {
3317       IS              pointIS;
3318       const PetscInt *points;
3319       PetscInt        Np, p;
3320 
3321       PetscCall(DMLabelGetStratumIS(label, values[v], &pointIS));
3322       PetscCall(ISGetLocalSize(pointIS, &Np));
3323       PetscCall(ISGetIndices(pointIS, &points));
3324       for (p = 0; p < Np; ++p) {
3325         const PetscInt point = points[p];
3326         PetscInt       subp;
3327 
3328         PetscCall(DMPlexGetPointDepth(dm, point, &d));
3329         subp = DMPlexFilterPoint_Internal(point, firstSubPoint[d], numSubPoints[d], subpoints[d]);
3330         if (subp >= 0) PetscCall(DMLabelSetValue(newlabel, subp, values[v]));
3331       }
3332       PetscCall(ISRestoreIndices(pointIS, &points));
3333       PetscCall(ISDestroy(&pointIS));
3334     }
3335     PetscCall(ISRestoreIndices(valueIS, &values));
3336     PetscCall(ISDestroy(&valueIS));
3337   }
3338   PetscFunctionReturn(PETSC_SUCCESS);
3339 }
3340 
3341 static PetscErrorCode DMPlexCreateSubmeshGeneric_Interpolated(DM dm, DMLabel label, PetscInt value, PetscBool markedFaces, PetscBool isCohesive, PetscInt cellHeight, DM subdm)
3342 {
3343   MPI_Comm         comm;
3344   DMLabel          subpointMap;
3345   IS              *subpointIS;
3346   const PetscInt **subpoints;
3347   PetscInt        *numSubPoints, *firstSubPoint, *coneNew, *orntNew;
3348   PetscInt         totSubPoints = 0, maxConeSize, dim, sdim, cdim, p, d, v;
3349   PetscMPIInt      rank;
3350 
3351   PetscFunctionBegin;
3352   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
3353   PetscCallMPI(MPI_Comm_rank(comm, &rank));
3354   /* Create subpointMap which marks the submesh */
3355   PetscCall(DMLabelCreate(PETSC_COMM_SELF, "subpoint_map", &subpointMap));
3356   PetscCall(DMPlexSetSubpointMap(subdm, subpointMap));
3357   if (cellHeight) {
3358     if (isCohesive) PetscCall(DMPlexMarkCohesiveSubmesh_Interpolated(dm, label, value, subpointMap, subdm));
3359     else PetscCall(DMPlexMarkSubmesh_Interpolated(dm, label, value, markedFaces, subpointMap, subdm));
3360   } else {
3361     DMLabel         depth;
3362     IS              pointIS;
3363     const PetscInt *points;
3364     PetscInt        numPoints = 0;
3365 
3366     PetscCall(DMPlexGetDepthLabel(dm, &depth));
3367     PetscCall(DMLabelGetStratumIS(label, value, &pointIS));
3368     if (pointIS) {
3369       PetscCall(ISGetIndices(pointIS, &points));
3370       PetscCall(ISGetLocalSize(pointIS, &numPoints));
3371     }
3372     for (p = 0; p < numPoints; ++p) {
3373       PetscInt *closure = NULL;
3374       PetscInt  closureSize, c, pdim;
3375 
3376       PetscCall(DMPlexGetTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closure));
3377       for (c = 0; c < closureSize * 2; c += 2) {
3378         PetscCall(DMLabelGetValue(depth, closure[c], &pdim));
3379         PetscCall(DMLabelSetValue(subpointMap, closure[c], pdim));
3380       }
3381       PetscCall(DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closure));
3382     }
3383     if (pointIS) PetscCall(ISRestoreIndices(pointIS, &points));
3384     PetscCall(ISDestroy(&pointIS));
3385   }
3386   /* Setup chart */
3387   PetscCall(DMGetDimension(dm, &dim));
3388   PetscCall(DMGetCoordinateDim(dm, &cdim));
3389   PetscCall(PetscMalloc4(dim + 1, &numSubPoints, dim + 1, &firstSubPoint, dim + 1, &subpointIS, dim + 1, &subpoints));
3390   for (d = 0; d <= dim; ++d) {
3391     PetscCall(DMLabelGetStratumSize(subpointMap, d, &numSubPoints[d]));
3392     totSubPoints += numSubPoints[d];
3393   }
3394   // Determine submesh dimension
3395   PetscCall(DMGetDimension(subdm, &sdim));
3396   if (sdim > 0) {
3397     // Calling function knows what dimension to use, and we include neighboring cells as well
3398     sdim = dim;
3399   } else {
3400     // We reset the subdimension based on what is being selected
3401     PetscInt lsdim;
3402     for (lsdim = dim; lsdim >= 0; --lsdim)
3403       if (numSubPoints[lsdim]) break;
3404     PetscCall(MPIU_Allreduce(&lsdim, &sdim, 1, MPIU_INT, MPI_MAX, comm));
3405     PetscCall(DMSetDimension(subdm, sdim));
3406     PetscCall(DMSetCoordinateDim(subdm, cdim));
3407   }
3408   PetscCall(DMPlexSetChart(subdm, 0, totSubPoints));
3409   PetscCall(DMPlexSetVTKCellHeight(subdm, cellHeight));
3410   /* Set cone sizes */
3411   firstSubPoint[sdim] = 0;
3412   firstSubPoint[0]    = firstSubPoint[sdim] + numSubPoints[sdim];
3413   if (sdim > 1) firstSubPoint[sdim - 1] = firstSubPoint[0] + numSubPoints[0];
3414   if (sdim > 2) firstSubPoint[sdim - 2] = firstSubPoint[sdim - 1] + numSubPoints[sdim - 1];
3415   for (d = 0; d <= sdim; ++d) {
3416     PetscCall(DMLabelGetStratumIS(subpointMap, d, &subpointIS[d]));
3417     if (subpointIS[d]) PetscCall(ISGetIndices(subpointIS[d], &subpoints[d]));
3418   }
3419   /* We do not want this label automatically computed, instead we compute it here */
3420   PetscCall(DMCreateLabel(subdm, "celltype"));
3421   for (d = 0; d <= sdim; ++d) {
3422     for (p = 0; p < numSubPoints[d]; ++p) {
3423       const PetscInt  point    = subpoints[d][p];
3424       const PetscInt  subpoint = firstSubPoint[d] + p;
3425       const PetscInt *cone;
3426       PetscInt        coneSize;
3427 
3428       PetscCall(DMPlexGetConeSize(dm, point, &coneSize));
3429       if (cellHeight && (d == sdim)) {
3430         PetscInt coneSizeNew, c, val;
3431 
3432         PetscCall(DMPlexGetCone(dm, point, &cone));
3433         for (c = 0, coneSizeNew = 0; c < coneSize; ++c) {
3434           PetscCall(DMLabelGetValue(subpointMap, cone[c], &val));
3435           if (val >= 0) coneSizeNew++;
3436         }
3437         PetscCall(DMPlexSetConeSize(subdm, subpoint, coneSizeNew));
3438         PetscCall(DMPlexSetCellType(subdm, subpoint, DM_POLYTOPE_FV_GHOST));
3439       } else {
3440         DMPolytopeType ct;
3441 
3442         PetscCall(DMPlexSetConeSize(subdm, subpoint, coneSize));
3443         PetscCall(DMPlexGetCellType(dm, point, &ct));
3444         PetscCall(DMPlexSetCellType(subdm, subpoint, ct));
3445       }
3446     }
3447   }
3448   PetscCall(DMLabelDestroy(&subpointMap));
3449   PetscCall(DMSetUp(subdm));
3450   /* Set cones */
3451   PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, NULL));
3452   PetscCall(PetscMalloc2(maxConeSize, &coneNew, maxConeSize, &orntNew));
3453   for (d = 0; d <= sdim; ++d) {
3454     for (p = 0; p < numSubPoints[d]; ++p) {
3455       const PetscInt  point    = subpoints[d][p];
3456       const PetscInt  subpoint = firstSubPoint[d] + p;
3457       const PetscInt *cone, *ornt;
3458       PetscInt        coneSize, subconeSize, coneSizeNew, c, subc, fornt = 0;
3459 
3460       if (d == sdim - 1) {
3461         const PetscInt *support, *cone, *ornt;
3462         PetscInt        supportSize, coneSize, s, subc;
3463 
3464         PetscCall(DMPlexGetSupport(dm, point, &support));
3465         PetscCall(DMPlexGetSupportSize(dm, point, &supportSize));
3466         for (s = 0; s < supportSize; ++s) {
3467           PetscBool isHybrid = PETSC_FALSE;
3468 
3469           PetscCall(DMPlexCellIsHybrid_Internal(dm, support[s], &isHybrid));
3470           if (!isHybrid) continue;
3471           PetscCall(PetscFindInt(support[s], numSubPoints[d + 1], subpoints[d + 1], &subc));
3472           if (subc >= 0) {
3473             const PetscInt ccell = subpoints[d + 1][subc];
3474 
3475             PetscCall(DMPlexGetCone(dm, ccell, &cone));
3476             PetscCall(DMPlexGetConeSize(dm, ccell, &coneSize));
3477             PetscCall(DMPlexGetConeOrientation(dm, ccell, &ornt));
3478             for (c = 0; c < coneSize; ++c) {
3479               if (cone[c] == point) {
3480                 fornt = ornt[c];
3481                 break;
3482               }
3483             }
3484             break;
3485           }
3486         }
3487       }
3488       PetscCall(DMPlexGetConeSize(dm, point, &coneSize));
3489       PetscCall(DMPlexGetConeSize(subdm, subpoint, &subconeSize));
3490       PetscCall(DMPlexGetCone(dm, point, &cone));
3491       PetscCall(DMPlexGetConeOrientation(dm, point, &ornt));
3492       for (c = 0, coneSizeNew = 0; c < coneSize; ++c) {
3493         PetscCall(PetscFindInt(cone[c], numSubPoints[d - 1], subpoints[d - 1], &subc));
3494         if (subc >= 0) {
3495           coneNew[coneSizeNew] = firstSubPoint[d - 1] + subc;
3496           orntNew[coneSizeNew] = ornt[c];
3497           ++coneSizeNew;
3498         }
3499       }
3500       PetscCheck(coneSizeNew == subconeSize, comm, PETSC_ERR_PLIB, "Number of cone points located %" PetscInt_FMT " does not match subcone size %" PetscInt_FMT, coneSizeNew, subconeSize);
3501       PetscCall(DMPlexSetCone(subdm, subpoint, coneNew));
3502       PetscCall(DMPlexSetConeOrientation(subdm, subpoint, orntNew));
3503       if (fornt < 0) PetscCall(DMPlexOrientPoint(subdm, subpoint, fornt));
3504     }
3505   }
3506   PetscCall(PetscFree2(coneNew, orntNew));
3507   PetscCall(DMPlexSymmetrize(subdm));
3508   PetscCall(DMPlexStratify(subdm));
3509   /* Build coordinates */
3510   {
3511     PetscSection coordSection, subCoordSection;
3512     Vec          coordinates, subCoordinates;
3513     PetscScalar *coords, *subCoords;
3514     PetscInt     cdim, numComp, coordSize;
3515     const char  *name;
3516 
3517     PetscCall(DMGetCoordinateDim(dm, &cdim));
3518     PetscCall(DMGetCoordinateSection(dm, &coordSection));
3519     PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
3520     PetscCall(DMGetCoordinateSection(subdm, &subCoordSection));
3521     PetscCall(PetscSectionSetNumFields(subCoordSection, 1));
3522     PetscCall(PetscSectionGetFieldComponents(coordSection, 0, &numComp));
3523     PetscCall(PetscSectionSetFieldComponents(subCoordSection, 0, numComp));
3524     PetscCall(PetscSectionSetChart(subCoordSection, firstSubPoint[0], firstSubPoint[0] + numSubPoints[0]));
3525     for (v = 0; v < numSubPoints[0]; ++v) {
3526       const PetscInt vertex    = subpoints[0][v];
3527       const PetscInt subvertex = firstSubPoint[0] + v;
3528       PetscInt       dof;
3529 
3530       PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3531       PetscCall(PetscSectionSetDof(subCoordSection, subvertex, dof));
3532       PetscCall(PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof));
3533     }
3534     PetscCall(PetscSectionSetUp(subCoordSection));
3535     PetscCall(PetscSectionGetStorageSize(subCoordSection, &coordSize));
3536     PetscCall(VecCreate(PETSC_COMM_SELF, &subCoordinates));
3537     PetscCall(PetscObjectGetName((PetscObject)coordinates, &name));
3538     PetscCall(PetscObjectSetName((PetscObject)subCoordinates, name));
3539     PetscCall(VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE));
3540     PetscCall(VecSetBlockSize(subCoordinates, cdim));
3541     PetscCall(VecSetType(subCoordinates, VECSTANDARD));
3542     PetscCall(VecGetArray(coordinates, &coords));
3543     PetscCall(VecGetArray(subCoordinates, &subCoords));
3544     for (v = 0; v < numSubPoints[0]; ++v) {
3545       const PetscInt vertex    = subpoints[0][v];
3546       const PetscInt subvertex = firstSubPoint[0] + v;
3547       PetscInt       dof, off, sdof, soff, d;
3548 
3549       PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3550       PetscCall(PetscSectionGetOffset(coordSection, vertex, &off));
3551       PetscCall(PetscSectionGetDof(subCoordSection, subvertex, &sdof));
3552       PetscCall(PetscSectionGetOffset(subCoordSection, subvertex, &soff));
3553       PetscCheck(dof == sdof, comm, PETSC_ERR_PLIB, "Coordinate dimension %" PetscInt_FMT " on subvertex %" PetscInt_FMT ", vertex %" PetscInt_FMT " should be %" PetscInt_FMT, sdof, subvertex, vertex, dof);
3554       for (d = 0; d < dof; ++d) subCoords[soff + d] = coords[off + d];
3555     }
3556     PetscCall(VecRestoreArray(coordinates, &coords));
3557     PetscCall(VecRestoreArray(subCoordinates, &subCoords));
3558     PetscCall(DMSetCoordinatesLocal(subdm, subCoordinates));
3559     PetscCall(VecDestroy(&subCoordinates));
3560   }
3561   /* Build SF: We need this complexity because subpoints might not be selected on the owning process */
3562   {
3563     PetscSF            sfPoint, sfPointSub;
3564     IS                 subpIS;
3565     const PetscSFNode *remotePoints;
3566     PetscSFNode       *sremotePoints = NULL, *newLocalPoints = NULL, *newOwners = NULL;
3567     const PetscInt    *localPoints, *subpoints, *rootdegree;
3568     PetscInt          *slocalPoints = NULL, *sortedPoints = NULL, *sortedIndices = NULL;
3569     PetscInt           numRoots, numLeaves, numSubpoints = 0, numSubroots, numSubleaves = 0, l, sl = 0, ll = 0, pStart, pEnd, p;
3570     PetscMPIInt        rank, size;
3571 
3572     PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
3573     PetscCallMPI(MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size));
3574     PetscCall(DMGetPointSF(dm, &sfPoint));
3575     PetscCall(DMGetPointSF(subdm, &sfPointSub));
3576     PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
3577     PetscCall(DMPlexGetChart(subdm, NULL, &numSubroots));
3578     PetscCall(DMPlexGetSubpointIS(subdm, &subpIS));
3579     if (subpIS) {
3580       PetscBool sorted = PETSC_TRUE;
3581 
3582       PetscCall(ISGetIndices(subpIS, &subpoints));
3583       PetscCall(ISGetLocalSize(subpIS, &numSubpoints));
3584       for (p = 1; p < numSubpoints; ++p) sorted = sorted && (subpoints[p] >= subpoints[p - 1]) ? PETSC_TRUE : PETSC_FALSE;
3585       if (!sorted) {
3586         PetscCall(PetscMalloc2(numSubpoints, &sortedPoints, numSubpoints, &sortedIndices));
3587         for (p = 0; p < numSubpoints; ++p) sortedIndices[p] = p;
3588         PetscCall(PetscArraycpy(sortedPoints, subpoints, numSubpoints));
3589         PetscCall(PetscSortIntWithArray(numSubpoints, sortedPoints, sortedIndices));
3590       }
3591     }
3592     PetscCall(PetscSFGetGraph(sfPoint, &numRoots, &numLeaves, &localPoints, &remotePoints));
3593     if (numRoots >= 0) {
3594       PetscCall(PetscSFComputeDegreeBegin(sfPoint, &rootdegree));
3595       PetscCall(PetscSFComputeDegreeEnd(sfPoint, &rootdegree));
3596       PetscCall(PetscMalloc2(pEnd - pStart, &newLocalPoints, numRoots, &newOwners));
3597       for (p = 0; p < pEnd - pStart; ++p) {
3598         newLocalPoints[p].rank  = -2;
3599         newLocalPoints[p].index = -2;
3600       }
3601       /* Set subleaves */
3602       for (l = 0; l < numLeaves; ++l) {
3603         const PetscInt point    = localPoints[l];
3604         const PetscInt subpoint = DMPlexFilterPointPerm_Internal(point, 0, numSubpoints, sortedPoints ? sortedPoints : subpoints, sortedIndices);
3605 
3606         if (subpoint < 0) continue;
3607         newLocalPoints[point - pStart].rank  = rank;
3608         newLocalPoints[point - pStart].index = subpoint;
3609         ++numSubleaves;
3610       }
3611       /* Must put in owned subpoints */
3612       for (p = pStart; p < pEnd; ++p) {
3613         newOwners[p - pStart].rank  = -3;
3614         newOwners[p - pStart].index = -3;
3615       }
3616       for (p = 0; p < numSubpoints; ++p) {
3617         /* Hold on to currently owned points */
3618         if (rootdegree[subpoints[p] - pStart]) newOwners[subpoints[p] - pStart].rank = rank + size;
3619         else newOwners[subpoints[p] - pStart].rank = rank;
3620         newOwners[subpoints[p] - pStart].index = p;
3621       }
3622       PetscCall(PetscSFReduceBegin(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC));
3623       PetscCall(PetscSFReduceEnd(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC));
3624       for (p = pStart; p < pEnd; ++p)
3625         if (newOwners[p - pStart].rank >= size) newOwners[p - pStart].rank -= size;
3626       PetscCall(PetscSFBcastBegin(sfPoint, MPIU_2INT, newOwners, newLocalPoints, MPI_REPLACE));
3627       PetscCall(PetscSFBcastEnd(sfPoint, MPIU_2INT, newOwners, newLocalPoints, MPI_REPLACE));
3628       PetscCall(PetscMalloc1(numSubleaves, &slocalPoints));
3629       PetscCall(PetscMalloc1(numSubleaves, &sremotePoints));
3630       for (l = 0; l < numLeaves; ++l) {
3631         const PetscInt point    = localPoints[l];
3632         const PetscInt subpoint = DMPlexFilterPointPerm_Internal(point, 0, numSubpoints, sortedPoints ? sortedPoints : subpoints, sortedIndices);
3633 
3634         if (subpoint < 0) continue;
3635         if (newLocalPoints[point].rank == rank) {
3636           ++ll;
3637           continue;
3638         }
3639         slocalPoints[sl]        = subpoint;
3640         sremotePoints[sl].rank  = newLocalPoints[point].rank;
3641         sremotePoints[sl].index = newLocalPoints[point].index;
3642         PetscCheck(sremotePoints[sl].rank >= 0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote rank for local point %" PetscInt_FMT, point);
3643         PetscCheck(sremotePoints[sl].index >= 0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote subpoint for local point %" PetscInt_FMT, point);
3644         ++sl;
3645       }
3646       PetscCheck(sl + ll == numSubleaves, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatch in number of subleaves %" PetscInt_FMT " + %" PetscInt_FMT " != %" PetscInt_FMT, sl, ll, numSubleaves);
3647       PetscCall(PetscFree2(newLocalPoints, newOwners));
3648       PetscCall(PetscSFSetGraph(sfPointSub, numSubroots, sl, slocalPoints, PETSC_OWN_POINTER, sremotePoints, PETSC_OWN_POINTER));
3649     }
3650     if (subpIS) PetscCall(ISRestoreIndices(subpIS, &subpoints));
3651     PetscCall(PetscFree2(sortedPoints, sortedIndices));
3652   }
3653   /* Filter labels */
3654   PetscCall(DMPlexFilterLabels_Internal(dm, numSubPoints, subpoints, firstSubPoint, subdm));
3655   /* Cleanup */
3656   for (d = 0; d <= sdim; ++d) {
3657     if (subpointIS[d]) PetscCall(ISRestoreIndices(subpointIS[d], &subpoints[d]));
3658     PetscCall(ISDestroy(&subpointIS[d]));
3659   }
3660   PetscCall(PetscFree4(numSubPoints, firstSubPoint, subpointIS, subpoints));
3661   PetscFunctionReturn(PETSC_SUCCESS);
3662 }
3663 
3664 static PetscErrorCode DMPlexCreateSubmesh_Interpolated(DM dm, DMLabel vertexLabel, PetscInt value, PetscBool markedFaces, DM subdm)
3665 {
3666   PetscFunctionBegin;
3667   PetscCall(DMPlexCreateSubmeshGeneric_Interpolated(dm, vertexLabel, value, markedFaces, PETSC_FALSE, 1, subdm));
3668   PetscFunctionReturn(PETSC_SUCCESS);
3669 }
3670 
3671 /*@
3672   DMPlexCreateSubmesh - Extract a hypersurface from the mesh using vertices defined by a label
3673 
3674   Input Parameters:
3675 + dm          - The original mesh
3676 . vertexLabel - The `DMLabel` marking points contained in the surface
3677 . value       - The label value to use
3678 - markedFaces - `PETSC_TRUE` if surface faces are marked in addition to vertices, `PETSC_FALSE` if only vertices are marked
3679 
3680   Output Parameter:
3681 . subdm - The surface mesh
3682 
3683   Level: developer
3684 
3685   Note:
3686   This function produces a `DMLabel` mapping original points in the submesh to their depth. This can be obtained using `DMPlexGetSubpointMap()`.
3687 
3688 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetSubpointMap()`, `DMGetLabel()`, `DMLabelSetValue()`
3689 @*/
3690 PetscErrorCode DMPlexCreateSubmesh(DM dm, DMLabel vertexLabel, PetscInt value, PetscBool markedFaces, DM *subdm)
3691 {
3692   DMPlexInterpolatedFlag interpolated;
3693   PetscInt               dim, cdim;
3694 
3695   PetscFunctionBegin;
3696   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3697   PetscAssertPointer(subdm, 5);
3698   PetscCall(DMGetDimension(dm, &dim));
3699   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), subdm));
3700   PetscCall(DMSetType(*subdm, DMPLEX));
3701   PetscCall(DMSetDimension(*subdm, dim - 1));
3702   PetscCall(DMGetCoordinateDim(dm, &cdim));
3703   PetscCall(DMSetCoordinateDim(*subdm, cdim));
3704   PetscCall(DMPlexIsInterpolated(dm, &interpolated));
3705   PetscCheck(interpolated != DMPLEX_INTERPOLATED_PARTIAL, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Not for partially interpolated meshes");
3706   if (interpolated) {
3707     PetscCall(DMPlexCreateSubmesh_Interpolated(dm, vertexLabel, value, markedFaces, *subdm));
3708   } else {
3709     PetscCall(DMPlexCreateSubmesh_Uninterpolated(dm, vertexLabel, value, *subdm));
3710   }
3711   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_TRUE, *subdm));
3712   PetscFunctionReturn(PETSC_SUCCESS);
3713 }
3714 
3715 static PetscErrorCode DMPlexCreateCohesiveSubmesh_Uninterpolated(DM dm, PetscBool hasLagrange, const char label[], PetscInt value, DM subdm)
3716 {
3717   MPI_Comm        comm;
3718   DMLabel         subpointMap;
3719   IS              subvertexIS;
3720   const PetscInt *subVertices;
3721   PetscInt        numSubVertices, firstSubVertex, numSubCells, *subCells = NULL;
3722   PetscInt       *subface, maxConeSize, numSubFaces, firstSubFace, newFacePoint, nFV;
3723   PetscInt        c, f;
3724 
3725   PetscFunctionBegin;
3726   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
3727   /* Create subpointMap which marks the submesh */
3728   PetscCall(DMLabelCreate(PETSC_COMM_SELF, "subpoint_map", &subpointMap));
3729   PetscCall(DMPlexSetSubpointMap(subdm, subpointMap));
3730   PetscCall(DMPlexMarkCohesiveSubmesh_Uninterpolated(dm, hasLagrange, label, value, subpointMap, &numSubFaces, &nFV, &subCells, subdm));
3731   /* Setup chart */
3732   PetscCall(DMLabelGetStratumSize(subpointMap, 0, &numSubVertices));
3733   PetscCall(DMLabelGetStratumSize(subpointMap, 2, &numSubCells));
3734   PetscCall(DMPlexSetChart(subdm, 0, numSubCells + numSubFaces + numSubVertices));
3735   PetscCall(DMPlexSetVTKCellHeight(subdm, 1));
3736   /* Set cone sizes */
3737   firstSubVertex = numSubCells;
3738   firstSubFace   = numSubCells + numSubVertices;
3739   newFacePoint   = firstSubFace;
3740   PetscCall(DMLabelGetStratumIS(subpointMap, 0, &subvertexIS));
3741   if (subvertexIS) PetscCall(ISGetIndices(subvertexIS, &subVertices));
3742   for (c = 0; c < numSubCells; ++c) PetscCall(DMPlexSetConeSize(subdm, c, 1));
3743   for (f = firstSubFace; f < firstSubFace + numSubFaces; ++f) PetscCall(DMPlexSetConeSize(subdm, f, nFV));
3744   PetscCall(DMSetUp(subdm));
3745   PetscCall(DMLabelDestroy(&subpointMap));
3746   /* Create face cones */
3747   PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, NULL));
3748   PetscCall(DMGetWorkArray(subdm, maxConeSize, MPIU_INT, (void **)&subface));
3749   for (c = 0; c < numSubCells; ++c) {
3750     const PetscInt  cell    = subCells[c];
3751     const PetscInt  subcell = c;
3752     const PetscInt *cone, *cells;
3753     PetscBool       isHybrid = PETSC_FALSE;
3754     PetscInt        numCells, subVertex, p, v;
3755 
3756     PetscCall(DMPlexCellIsHybrid_Internal(dm, cell, &isHybrid));
3757     if (!isHybrid) continue;
3758     PetscCall(DMPlexGetCone(dm, cell, &cone));
3759     for (v = 0; v < nFV; ++v) {
3760       PetscCall(PetscFindInt(cone[v], numSubVertices, subVertices, &subVertex));
3761       subface[v] = firstSubVertex + subVertex;
3762     }
3763     PetscCall(DMPlexSetCone(subdm, newFacePoint, subface));
3764     PetscCall(DMPlexSetCone(subdm, subcell, &newFacePoint));
3765     PetscCall(DMPlexGetJoin(dm, nFV, cone, &numCells, &cells));
3766     /* Not true in parallel
3767     PetscCheck(numCells == 2,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive cells should separate two cells"); */
3768     for (p = 0; p < numCells; ++p) {
3769       PetscInt  negsubcell;
3770       PetscBool isHybrid = PETSC_FALSE;
3771 
3772       PetscCall(DMPlexCellIsHybrid_Internal(dm, cells[p], &isHybrid));
3773       if (isHybrid) continue;
3774       /* I know this is a crap search */
3775       for (negsubcell = 0; negsubcell < numSubCells; ++negsubcell) {
3776         if (subCells[negsubcell] == cells[p]) break;
3777       }
3778       PetscCheck(negsubcell != numSubCells, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not find negative face neighbor for cohesive cell %" PetscInt_FMT, cell);
3779       PetscCall(DMPlexSetCone(subdm, negsubcell, &newFacePoint));
3780     }
3781     PetscCall(DMPlexRestoreJoin(dm, nFV, cone, &numCells, &cells));
3782     ++newFacePoint;
3783   }
3784   PetscCall(DMRestoreWorkArray(subdm, maxConeSize, MPIU_INT, (void **)&subface));
3785   PetscCall(DMPlexSymmetrize(subdm));
3786   PetscCall(DMPlexStratify(subdm));
3787   /* Build coordinates */
3788   {
3789     PetscSection coordSection, subCoordSection;
3790     Vec          coordinates, subCoordinates;
3791     PetscScalar *coords, *subCoords;
3792     PetscInt     cdim, numComp, coordSize, v;
3793     const char  *name;
3794 
3795     PetscCall(DMGetCoordinateDim(dm, &cdim));
3796     PetscCall(DMGetCoordinateSection(dm, &coordSection));
3797     PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
3798     PetscCall(DMGetCoordinateSection(subdm, &subCoordSection));
3799     PetscCall(PetscSectionSetNumFields(subCoordSection, 1));
3800     PetscCall(PetscSectionGetFieldComponents(coordSection, 0, &numComp));
3801     PetscCall(PetscSectionSetFieldComponents(subCoordSection, 0, numComp));
3802     PetscCall(PetscSectionSetChart(subCoordSection, firstSubVertex, firstSubVertex + numSubVertices));
3803     for (v = 0; v < numSubVertices; ++v) {
3804       const PetscInt vertex    = subVertices[v];
3805       const PetscInt subvertex = firstSubVertex + v;
3806       PetscInt       dof;
3807 
3808       PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3809       PetscCall(PetscSectionSetDof(subCoordSection, subvertex, dof));
3810       PetscCall(PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof));
3811     }
3812     PetscCall(PetscSectionSetUp(subCoordSection));
3813     PetscCall(PetscSectionGetStorageSize(subCoordSection, &coordSize));
3814     PetscCall(VecCreate(PETSC_COMM_SELF, &subCoordinates));
3815     PetscCall(PetscObjectGetName((PetscObject)coordinates, &name));
3816     PetscCall(PetscObjectSetName((PetscObject)subCoordinates, name));
3817     PetscCall(VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE));
3818     PetscCall(VecSetBlockSize(subCoordinates, cdim));
3819     PetscCall(VecSetType(subCoordinates, VECSTANDARD));
3820     PetscCall(VecGetArray(coordinates, &coords));
3821     PetscCall(VecGetArray(subCoordinates, &subCoords));
3822     for (v = 0; v < numSubVertices; ++v) {
3823       const PetscInt vertex    = subVertices[v];
3824       const PetscInt subvertex = firstSubVertex + v;
3825       PetscInt       dof, off, sdof, soff, d;
3826 
3827       PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3828       PetscCall(PetscSectionGetOffset(coordSection, vertex, &off));
3829       PetscCall(PetscSectionGetDof(subCoordSection, subvertex, &sdof));
3830       PetscCall(PetscSectionGetOffset(subCoordSection, subvertex, &soff));
3831       PetscCheck(dof == sdof, comm, PETSC_ERR_PLIB, "Coordinate dimension %" PetscInt_FMT " on subvertex %" PetscInt_FMT ", vertex %" PetscInt_FMT " should be %" PetscInt_FMT, sdof, subvertex, vertex, dof);
3832       for (d = 0; d < dof; ++d) subCoords[soff + d] = coords[off + d];
3833     }
3834     PetscCall(VecRestoreArray(coordinates, &coords));
3835     PetscCall(VecRestoreArray(subCoordinates, &subCoords));
3836     PetscCall(DMSetCoordinatesLocal(subdm, subCoordinates));
3837     PetscCall(VecDestroy(&subCoordinates));
3838   }
3839   /* Build SF */
3840   CHKMEMQ;
3841   {
3842     PetscSF            sfPoint, sfPointSub;
3843     const PetscSFNode *remotePoints;
3844     PetscSFNode       *sremotePoints, *newLocalPoints, *newOwners;
3845     const PetscInt    *localPoints;
3846     PetscInt          *slocalPoints;
3847     PetscInt           numRoots, numLeaves, numSubRoots = numSubCells + numSubFaces + numSubVertices, numSubLeaves = 0, l, sl, ll, pStart, pEnd, p, vStart, vEnd;
3848     PetscMPIInt        rank;
3849 
3850     PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
3851     PetscCall(DMGetPointSF(dm, &sfPoint));
3852     PetscCall(DMGetPointSF(subdm, &sfPointSub));
3853     PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
3854     PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
3855     PetscCall(PetscSFGetGraph(sfPoint, &numRoots, &numLeaves, &localPoints, &remotePoints));
3856     if (numRoots >= 0) {
3857       /* Only vertices should be shared */
3858       PetscCall(PetscMalloc2(pEnd - pStart, &newLocalPoints, numRoots, &newOwners));
3859       for (p = 0; p < pEnd - pStart; ++p) {
3860         newLocalPoints[p].rank  = -2;
3861         newLocalPoints[p].index = -2;
3862       }
3863       /* Set subleaves */
3864       for (l = 0; l < numLeaves; ++l) {
3865         const PetscInt point    = localPoints[l];
3866         const PetscInt subPoint = DMPlexFilterPoint_Internal(point, firstSubVertex, numSubVertices, subVertices);
3867 
3868         PetscCheck(!(point < vStart) || !(point >= vEnd), PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Should not be mapping anything but vertices, %" PetscInt_FMT, point);
3869         if (subPoint < 0) continue;
3870         newLocalPoints[point - pStart].rank  = rank;
3871         newLocalPoints[point - pStart].index = subPoint;
3872         ++numSubLeaves;
3873       }
3874       /* Must put in owned subpoints */
3875       for (p = pStart; p < pEnd; ++p) {
3876         const PetscInt subPoint = DMPlexFilterPoint_Internal(p, firstSubVertex, numSubVertices, subVertices);
3877 
3878         if (subPoint < 0) {
3879           newOwners[p - pStart].rank  = -3;
3880           newOwners[p - pStart].index = -3;
3881         } else {
3882           newOwners[p - pStart].rank  = rank;
3883           newOwners[p - pStart].index = subPoint;
3884         }
3885       }
3886       PetscCall(PetscSFReduceBegin(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC));
3887       PetscCall(PetscSFReduceEnd(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC));
3888       PetscCall(PetscSFBcastBegin(sfPoint, MPIU_2INT, newOwners, newLocalPoints, MPI_REPLACE));
3889       PetscCall(PetscSFBcastEnd(sfPoint, MPIU_2INT, newOwners, newLocalPoints, MPI_REPLACE));
3890       PetscCall(PetscMalloc1(numSubLeaves, &slocalPoints));
3891       PetscCall(PetscMalloc1(numSubLeaves, &sremotePoints));
3892       for (l = 0, sl = 0, ll = 0; l < numLeaves; ++l) {
3893         const PetscInt point    = localPoints[l];
3894         const PetscInt subPoint = DMPlexFilterPoint_Internal(point, firstSubVertex, numSubVertices, subVertices);
3895 
3896         if (subPoint < 0) continue;
3897         if (newLocalPoints[point].rank == rank) {
3898           ++ll;
3899           continue;
3900         }
3901         slocalPoints[sl]        = subPoint;
3902         sremotePoints[sl].rank  = newLocalPoints[point].rank;
3903         sremotePoints[sl].index = newLocalPoints[point].index;
3904         PetscCheck(sremotePoints[sl].rank >= 0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote rank for local point %" PetscInt_FMT, point);
3905         PetscCheck(sremotePoints[sl].index >= 0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote subpoint for local point %" PetscInt_FMT, point);
3906         ++sl;
3907       }
3908       PetscCall(PetscFree2(newLocalPoints, newOwners));
3909       PetscCheck(sl + ll == numSubLeaves, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatch in number of subleaves %" PetscInt_FMT " + %" PetscInt_FMT " != %" PetscInt_FMT, sl, ll, numSubLeaves);
3910       PetscCall(PetscSFSetGraph(sfPointSub, numSubRoots, sl, slocalPoints, PETSC_OWN_POINTER, sremotePoints, PETSC_OWN_POINTER));
3911     }
3912   }
3913   CHKMEMQ;
3914   /* Cleanup */
3915   if (subvertexIS) PetscCall(ISRestoreIndices(subvertexIS, &subVertices));
3916   PetscCall(ISDestroy(&subvertexIS));
3917   PetscCall(PetscFree(subCells));
3918   PetscFunctionReturn(PETSC_SUCCESS);
3919 }
3920 
3921 static PetscErrorCode DMPlexCreateCohesiveSubmesh_Interpolated(DM dm, const char labelname[], PetscInt value, DM subdm)
3922 {
3923   DMLabel label = NULL;
3924 
3925   PetscFunctionBegin;
3926   if (labelname) PetscCall(DMGetLabel(dm, labelname, &label));
3927   PetscCall(DMPlexCreateSubmeshGeneric_Interpolated(dm, label, value, PETSC_FALSE, PETSC_TRUE, 1, subdm));
3928   PetscFunctionReturn(PETSC_SUCCESS);
3929 }
3930 
3931 /*@C
3932   DMPlexCreateCohesiveSubmesh - Extract from a mesh with cohesive cells the hypersurface defined by one face of the cells. Optionally, a label can be given to restrict the cells.
3933 
3934   Input Parameters:
3935 + dm          - The original mesh
3936 . hasLagrange - The mesh has Lagrange unknowns in the cohesive cells
3937 . label       - A label name, or `NULL`
3938 - value       - A label value
3939 
3940   Output Parameter:
3941 . subdm - The surface mesh
3942 
3943   Level: developer
3944 
3945   Note:
3946   This function produces a `DMLabel` mapping original points in the submesh to their depth. This can be obtained using `DMPlexGetSubpointMap()`.
3947 
3948 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetSubpointMap()`, `DMPlexCreateSubmesh()`
3949 @*/
3950 PetscErrorCode DMPlexCreateCohesiveSubmesh(DM dm, PetscBool hasLagrange, const char label[], PetscInt value, DM *subdm)
3951 {
3952   PetscInt dim, cdim, depth;
3953 
3954   PetscFunctionBegin;
3955   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3956   PetscAssertPointer(subdm, 5);
3957   PetscCall(DMGetDimension(dm, &dim));
3958   PetscCall(DMPlexGetDepth(dm, &depth));
3959   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), subdm));
3960   PetscCall(DMSetType(*subdm, DMPLEX));
3961   PetscCall(DMSetDimension(*subdm, dim - 1));
3962   PetscCall(DMGetCoordinateDim(dm, &cdim));
3963   PetscCall(DMSetCoordinateDim(*subdm, cdim));
3964   if (depth == dim) {
3965     PetscCall(DMPlexCreateCohesiveSubmesh_Interpolated(dm, label, value, *subdm));
3966   } else {
3967     PetscCall(DMPlexCreateCohesiveSubmesh_Uninterpolated(dm, hasLagrange, label, value, *subdm));
3968   }
3969   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_TRUE, *subdm));
3970   PetscFunctionReturn(PETSC_SUCCESS);
3971 }
3972 
3973 /*@
3974   DMPlexReorderCohesiveSupports - Ensure that face supports for cohesive end caps are ordered
3975 
3976   Not Collective
3977 
3978   Input Parameter:
3979 . dm - The `DM` containing cohesive cells
3980 
3981   Level: developer
3982 
3983   Note:
3984   For the negative size (first) face, the cohesive cell should be first in the support, and for
3985   the positive side (second) face, the cohesive cell should be second in the support.
3986 
3987 .seealso: `DMPlexConstructCohesiveCells()`, `DMPlexCreateCohesiveSubmesh()`
3988 @*/
3989 PetscErrorCode DMPlexReorderCohesiveSupports(DM dm)
3990 {
3991   PetscInt dim, cStart, cEnd;
3992 
3993   PetscFunctionBegin;
3994   PetscCall(DMGetDimension(dm, &dim));
3995   PetscCall(DMPlexGetTensorPrismBounds_Internal(dm, dim, &cStart, &cEnd));
3996   for (PetscInt c = cStart; c < cEnd; ++c) {
3997     const PetscInt *cone;
3998     PetscInt        coneSize;
3999 
4000     PetscCall(DMPlexGetConeSize(dm, c, &coneSize));
4001     PetscCall(DMPlexGetCone(dm, c, &cone));
4002     PetscCheck(coneSize >= 2, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Hybrid cell %" PetscInt_FMT " cone size %" PetscInt_FMT " must be >= 2", c, coneSize);
4003     for (PetscInt s = 0; s < 2; ++s) {
4004       const PetscInt *supp;
4005       PetscInt        suppSize, newsupp[2];
4006 
4007       PetscCall(DMPlexGetSupportSize(dm, cone[s], &suppSize));
4008       PetscCall(DMPlexGetSupport(dm, cone[s], &supp));
4009       if (suppSize == 2) {
4010         /* Reorder hybrid end cap support */
4011         if (supp[s] == c) {
4012           newsupp[0] = supp[1];
4013           newsupp[1] = supp[0];
4014         } else {
4015           newsupp[0] = supp[0];
4016           newsupp[1] = supp[1];
4017         }
4018         PetscCheck(newsupp[1 - s] == c, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Hybrid end cap %" PetscInt_FMT " support entry %" PetscInt_FMT " != %" PetscInt_FMT " hybrid cell", cone[s], newsupp[1 - s], c);
4019         PetscCall(DMPlexSetSupport(dm, cone[s], newsupp));
4020       }
4021     }
4022   }
4023   PetscFunctionReturn(PETSC_SUCCESS);
4024 }
4025 
4026 /*@
4027   DMPlexFilter - Extract a subset of mesh cells defined by a label as a separate mesh
4028 
4029   Input Parameters:
4030 + dm        - The original mesh
4031 . cellLabel - The `DMLabel` marking cells contained in the new mesh
4032 - value     - The label value to use
4033 
4034   Output Parameter:
4035 . subdm - The new mesh
4036 
4037   Level: developer
4038 
4039   Note:
4040   This function produces a `DMLabel` mapping original points in the submesh to their depth. This can be obtained using `DMPlexGetSubpointMap()`.
4041 
4042 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetSubpointMap()`, `DMGetLabel()`, `DMLabelSetValue()`, `DMPlexCreateSubmesh()`
4043 @*/
4044 PetscErrorCode DMPlexFilter(DM dm, DMLabel cellLabel, PetscInt value, DM *subdm)
4045 {
4046   PetscInt dim, overlap;
4047 
4048   PetscFunctionBegin;
4049   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4050   PetscAssertPointer(subdm, 4);
4051   PetscCall(DMGetDimension(dm, &dim));
4052   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), subdm));
4053   PetscCall(DMSetType(*subdm, DMPLEX));
4054   /* Extract submesh in place, could be empty on some procs, could have inconsistency if procs do not both extract a shared cell */
4055   PetscCall(DMPlexCreateSubmeshGeneric_Interpolated(dm, cellLabel, value, PETSC_FALSE, PETSC_FALSE, 0, *subdm));
4056   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_TRUE, *subdm));
4057   // It is possible to obtain a surface mesh where some faces are in SF
4058   //   We should either mark the mesh as having an overlap, or delete these from the SF
4059   PetscCall(DMPlexGetOverlap(dm, &overlap));
4060   if (!overlap) {
4061     PetscSF         sf;
4062     const PetscInt *leaves;
4063     PetscInt        cStart, cEnd, Nl;
4064     PetscBool       hasSubcell = PETSC_FALSE, ghasSubcell;
4065 
4066     PetscCall(DMPlexGetHeightStratum(*subdm, 0, &cStart, &cEnd));
4067     PetscCall(DMGetPointSF(*subdm, &sf));
4068     PetscCall(PetscSFGetGraph(sf, NULL, &Nl, &leaves, NULL));
4069     for (PetscInt l = 0; l < Nl; ++l) {
4070       const PetscInt point = leaves ? leaves[l] : l;
4071 
4072       if (point >= cStart && point < cEnd) {
4073         hasSubcell = PETSC_TRUE;
4074         break;
4075       }
4076     }
4077     PetscCall(MPIU_Allreduce(&hasSubcell, &ghasSubcell, 1, MPIU_BOOL, MPI_LOR, PetscObjectComm((PetscObject)dm)));
4078     if (ghasSubcell) PetscCall(DMPlexSetOverlap(*subdm, NULL, 1));
4079   }
4080   PetscFunctionReturn(PETSC_SUCCESS);
4081 }
4082 
4083 /*@
4084   DMPlexGetSubpointMap - Returns a `DMLabel` with point dimension as values
4085 
4086   Input Parameter:
4087 . dm - The submesh `DM`
4088 
4089   Output Parameter:
4090 . subpointMap - The `DMLabel` of all the points from the original mesh in this submesh, or `NULL` if this is not a submesh
4091 
4092   Level: developer
4093 
4094 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateSubmesh()`, `DMPlexGetSubpointIS()`
4095 @*/
4096 PetscErrorCode DMPlexGetSubpointMap(DM dm, DMLabel *subpointMap)
4097 {
4098   PetscFunctionBegin;
4099   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4100   PetscAssertPointer(subpointMap, 2);
4101   *subpointMap = ((DM_Plex *)dm->data)->subpointMap;
4102   PetscFunctionReturn(PETSC_SUCCESS);
4103 }
4104 
4105 /*@
4106   DMPlexSetSubpointMap - Sets the `DMLabel` with point dimension as values
4107 
4108   Input Parameters:
4109 + dm          - The submesh `DM`
4110 - subpointMap - The `DMLabel` of all the points from the original mesh in this submesh
4111 
4112   Level: developer
4113 
4114   Note:
4115   Should normally not be called by the user, since it is set in `DMPlexCreateSubmesh()`
4116 
4117 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateSubmesh()`, `DMPlexGetSubpointIS()`
4118 @*/
4119 PetscErrorCode DMPlexSetSubpointMap(DM dm, DMLabel subpointMap)
4120 {
4121   DM_Plex *mesh = (DM_Plex *)dm->data;
4122   DMLabel  tmp;
4123 
4124   PetscFunctionBegin;
4125   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4126   tmp               = mesh->subpointMap;
4127   mesh->subpointMap = subpointMap;
4128   PetscCall(PetscObjectReference((PetscObject)mesh->subpointMap));
4129   PetscCall(DMLabelDestroy(&tmp));
4130   PetscFunctionReturn(PETSC_SUCCESS);
4131 }
4132 
4133 static PetscErrorCode DMPlexCreateSubpointIS_Internal(DM dm, IS *subpointIS)
4134 {
4135   DMLabel  spmap;
4136   PetscInt depth, d;
4137 
4138   PetscFunctionBegin;
4139   PetscCall(DMPlexGetSubpointMap(dm, &spmap));
4140   PetscCall(DMPlexGetDepth(dm, &depth));
4141   if (spmap && depth >= 0) {
4142     DM_Plex  *mesh = (DM_Plex *)dm->data;
4143     PetscInt *points, *depths;
4144     PetscInt  pStart, pEnd, p, off;
4145 
4146     PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
4147     PetscCheck(!pStart, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Submeshes must start the point numbering at 0, not %" PetscInt_FMT, pStart);
4148     PetscCall(PetscMalloc1(pEnd, &points));
4149     PetscCall(DMGetWorkArray(dm, depth + 1, MPIU_INT, &depths));
4150     depths[0] = depth;
4151     depths[1] = 0;
4152     for (d = 2; d <= depth; ++d) depths[d] = depth + 1 - d;
4153     for (d = 0, off = 0; d <= depth; ++d) {
4154       const PetscInt dep = depths[d];
4155       PetscInt       depStart, depEnd, n;
4156 
4157       PetscCall(DMPlexGetDepthStratum(dm, dep, &depStart, &depEnd));
4158       PetscCall(DMLabelGetStratumSize(spmap, dep, &n));
4159       if (((d < 2) && (depth > 1)) || (d == 1)) { /* Only check vertices and cells for now since the map is broken for others */
4160         PetscCheck(n == depEnd - depStart, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "The number of mapped submesh points %" PetscInt_FMT " at depth %" PetscInt_FMT " should be %" PetscInt_FMT, n, dep, depEnd - depStart);
4161       } else {
4162         if (!n) {
4163           if (d == 0) {
4164             /* Missing cells */
4165             for (p = 0; p < depEnd - depStart; ++p, ++off) points[off] = -1;
4166           } else {
4167             /* Missing faces */
4168             for (p = 0; p < depEnd - depStart; ++p, ++off) points[off] = PETSC_MAX_INT;
4169           }
4170         }
4171       }
4172       if (n) {
4173         IS              is;
4174         const PetscInt *opoints;
4175 
4176         PetscCall(DMLabelGetStratumIS(spmap, dep, &is));
4177         PetscCall(ISGetIndices(is, &opoints));
4178         for (p = 0; p < n; ++p, ++off) points[off] = opoints[p];
4179         PetscCall(ISRestoreIndices(is, &opoints));
4180         PetscCall(ISDestroy(&is));
4181       }
4182     }
4183     PetscCall(DMRestoreWorkArray(dm, depth + 1, MPIU_INT, &depths));
4184     PetscCheck(off == pEnd, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "The number of mapped submesh points %" PetscInt_FMT " should be %" PetscInt_FMT, off, pEnd);
4185     PetscCall(ISCreateGeneral(PETSC_COMM_SELF, pEnd, points, PETSC_OWN_POINTER, subpointIS));
4186     PetscCall(PetscObjectStateGet((PetscObject)spmap, &mesh->subpointState));
4187   }
4188   PetscFunctionReturn(PETSC_SUCCESS);
4189 }
4190 
4191 /*@
4192   DMPlexGetSubpointIS - Returns an `IS` covering the entire subdm chart with the original points as data
4193 
4194   Input Parameter:
4195 . dm - The submesh `DM`
4196 
4197   Output Parameter:
4198 . subpointIS - The `IS` of all the points from the original mesh in this submesh, or `NULL` if this is not a submesh
4199 
4200   Level: developer
4201 
4202   Note:
4203   This `IS` is guaranteed to be sorted by the construction of the submesh
4204 
4205 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateSubmesh()`, `DMPlexGetSubpointMap()`
4206 @*/
4207 PetscErrorCode DMPlexGetSubpointIS(DM dm, IS *subpointIS)
4208 {
4209   DM_Plex         *mesh = (DM_Plex *)dm->data;
4210   DMLabel          spmap;
4211   PetscObjectState state;
4212 
4213   PetscFunctionBegin;
4214   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4215   PetscAssertPointer(subpointIS, 2);
4216   PetscCall(DMPlexGetSubpointMap(dm, &spmap));
4217   PetscCall(PetscObjectStateGet((PetscObject)spmap, &state));
4218   if (state != mesh->subpointState || !mesh->subpointIS) PetscCall(DMPlexCreateSubpointIS_Internal(dm, &mesh->subpointIS));
4219   *subpointIS = mesh->subpointIS;
4220   PetscFunctionReturn(PETSC_SUCCESS);
4221 }
4222 
4223 /*@
4224   DMGetEnclosureRelation - Get the relationship between `dmA` and `dmB`
4225 
4226   Input Parameters:
4227 + dmA - The first `DM`
4228 - dmB - The second `DM`
4229 
4230   Output Parameter:
4231 . rel - The relation of `dmA` to `dmB`
4232 
4233   Level: intermediate
4234 
4235 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMGetEnclosurePoint()`
4236 @*/
4237 PetscErrorCode DMGetEnclosureRelation(DM dmA, DM dmB, DMEnclosureType *rel)
4238 {
4239   DM       plexA, plexB, sdm;
4240   DMLabel  spmap;
4241   PetscInt pStartA, pEndA, pStartB, pEndB, NpA, NpB;
4242 
4243   PetscFunctionBegin;
4244   PetscAssertPointer(rel, 3);
4245   *rel = DM_ENC_NONE;
4246   if (!dmA || !dmB) PetscFunctionReturn(PETSC_SUCCESS);
4247   PetscValidHeaderSpecific(dmA, DM_CLASSID, 1);
4248   PetscValidHeaderSpecific(dmB, DM_CLASSID, 2);
4249   if (dmA == dmB) {
4250     *rel = DM_ENC_EQUALITY;
4251     PetscFunctionReturn(PETSC_SUCCESS);
4252   }
4253   PetscCall(DMConvert(dmA, DMPLEX, &plexA));
4254   PetscCall(DMConvert(dmB, DMPLEX, &plexB));
4255   PetscCall(DMPlexGetChart(plexA, &pStartA, &pEndA));
4256   PetscCall(DMPlexGetChart(plexB, &pStartB, &pEndB));
4257   /* Assumption 1: subDMs have smaller charts than the DMs that they originate from
4258     - The degenerate case of a subdomain which includes all of the domain on some process can be treated as equality */
4259   if ((pStartA == pStartB) && (pEndA == pEndB)) {
4260     *rel = DM_ENC_EQUALITY;
4261     goto end;
4262   }
4263   NpA = pEndA - pStartA;
4264   NpB = pEndB - pStartB;
4265   if (NpA == NpB) goto end;
4266   sdm = NpA > NpB ? plexB : plexA; /* The other is the original, enclosing dm */
4267   PetscCall(DMPlexGetSubpointMap(sdm, &spmap));
4268   if (!spmap) goto end;
4269   /* TODO Check the space mapped to by subpointMap is same size as dm */
4270   if (NpA > NpB) {
4271     *rel = DM_ENC_SUPERMESH;
4272   } else {
4273     *rel = DM_ENC_SUBMESH;
4274   }
4275 end:
4276   PetscCall(DMDestroy(&plexA));
4277   PetscCall(DMDestroy(&plexB));
4278   PetscFunctionReturn(PETSC_SUCCESS);
4279 }
4280 
4281 /*@
4282   DMGetEnclosurePoint - Get the point `pA` in `dmA` which corresponds to the point `pB` in `dmB`
4283 
4284   Input Parameters:
4285 + dmA   - The first `DM`
4286 . dmB   - The second `DM`
4287 . etype - The type of enclosure relation that `dmA` has to `dmB`
4288 - pB    - A point of `dmB`
4289 
4290   Output Parameter:
4291 . pA - The corresponding point of `dmA`
4292 
4293   Level: intermediate
4294 
4295 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMGetEnclosureRelation()`
4296 @*/
4297 PetscErrorCode DMGetEnclosurePoint(DM dmA, DM dmB, DMEnclosureType etype, PetscInt pB, PetscInt *pA)
4298 {
4299   DM              sdm;
4300   IS              subpointIS;
4301   const PetscInt *subpoints;
4302   PetscInt        numSubpoints;
4303 
4304   PetscFunctionBegin;
4305   /* TODO Cache the IS, making it look like an index */
4306   switch (etype) {
4307   case DM_ENC_SUPERMESH:
4308     sdm = dmB;
4309     PetscCall(DMPlexGetSubpointIS(sdm, &subpointIS));
4310     PetscCall(ISGetIndices(subpointIS, &subpoints));
4311     *pA = subpoints[pB];
4312     PetscCall(ISRestoreIndices(subpointIS, &subpoints));
4313     break;
4314   case DM_ENC_SUBMESH:
4315     sdm = dmA;
4316     PetscCall(DMPlexGetSubpointIS(sdm, &subpointIS));
4317     PetscCall(ISGetLocalSize(subpointIS, &numSubpoints));
4318     PetscCall(ISGetIndices(subpointIS, &subpoints));
4319     PetscCall(PetscFindInt(pB, numSubpoints, subpoints, pA));
4320     if (*pA < 0) {
4321       PetscCall(DMViewFromOptions(dmA, NULL, "-dm_enc_A_view"));
4322       PetscCall(DMViewFromOptions(dmB, NULL, "-dm_enc_B_view"));
4323       SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Point %" PetscInt_FMT " not found in submesh", pB);
4324     }
4325     PetscCall(ISRestoreIndices(subpointIS, &subpoints));
4326     break;
4327   case DM_ENC_EQUALITY:
4328   case DM_ENC_NONE:
4329     *pA = pB;
4330     break;
4331   case DM_ENC_UNKNOWN: {
4332     DMEnclosureType enc;
4333 
4334     PetscCall(DMGetEnclosureRelation(dmA, dmB, &enc));
4335     PetscCall(DMGetEnclosurePoint(dmA, dmB, enc, pB, pA));
4336   } break;
4337   default:
4338     SETERRQ(PetscObjectComm((PetscObject)dmA), PETSC_ERR_ARG_OUTOFRANGE, "Invalid enclosure type %d", (int)etype);
4339   }
4340   PetscFunctionReturn(PETSC_SUCCESS);
4341 }
4342