xref: /petsc/src/dm/impls/plex/plexsubmesh.c (revision 07a75e276c69ab171da0f98d0260cb13c2616414)
1 #include <petsc/private/dmpleximpl.h>  /*I      "petscdmplex.h"    I*/
2 #include <petsc/private/dmlabelimpl.h> /*I      "petscdmlabel.h"   I*/
3 #include <petscsf.h>
4 
5 static PetscErrorCode DMPlexCellIsHybrid_Internal(DM dm, PetscInt p, PetscBool *isHybrid)
6 {
7   DMPolytopeType ct;
8 
9   PetscFunctionBegin;
10   PetscCall(DMPlexGetCellType(dm, p, &ct));
11   switch (ct) {
12   case DM_POLYTOPE_POINT_PRISM_TENSOR:
13   case DM_POLYTOPE_SEG_PRISM_TENSOR:
14   case DM_POLYTOPE_TRI_PRISM_TENSOR:
15   case DM_POLYTOPE_QUAD_PRISM_TENSOR:
16     *isHybrid = PETSC_TRUE;
17     break;
18   default:
19     *isHybrid = PETSC_FALSE;
20     break;
21   }
22   PetscFunctionReturn(PETSC_SUCCESS);
23 }
24 
25 static PetscErrorCode DMPlexGetTensorPrismBounds_Internal(DM dm, PetscInt dim, PetscInt *cStart, PetscInt *cEnd)
26 {
27   DMLabel ctLabel;
28 
29   PetscFunctionBegin;
30   if (cStart) *cStart = -1;
31   if (cEnd) *cEnd = -1;
32   PetscCall(DMPlexGetCellTypeLabel(dm, &ctLabel));
33   switch (dim) {
34   case 1:
35     PetscCall(DMLabelGetStratumBounds(ctLabel, DM_POLYTOPE_POINT_PRISM_TENSOR, cStart, cEnd));
36     break;
37   case 2:
38     PetscCall(DMLabelGetStratumBounds(ctLabel, DM_POLYTOPE_SEG_PRISM_TENSOR, cStart, cEnd));
39     break;
40   case 3:
41     PetscCall(DMLabelGetStratumBounds(ctLabel, DM_POLYTOPE_TRI_PRISM_TENSOR, cStart, cEnd));
42     if (*cStart < 0) PetscCall(DMLabelGetStratumBounds(ctLabel, DM_POLYTOPE_QUAD_PRISM_TENSOR, cStart, cEnd));
43     break;
44   default:
45     PetscFunctionReturn(PETSC_SUCCESS);
46   }
47   PetscFunctionReturn(PETSC_SUCCESS);
48 }
49 
50 static PetscErrorCode DMPlexMarkBoundaryFaces_Internal(DM dm, PetscInt val, PetscInt cellHeight, DMLabel label)
51 {
52   PetscInt           depth, pStart, pEnd, fStart, fEnd, f, supportSize, nroots = -1, nleaves = -1;
53   PetscSF            sf;
54   const PetscSFNode *iremote = NULL;
55   const PetscInt    *ilocal  = NULL;
56   PetscInt          *leafData;
57 
58   PetscFunctionBegin;
59   PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
60   PetscCall(DMPlexGetDepth(dm, &depth));
61   if (depth >= cellHeight + 1) {
62     PetscCall(DMPlexGetHeightStratum(dm, cellHeight + 1, &fStart, &fEnd));
63   } else {
64     /* Note DMPlexGetHeightStratum() returns fStart, fEnd = pStart, pEnd */
65     /* if height > depth, which is not what we want here.                */
66     fStart = 0;
67     fEnd   = 0;
68   }
69   PetscCall(PetscCalloc1(pEnd - pStart, &leafData));
70   leafData -= pStart;
71   PetscCall(DMGetPointSF(dm, &sf));
72   if (sf) PetscCall(PetscSFGetGraph(sf, &nroots, &nleaves, &ilocal, &iremote));
73   if (sf && nroots >= 0) {
74     PetscInt        cStart, cEnd, c, i;
75     PetscInt       *rootData, *rootData1, *cellOwners, hasTwoSupportCells = -2;
76     const PetscInt *support;
77     PetscMPIInt     rank;
78 
79     PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
80     PetscCall(DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd));
81     PetscCall(PetscCalloc3(pEnd - pStart, &rootData, pEnd - pStart, &rootData1, cEnd - cStart, &cellOwners));
82     rootData -= pStart;
83     rootData1 -= pStart;
84     for (c = cStart; c < cEnd; ++c) cellOwners[c - cStart] = (PetscInt)rank;
85     for (i = 0; i < nleaves; ++i) {
86       c = ilocal ? ilocal[i] : i;
87       if (c >= cStart && c < cEnd) cellOwners[c - cStart] = iremote[i].rank;
88     }
89     for (f = fStart; f < fEnd; ++f) {
90       PetscCall(DMPlexGetSupportSize(dm, f, &supportSize));
91       if (supportSize == 1) {
92         PetscCall(DMPlexGetSupport(dm, f, &support));
93         leafData[f] = cellOwners[support[0] - cStart];
94       } else {
95         /* TODO: When using DMForest, we could have a parent facet (a coarse facet)     */
96         /*       supportSize of which alone does not tell us if it is an interior       */
97         /*       facet or an exterior facet. Those facets can be identified by checking */
98         /*       if they are in the parent tree. We should probably skip those parent   */
99         /*       facets here, which will allow for including the following check, and   */
100         /*       see if they are exterior facets or not afterwards by checking if the   */
101         /*       children are exterior or not.                                          */
102         /* PetscCheck(supportSize == 2, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid facet support size (%" PetscInt_FMT ") on facet (%" PetscInt_FMT ")", supportSize, f); */
103         leafData[f] = hasTwoSupportCells; /* some negative PetscInt */
104       }
105       rootData[f]  = leafData[f];
106       rootData1[f] = leafData[f];
107     }
108     PetscCall(PetscSFReduceBegin(sf, MPIU_INT, leafData, rootData, MPI_MIN));
109     PetscCall(PetscSFReduceBegin(sf, MPIU_INT, leafData, rootData1, MPI_MAX));
110     PetscCall(PetscSFReduceEnd(sf, MPIU_INT, leafData, rootData, MPI_MIN));
111     PetscCall(PetscSFReduceEnd(sf, MPIU_INT, leafData, rootData1, MPI_MAX));
112     for (f = fStart; f < fEnd; ++f) {
113       /* Store global support size of f.                                                    */
114       /* Facet f is an interior facet if and only if one of the following two is satisfied: */
115       /* 1. supportSize is 2 on some rank.                                                  */
116       /* 2. supportSize is 1 on any rank that can see f, but f is on a partition boundary;  */
117       /*    i.e., rootData[f] < rootData1[f].                                               */
118       rootData[f] = (rootData[f] == hasTwoSupportCells || (rootData[f] < rootData1[f])) ? 2 : 1;
119       leafData[f] = rootData[f];
120     }
121     PetscCall(PetscSFBcastBegin(sf, MPIU_INT, rootData, leafData, MPI_REPLACE));
122     PetscCall(PetscSFBcastEnd(sf, MPIU_INT, rootData, leafData, MPI_REPLACE));
123     rootData += pStart;
124     rootData1 += pStart;
125     PetscCall(PetscFree3(rootData, rootData1, cellOwners));
126   } else {
127     for (f = fStart; f < fEnd; ++f) {
128       PetscCall(DMPlexGetSupportSize(dm, f, &supportSize));
129       leafData[f] = supportSize;
130     }
131   }
132   for (f = fStart; f < fEnd; ++f) {
133     if (leafData[f] == 1) {
134       if (val < 0) {
135         PetscInt *closure = NULL;
136         PetscInt  clSize, cl, cval;
137 
138         PetscCall(DMPlexGetTransitiveClosure(dm, f, PETSC_TRUE, &clSize, &closure));
139         for (cl = 0; cl < clSize * 2; cl += 2) {
140           PetscCall(DMLabelGetValue(label, closure[cl], &cval));
141           if (cval < 0) continue;
142           PetscCall(DMLabelSetValue(label, f, cval));
143           break;
144         }
145         if (cl == clSize * 2) PetscCall(DMLabelSetValue(label, f, 1));
146         PetscCall(DMPlexRestoreTransitiveClosure(dm, f, PETSC_TRUE, &clSize, &closure));
147       } else {
148         PetscCall(DMLabelSetValue(label, f, val));
149       }
150     } else {
151       /* TODO: See the above comment on DMForest */
152       /* PetscCheck(leafData[f] == 2, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid facet support size (%" PetscInt_FMT ") on facet (%" PetscInt_FMT ")", leafData[f], f); */
153     }
154   }
155   leafData += pStart;
156   PetscCall(PetscFree(leafData));
157   PetscFunctionReturn(PETSC_SUCCESS);
158 }
159 
160 /*@
161   DMPlexMarkBoundaryFaces - Mark all faces on the boundary
162 
163   Collective
164 
165   Input Parameters:
166 + dm  - The original `DM`
167 - val - The marker value, or `PETSC_DETERMINE` to use some value in the closure (or 1 if none are found)
168 
169   Output Parameter:
170 . label - The `DMLabel` marking boundary faces with the given value
171 
172   Level: developer
173 
174   Note:
175   This function will use the point `PetscSF` from the input `DM` and the ownership of the support cells to exclude points on the partition boundary from being marked. If you also wish to mark the partition boundary, you can use `DMSetPointSF()` to temporarily set it to `NULL`, and then reset it to the original object after the call.
176 
177   In DMForest there can be facets support sizes of which alone can not determine whether they are on the boundary. Currently, this function is not guaranteed to produce the correct result in such case.
178 
179 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMLabelCreate()`, `DMCreateLabel()`
180 @*/
181 PetscErrorCode DMPlexMarkBoundaryFaces(DM dm, PetscInt val, DMLabel label)
182 {
183   DMPlexInterpolatedFlag flg;
184 
185   PetscFunctionBegin;
186   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
187   PetscCall(DMPlexIsInterpolated(dm, &flg));
188   PetscCheck(flg == DMPLEX_INTERPOLATED_FULL, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM is not fully interpolated on this rank");
189   PetscCall(DMPlexMarkBoundaryFaces_Internal(dm, val, 0, label));
190   PetscFunctionReturn(PETSC_SUCCESS);
191 }
192 
193 static PetscErrorCode DMPlexLabelComplete_Internal(DM dm, DMLabel label, PetscBool completeCells)
194 {
195   IS              valueIS;
196   PetscSF         sfPoint;
197   const PetscInt *values;
198   PetscInt        numValues, v, cStart, cEnd, nroots;
199 
200   PetscFunctionBegin;
201   PetscCall(DMLabelGetNumValues(label, &numValues));
202   PetscCall(DMLabelGetValueIS(label, &valueIS));
203   PetscCall(DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd));
204   PetscCall(ISGetIndices(valueIS, &values));
205   for (v = 0; v < numValues; ++v) {
206     IS              pointIS;
207     const PetscInt *points;
208     PetscInt        numPoints, p;
209 
210     PetscCall(DMLabelGetStratumSize(label, values[v], &numPoints));
211     PetscCall(DMLabelGetStratumIS(label, values[v], &pointIS));
212     PetscCall(ISGetIndices(pointIS, &points));
213     for (p = 0; p < numPoints; ++p) {
214       PetscInt  q       = points[p];
215       PetscInt *closure = NULL;
216       PetscInt  closureSize, c;
217 
218       if (cStart <= q && q < cEnd && !completeCells) { /* skip cells */
219         continue;
220       }
221       PetscCall(DMPlexGetTransitiveClosure(dm, q, PETSC_TRUE, &closureSize, &closure));
222       for (c = 0; c < closureSize * 2; c += 2) PetscCall(DMLabelSetValue(label, closure[c], values[v]));
223       PetscCall(DMPlexRestoreTransitiveClosure(dm, q, PETSC_TRUE, &closureSize, &closure));
224     }
225     PetscCall(ISRestoreIndices(pointIS, &points));
226     PetscCall(ISDestroy(&pointIS));
227   }
228   PetscCall(ISRestoreIndices(valueIS, &values));
229   PetscCall(ISDestroy(&valueIS));
230   PetscCall(DMGetPointSF(dm, &sfPoint));
231   PetscCall(PetscSFGetGraph(sfPoint, &nroots, NULL, NULL, NULL));
232   if (nroots >= 0) {
233     DMLabel         lblRoots, lblLeaves;
234     IS              valueIS, pointIS;
235     const PetscInt *values;
236     PetscInt        numValues, v;
237 
238     /* Pull point contributions from remote leaves into local roots */
239     PetscCall(DMLabelGather(label, sfPoint, &lblLeaves));
240     PetscCall(DMLabelGetValueIS(lblLeaves, &valueIS));
241     PetscCall(ISGetLocalSize(valueIS, &numValues));
242     PetscCall(ISGetIndices(valueIS, &values));
243     for (v = 0; v < numValues; ++v) {
244       const PetscInt value = values[v];
245 
246       PetscCall(DMLabelGetStratumIS(lblLeaves, value, &pointIS));
247       PetscCall(DMLabelInsertIS(label, pointIS, value));
248       PetscCall(ISDestroy(&pointIS));
249     }
250     PetscCall(ISRestoreIndices(valueIS, &values));
251     PetscCall(ISDestroy(&valueIS));
252     PetscCall(DMLabelDestroy(&lblLeaves));
253     /* Push point contributions from roots into remote leaves */
254     PetscCall(DMLabelDistribute(label, sfPoint, &lblRoots));
255     PetscCall(DMLabelGetValueIS(lblRoots, &valueIS));
256     PetscCall(ISGetLocalSize(valueIS, &numValues));
257     PetscCall(ISGetIndices(valueIS, &values));
258     for (v = 0; v < numValues; ++v) {
259       const PetscInt value = values[v];
260 
261       PetscCall(DMLabelGetStratumIS(lblRoots, value, &pointIS));
262       PetscCall(DMLabelInsertIS(label, pointIS, value));
263       PetscCall(ISDestroy(&pointIS));
264     }
265     PetscCall(ISRestoreIndices(valueIS, &values));
266     PetscCall(ISDestroy(&valueIS));
267     PetscCall(DMLabelDestroy(&lblRoots));
268   }
269   PetscFunctionReturn(PETSC_SUCCESS);
270 }
271 
272 /*@
273   DMPlexLabelComplete - Starting with a label marking points on a surface, we add the transitive closure to the surface
274 
275   Input Parameters:
276 + dm    - The `DM`
277 - label - A `DMLabel` marking the surface points
278 
279   Output Parameter:
280 . label - A `DMLabel` marking all surface points in the transitive closure
281 
282   Level: developer
283 
284 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexLabelCohesiveComplete()`
285 @*/
286 PetscErrorCode DMPlexLabelComplete(DM dm, DMLabel label)
287 {
288   PetscFunctionBegin;
289   PetscCall(DMPlexLabelComplete_Internal(dm, label, PETSC_TRUE));
290   PetscFunctionReturn(PETSC_SUCCESS);
291 }
292 
293 /*@
294   DMPlexLabelAddCells - Starting with a label marking points on a surface, we add a cell for each point
295 
296   Input Parameters:
297 + dm    - The `DM`
298 - label - A `DMLabel` marking the surface points
299 
300   Output Parameter:
301 . label - A `DMLabel` incorporating cells
302 
303   Level: developer
304 
305   Note:
306   The cells allow FEM boundary conditions to be applied using the cell geometry
307 
308 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexLabelAddFaceCells()`, `DMPlexLabelComplete()`, `DMPlexLabelCohesiveComplete()`
309 @*/
310 PetscErrorCode DMPlexLabelAddCells(DM dm, DMLabel label)
311 {
312   IS              valueIS;
313   const PetscInt *values;
314   PetscInt        numValues, v, csStart, csEnd, chStart, chEnd;
315 
316   PetscFunctionBegin;
317   PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &csStart, &csEnd));
318   PetscCall(DMPlexGetHeightStratum(dm, 0, &chStart, &chEnd));
319   PetscCall(DMLabelGetNumValues(label, &numValues));
320   PetscCall(DMLabelGetValueIS(label, &valueIS));
321   PetscCall(ISGetIndices(valueIS, &values));
322   for (v = 0; v < numValues; ++v) {
323     IS              pointIS;
324     const PetscInt *points;
325     PetscInt        numPoints, p;
326 
327     PetscCall(DMLabelGetStratumSize(label, values[v], &numPoints));
328     PetscCall(DMLabelGetStratumIS(label, values[v], &pointIS));
329     PetscCall(ISGetIndices(pointIS, &points));
330     for (p = 0; p < numPoints; ++p) {
331       const PetscInt point   = points[p];
332       PetscInt      *closure = NULL;
333       PetscInt       closureSize, cl, h, pStart, pEnd, cStart, cEnd;
334 
335       // If the point is a hybrid, allow hybrid cells
336       PetscCall(DMPlexGetPointHeight(dm, point, &h));
337       PetscCall(DMPlexGetSimplexOrBoxCells(dm, h, &pStart, &pEnd));
338       if (point >= pStart && point < pEnd) {
339         cStart = csStart;
340         cEnd   = csEnd;
341       } else {
342         cStart = chStart;
343         cEnd   = chEnd;
344       }
345 
346       PetscCall(DMPlexGetTransitiveClosure(dm, points[p], PETSC_FALSE, &closureSize, &closure));
347       for (cl = closureSize - 1; cl > 0; --cl) {
348         const PetscInt cell = closure[cl * 2];
349         if ((cell >= cStart) && (cell < cEnd)) {
350           PetscCall(DMLabelSetValue(label, cell, values[v]));
351           break;
352         }
353       }
354       PetscCall(DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_FALSE, &closureSize, &closure));
355     }
356     PetscCall(ISRestoreIndices(pointIS, &points));
357     PetscCall(ISDestroy(&pointIS));
358   }
359   PetscCall(ISRestoreIndices(valueIS, &values));
360   PetscCall(ISDestroy(&valueIS));
361   PetscFunctionReturn(PETSC_SUCCESS);
362 }
363 
364 /*@
365   DMPlexLabelAddFaceCells - Starting with a label marking faces on a surface, we add a cell for each face
366 
367   Input Parameters:
368 + dm    - The `DM`
369 - label - A `DMLabel` marking the surface points
370 
371   Output Parameter:
372 . label - A `DMLabel` incorporating cells
373 
374   Level: developer
375 
376   Note:
377   The cells allow FEM boundary conditions to be applied using the cell geometry
378 
379 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexLabelAddCells()`, `DMPlexLabelComplete()`, `DMPlexLabelCohesiveComplete()`
380 @*/
381 PetscErrorCode DMPlexLabelAddFaceCells(DM dm, DMLabel label)
382 {
383   IS              valueIS;
384   const PetscInt *values;
385   PetscInt        numValues, v, cStart, cEnd, fStart, fEnd;
386 
387   PetscFunctionBegin;
388   PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd));
389   PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
390   PetscCall(DMLabelGetNumValues(label, &numValues));
391   PetscCall(DMLabelGetValueIS(label, &valueIS));
392   PetscCall(ISGetIndices(valueIS, &values));
393   for (v = 0; v < numValues; ++v) {
394     IS              pointIS;
395     const PetscInt *points;
396     PetscInt        numPoints, p;
397 
398     PetscCall(DMLabelGetStratumSize(label, values[v], &numPoints));
399     PetscCall(DMLabelGetStratumIS(label, values[v], &pointIS));
400     PetscCall(ISGetIndices(pointIS, &points));
401     for (p = 0; p < numPoints; ++p) {
402       const PetscInt face    = points[p];
403       PetscInt      *closure = NULL;
404       PetscInt       closureSize, cl;
405 
406       if ((face < fStart) || (face >= fEnd)) continue;
407       PetscCall(DMPlexGetTransitiveClosure(dm, face, PETSC_FALSE, &closureSize, &closure));
408       for (cl = closureSize - 1; cl > 0; --cl) {
409         const PetscInt cell = closure[cl * 2];
410         if ((cell >= cStart) && (cell < cEnd)) {
411           PetscCall(DMLabelSetValue(label, cell, values[v]));
412           break;
413         }
414       }
415       PetscCall(DMPlexRestoreTransitiveClosure(dm, face, PETSC_FALSE, &closureSize, &closure));
416     }
417     PetscCall(ISRestoreIndices(pointIS, &points));
418     PetscCall(ISDestroy(&pointIS));
419   }
420   PetscCall(ISRestoreIndices(valueIS, &values));
421   PetscCall(ISDestroy(&valueIS));
422   PetscFunctionReturn(PETSC_SUCCESS);
423 }
424 
425 /*@
426   DMPlexLabelClearCells - Remove cells from a label
427 
428   Input Parameters:
429 + dm    - The `DM`
430 - label - A `DMLabel` marking surface points and their adjacent cells
431 
432   Output Parameter:
433 . label - A `DMLabel` without cells
434 
435   Level: developer
436 
437   Note:
438   This undoes `DMPlexLabelAddCells()` or `DMPlexLabelAddFaceCells()`
439 
440 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexLabelComplete()`, `DMPlexLabelCohesiveComplete()`, `DMPlexLabelAddCells()`
441 @*/
442 PetscErrorCode DMPlexLabelClearCells(DM dm, DMLabel label)
443 {
444   IS              valueIS;
445   const PetscInt *values;
446   PetscInt        numValues, v, cStart, cEnd;
447 
448   PetscFunctionBegin;
449   PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd));
450   PetscCall(DMLabelGetNumValues(label, &numValues));
451   PetscCall(DMLabelGetValueIS(label, &valueIS));
452   PetscCall(ISGetIndices(valueIS, &values));
453   for (v = 0; v < numValues; ++v) {
454     IS              pointIS;
455     const PetscInt *points;
456     PetscInt        numPoints, p;
457 
458     PetscCall(DMLabelGetStratumSize(label, values[v], &numPoints));
459     PetscCall(DMLabelGetStratumIS(label, values[v], &pointIS));
460     PetscCall(ISGetIndices(pointIS, &points));
461     for (p = 0; p < numPoints; ++p) {
462       PetscInt point = points[p];
463 
464       if (point >= cStart && point < cEnd) PetscCall(DMLabelClearValue(label, point, values[v]));
465     }
466     PetscCall(ISRestoreIndices(pointIS, &points));
467     PetscCall(ISDestroy(&pointIS));
468   }
469   PetscCall(ISRestoreIndices(valueIS, &values));
470   PetscCall(ISDestroy(&valueIS));
471   PetscFunctionReturn(PETSC_SUCCESS);
472 }
473 
474 /* take (oldEnd, added) pairs, ordered by height and convert them to (oldstart, newstart) pairs, ordered by ascending
475  * index (skipping first, which is (0,0)) */
476 static inline PetscErrorCode DMPlexShiftPointSetUp_Internal(PetscInt depth, PetscInt depthShift[])
477 {
478   PetscInt d, off = 0;
479 
480   PetscFunctionBegin;
481   /* sort by (oldend): yes this is an O(n^2) sort, we expect depth <= 3 */
482   for (d = 0; d < depth; d++) {
483     PetscInt firstd     = d;
484     PetscInt firstStart = depthShift[2 * d];
485     PetscInt e;
486 
487     for (e = d + 1; e <= depth; e++) {
488       if (depthShift[2 * e] < firstStart) {
489         firstd     = e;
490         firstStart = depthShift[2 * d];
491       }
492     }
493     if (firstd != d) {
494       PetscInt swap[2];
495 
496       e                     = firstd;
497       swap[0]               = depthShift[2 * d];
498       swap[1]               = depthShift[2 * d + 1];
499       depthShift[2 * d]     = depthShift[2 * e];
500       depthShift[2 * d + 1] = depthShift[2 * e + 1];
501       depthShift[2 * e]     = swap[0];
502       depthShift[2 * e + 1] = swap[1];
503     }
504   }
505   /* convert (oldstart, added) to (oldstart, newstart) */
506   for (d = 0; d <= depth; d++) {
507     off += depthShift[2 * d + 1];
508     depthShift[2 * d + 1] = depthShift[2 * d] + off;
509   }
510   PetscFunctionReturn(PETSC_SUCCESS);
511 }
512 
513 /* depthShift is a list of (old, new) pairs */
514 static inline PetscInt DMPlexShiftPoint_Internal(PetscInt p, PetscInt depth, PetscInt depthShift[])
515 {
516   PetscInt d;
517   PetscInt newOff = 0;
518 
519   for (d = 0; d <= depth; d++) {
520     if (p < depthShift[2 * d]) return p + newOff;
521     else newOff = depthShift[2 * d + 1] - depthShift[2 * d];
522   }
523   return p + newOff;
524 }
525 
526 /* depthShift is a list of (old, new) pairs */
527 static inline PetscInt DMPlexShiftPointInverse_Internal(PetscInt p, PetscInt depth, PetscInt depthShift[])
528 {
529   PetscInt d;
530   PetscInt newOff = 0;
531 
532   for (d = 0; d <= depth; d++) {
533     if (p < depthShift[2 * d + 1]) return p + newOff;
534     else newOff = depthShift[2 * d] - depthShift[2 * d + 1];
535   }
536   return p + newOff;
537 }
538 
539 static PetscErrorCode DMPlexShiftSizes_Internal(DM dm, PetscInt depthShift[], DM dmNew)
540 {
541   PetscInt depth = 0, d, pStart, pEnd, p;
542   DMLabel  depthLabel;
543 
544   PetscFunctionBegin;
545   PetscCall(DMPlexGetDepth(dm, &depth));
546   if (depth < 0) PetscFunctionReturn(PETSC_SUCCESS);
547   /* Step 1: Expand chart */
548   PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
549   pEnd = DMPlexShiftPoint_Internal(pEnd, depth, depthShift);
550   PetscCall(DMPlexSetChart(dmNew, pStart, pEnd));
551   PetscCall(DMCreateLabel(dmNew, "depth"));
552   PetscCall(DMPlexGetDepthLabel(dmNew, &depthLabel));
553   PetscCall(DMCreateLabel(dmNew, "celltype"));
554   /* Step 2: Set cone and support sizes */
555   for (d = 0; d <= depth; ++d) {
556     PetscInt pStartNew, pEndNew;
557     IS       pIS;
558 
559     PetscCall(DMPlexGetDepthStratum(dm, d, &pStart, &pEnd));
560     pStartNew = DMPlexShiftPoint_Internal(pStart, depth, depthShift);
561     pEndNew   = DMPlexShiftPoint_Internal(pEnd, depth, depthShift);
562     PetscCall(ISCreateStride(PETSC_COMM_SELF, pEndNew - pStartNew, pStartNew, 1, &pIS));
563     PetscCall(DMLabelSetStratumIS(depthLabel, d, pIS));
564     PetscCall(ISDestroy(&pIS));
565     for (p = pStart; p < pEnd; ++p) {
566       PetscInt       newp = DMPlexShiftPoint_Internal(p, depth, depthShift);
567       PetscInt       size;
568       DMPolytopeType ct;
569 
570       PetscCall(DMPlexGetConeSize(dm, p, &size));
571       PetscCall(DMPlexSetConeSize(dmNew, newp, size));
572       PetscCall(DMPlexGetSupportSize(dm, p, &size));
573       PetscCall(DMPlexSetSupportSize(dmNew, newp, size));
574       PetscCall(DMPlexGetCellType(dm, p, &ct));
575       PetscCall(DMPlexSetCellType(dmNew, newp, ct));
576     }
577   }
578   PetscFunctionReturn(PETSC_SUCCESS);
579 }
580 
581 static PetscErrorCode DMPlexShiftPoints_Internal(DM dm, PetscInt depthShift[], DM dmNew)
582 {
583   PetscInt *newpoints;
584   PetscInt  depth = 0, maxConeSize, maxSupportSize, maxConeSizeNew, maxSupportSizeNew, pStart, pEnd, p;
585 
586   PetscFunctionBegin;
587   PetscCall(DMPlexGetDepth(dm, &depth));
588   if (depth < 0) PetscFunctionReturn(PETSC_SUCCESS);
589   PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize));
590   PetscCall(DMPlexGetMaxSizes(dmNew, &maxConeSizeNew, &maxSupportSizeNew));
591   PetscCall(PetscMalloc1(PetscMax(PetscMax(maxConeSize, maxSupportSize), PetscMax(maxConeSizeNew, maxSupportSizeNew)), &newpoints));
592   /* Step 5: Set cones and supports */
593   PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
594   for (p = pStart; p < pEnd; ++p) {
595     const PetscInt *points = NULL, *orientations = NULL;
596     PetscInt        size, sizeNew, i, newp = DMPlexShiftPoint_Internal(p, depth, depthShift);
597 
598     PetscCall(DMPlexGetConeSize(dm, p, &size));
599     PetscCall(DMPlexGetCone(dm, p, &points));
600     PetscCall(DMPlexGetConeOrientation(dm, p, &orientations));
601     for (i = 0; i < size; ++i) newpoints[i] = DMPlexShiftPoint_Internal(points[i], depth, depthShift);
602     PetscCall(DMPlexSetCone(dmNew, newp, newpoints));
603     PetscCall(DMPlexSetConeOrientation(dmNew, newp, orientations));
604     PetscCall(DMPlexGetSupportSize(dm, p, &size));
605     PetscCall(DMPlexGetSupportSize(dmNew, newp, &sizeNew));
606     PetscCall(DMPlexGetSupport(dm, p, &points));
607     for (i = 0; i < size; ++i) newpoints[i] = DMPlexShiftPoint_Internal(points[i], depth, depthShift);
608     for (i = size; i < sizeNew; ++i) newpoints[i] = 0;
609     PetscCall(DMPlexSetSupport(dmNew, newp, newpoints));
610   }
611   PetscCall(PetscFree(newpoints));
612   PetscFunctionReturn(PETSC_SUCCESS);
613 }
614 
615 static PetscErrorCode DMPlexShiftCoordinates_Internal(DM dm, PetscInt depthShift[], DM dmNew)
616 {
617   PetscSection coordSection, newCoordSection;
618   Vec          coordinates, newCoordinates;
619   PetscScalar *coords, *newCoords;
620   PetscInt     coordSize, sStart, sEnd;
621   PetscInt     dim, depth = 0, cStart, cEnd, cStartNew, cEndNew, c, vStart, vEnd, vStartNew, vEndNew, v;
622   PetscBool    hasCells;
623 
624   PetscFunctionBegin;
625   PetscCall(DMGetCoordinateDim(dm, &dim));
626   PetscCall(DMSetCoordinateDim(dmNew, dim));
627   PetscCall(DMPlexGetDepth(dm, &depth));
628   /* Step 8: Convert coordinates */
629   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
630   PetscCall(DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd));
631   PetscCall(DMPlexGetDepthStratum(dmNew, 0, &vStartNew, &vEndNew));
632   PetscCall(DMPlexGetHeightStratum(dmNew, 0, &cStartNew, &cEndNew));
633   PetscCall(DMGetCoordinateSection(dm, &coordSection));
634   PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dm), &newCoordSection));
635   PetscCall(PetscSectionSetNumFields(newCoordSection, 1));
636   PetscCall(PetscSectionSetFieldComponents(newCoordSection, 0, dim));
637   PetscCall(PetscSectionGetChart(coordSection, &sStart, &sEnd));
638   hasCells = sStart == cStart ? PETSC_TRUE : PETSC_FALSE;
639   PetscCall(PetscSectionSetChart(newCoordSection, hasCells ? cStartNew : vStartNew, vEndNew));
640   if (hasCells) {
641     for (c = cStart; c < cEnd; ++c) {
642       PetscInt cNew = DMPlexShiftPoint_Internal(c, depth, depthShift), dof;
643 
644       PetscCall(PetscSectionGetDof(coordSection, c, &dof));
645       PetscCall(PetscSectionSetDof(newCoordSection, cNew, dof));
646       PetscCall(PetscSectionSetFieldDof(newCoordSection, cNew, 0, dof));
647     }
648   }
649   for (v = vStartNew; v < vEndNew; ++v) {
650     PetscCall(PetscSectionSetDof(newCoordSection, v, dim));
651     PetscCall(PetscSectionSetFieldDof(newCoordSection, v, 0, dim));
652   }
653   PetscCall(PetscSectionSetUp(newCoordSection));
654   PetscCall(DMSetCoordinateSection(dmNew, PETSC_DETERMINE, newCoordSection));
655   PetscCall(PetscSectionGetStorageSize(newCoordSection, &coordSize));
656   PetscCall(VecCreate(PETSC_COMM_SELF, &newCoordinates));
657   PetscCall(PetscObjectSetName((PetscObject)newCoordinates, "coordinates"));
658   PetscCall(VecSetSizes(newCoordinates, coordSize, PETSC_DETERMINE));
659   PetscCall(VecSetBlockSize(newCoordinates, dim));
660   PetscCall(VecSetType(newCoordinates, VECSTANDARD));
661   PetscCall(DMSetCoordinatesLocal(dmNew, newCoordinates));
662   PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
663   PetscCall(VecGetArray(coordinates, &coords));
664   PetscCall(VecGetArray(newCoordinates, &newCoords));
665   if (hasCells) {
666     for (c = cStart; c < cEnd; ++c) {
667       PetscInt cNew = DMPlexShiftPoint_Internal(c, depth, depthShift), dof, off, noff, d;
668 
669       PetscCall(PetscSectionGetDof(coordSection, c, &dof));
670       PetscCall(PetscSectionGetOffset(coordSection, c, &off));
671       PetscCall(PetscSectionGetOffset(newCoordSection, cNew, &noff));
672       for (d = 0; d < dof; ++d) newCoords[noff + d] = coords[off + d];
673     }
674   }
675   for (v = vStart; v < vEnd; ++v) {
676     PetscInt dof, off, noff, d;
677 
678     PetscCall(PetscSectionGetDof(coordSection, v, &dof));
679     PetscCall(PetscSectionGetOffset(coordSection, v, &off));
680     PetscCall(PetscSectionGetOffset(newCoordSection, DMPlexShiftPoint_Internal(v, depth, depthShift), &noff));
681     for (d = 0; d < dof; ++d) newCoords[noff + d] = coords[off + d];
682   }
683   PetscCall(VecRestoreArray(coordinates, &coords));
684   PetscCall(VecRestoreArray(newCoordinates, &newCoords));
685   PetscCall(VecDestroy(&newCoordinates));
686   PetscCall(PetscSectionDestroy(&newCoordSection));
687   PetscFunctionReturn(PETSC_SUCCESS);
688 }
689 
690 static PetscErrorCode DMPlexShiftSF_Single(DM dm, PetscInt depthShift[], PetscSF sf, PetscSF sfNew)
691 {
692   const PetscSFNode *remotePoints;
693   PetscSFNode       *gremotePoints;
694   const PetscInt    *localPoints;
695   PetscInt          *glocalPoints, *newLocation, *newRemoteLocation;
696   PetscInt           numRoots, numLeaves, l, pStart, pEnd, depth = 0, totShift = 0;
697 
698   PetscFunctionBegin;
699   PetscCall(DMPlexGetDepth(dm, &depth));
700   PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
701   PetscCall(PetscSFGetGraph(sf, &numRoots, &numLeaves, &localPoints, &remotePoints));
702   totShift = DMPlexShiftPoint_Internal(pEnd, depth, depthShift) - pEnd;
703   if (numRoots >= 0) {
704     PetscCall(PetscMalloc2(numRoots, &newLocation, pEnd - pStart, &newRemoteLocation));
705     for (l = 0; l < numRoots; ++l) newLocation[l] = DMPlexShiftPoint_Internal(l, depth, depthShift);
706     PetscCall(PetscSFBcastBegin(sf, MPIU_INT, newLocation, newRemoteLocation, MPI_REPLACE));
707     PetscCall(PetscSFBcastEnd(sf, MPIU_INT, newLocation, newRemoteLocation, MPI_REPLACE));
708     PetscCall(PetscMalloc1(numLeaves, &glocalPoints));
709     PetscCall(PetscMalloc1(numLeaves, &gremotePoints));
710     for (l = 0; l < numLeaves; ++l) {
711       glocalPoints[l]        = DMPlexShiftPoint_Internal(localPoints[l], depth, depthShift);
712       gremotePoints[l].rank  = remotePoints[l].rank;
713       gremotePoints[l].index = newRemoteLocation[localPoints[l]];
714     }
715     PetscCall(PetscFree2(newLocation, newRemoteLocation));
716     PetscCall(PetscSFSetGraph(sfNew, numRoots + totShift, numLeaves, glocalPoints, PETSC_OWN_POINTER, gremotePoints, PETSC_OWN_POINTER));
717   }
718   PetscFunctionReturn(PETSC_SUCCESS);
719 }
720 
721 static PetscErrorCode DMPlexShiftSF_Internal(DM dm, PetscInt depthShift[], DM dmNew)
722 {
723   PetscSF   sfPoint, sfPointNew;
724   PetscBool useNatural;
725 
726   PetscFunctionBegin;
727   /* Step 9: Convert pointSF */
728   PetscCall(DMGetPointSF(dm, &sfPoint));
729   PetscCall(DMGetPointSF(dmNew, &sfPointNew));
730   PetscCall(DMPlexShiftSF_Single(dm, depthShift, sfPoint, sfPointNew));
731   /* Step 9b: Convert naturalSF */
732   PetscCall(DMGetUseNatural(dm, &useNatural));
733   if (useNatural) {
734     PetscSF sfNat, sfNatNew;
735 
736     PetscCall(DMSetUseNatural(dmNew, useNatural));
737     PetscCall(DMGetNaturalSF(dm, &sfNat));
738     PetscCall(DMGetNaturalSF(dmNew, &sfNatNew));
739     PetscCall(DMPlexShiftSF_Single(dm, depthShift, sfNat, sfNatNew));
740   }
741   PetscFunctionReturn(PETSC_SUCCESS);
742 }
743 
744 static PetscErrorCode DMPlexShiftLabels_Internal(DM dm, PetscInt depthShift[], DM dmNew)
745 {
746   PetscInt depth = 0, numLabels, l;
747 
748   PetscFunctionBegin;
749   PetscCall(DMPlexGetDepth(dm, &depth));
750   /* Step 10: Convert labels */
751   PetscCall(DMGetNumLabels(dm, &numLabels));
752   for (l = 0; l < numLabels; ++l) {
753     DMLabel         label, newlabel;
754     const char     *lname;
755     PetscBool       isDepth, isDim;
756     IS              valueIS;
757     const PetscInt *values;
758     PetscInt        numValues, val;
759 
760     PetscCall(DMGetLabelName(dm, l, &lname));
761     PetscCall(PetscStrcmp(lname, "depth", &isDepth));
762     if (isDepth) continue;
763     PetscCall(PetscStrcmp(lname, "dim", &isDim));
764     if (isDim) continue;
765     PetscCall(DMCreateLabel(dmNew, lname));
766     PetscCall(DMGetLabel(dm, lname, &label));
767     PetscCall(DMGetLabel(dmNew, lname, &newlabel));
768     PetscCall(DMLabelGetDefaultValue(label, &val));
769     PetscCall(DMLabelSetDefaultValue(newlabel, val));
770     PetscCall(DMLabelGetValueIS(label, &valueIS));
771     PetscCall(ISGetLocalSize(valueIS, &numValues));
772     PetscCall(ISGetIndices(valueIS, &values));
773     for (val = 0; val < numValues; ++val) {
774       IS              pointIS;
775       const PetscInt *points;
776       PetscInt        numPoints, p;
777 
778       PetscCall(DMLabelGetStratumIS(label, values[val], &pointIS));
779       PetscCall(ISGetLocalSize(pointIS, &numPoints));
780       PetscCall(ISGetIndices(pointIS, &points));
781       for (p = 0; p < numPoints; ++p) {
782         const PetscInt newpoint = DMPlexShiftPoint_Internal(points[p], depth, depthShift);
783 
784         PetscCall(DMLabelSetValue(newlabel, newpoint, values[val]));
785       }
786       PetscCall(ISRestoreIndices(pointIS, &points));
787       PetscCall(ISDestroy(&pointIS));
788     }
789     PetscCall(ISRestoreIndices(valueIS, &values));
790     PetscCall(ISDestroy(&valueIS));
791   }
792   PetscFunctionReturn(PETSC_SUCCESS);
793 }
794 
795 static PetscErrorCode DMPlexCreateVTKLabel_Internal(DM dm, PetscBool createGhostLabel, DM dmNew)
796 {
797   PetscSF            sfPoint;
798   DMLabel            vtkLabel, ghostLabel = NULL;
799   const PetscSFNode *leafRemote;
800   const PetscInt    *leafLocal;
801   PetscInt           cellHeight, cStart, cEnd, c, fStart, fEnd, f, numLeaves, l;
802   PetscMPIInt        rank;
803 
804   PetscFunctionBegin;
805   /* Step 11: Make label for output (vtk) and to mark ghost points (ghost) */
806   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
807   PetscCall(DMGetPointSF(dm, &sfPoint));
808   PetscCall(DMPlexGetVTKCellHeight(dmNew, &cellHeight));
809   PetscCall(DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd));
810   PetscCall(PetscSFGetGraph(sfPoint, NULL, &numLeaves, &leafLocal, &leafRemote));
811   PetscCall(DMCreateLabel(dmNew, "vtk"));
812   PetscCall(DMGetLabel(dmNew, "vtk", &vtkLabel));
813   if (createGhostLabel) {
814     PetscCall(DMCreateLabel(dmNew, "ghost"));
815     PetscCall(DMGetLabel(dmNew, "ghost", &ghostLabel));
816   }
817   for (l = 0, c = cStart; l < numLeaves && c < cEnd; ++l, ++c) {
818     for (; c < leafLocal[l] && c < cEnd; ++c) PetscCall(DMLabelSetValue(vtkLabel, c, 1));
819     if (leafLocal[l] >= cEnd) break;
820     if (leafRemote[l].rank == rank) {
821       PetscCall(DMLabelSetValue(vtkLabel, c, 1));
822     } else if (ghostLabel) PetscCall(DMLabelSetValue(ghostLabel, c, 2));
823   }
824   for (; c < cEnd; ++c) PetscCall(DMLabelSetValue(vtkLabel, c, 1));
825   if (ghostLabel) {
826     PetscCall(DMPlexGetHeightStratum(dmNew, 1, &fStart, &fEnd));
827     for (f = fStart; f < fEnd; ++f) {
828       PetscInt numCells;
829 
830       PetscCall(DMPlexGetSupportSize(dmNew, f, &numCells));
831       if (numCells < 2) {
832         PetscCall(DMLabelSetValue(ghostLabel, f, 1));
833       } else {
834         const PetscInt *cells = NULL;
835         PetscInt        vA, vB;
836 
837         PetscCall(DMPlexGetSupport(dmNew, f, &cells));
838         PetscCall(DMLabelGetValue(vtkLabel, cells[0], &vA));
839         PetscCall(DMLabelGetValue(vtkLabel, cells[1], &vB));
840         if (vA != 1 && vB != 1) PetscCall(DMLabelSetValue(ghostLabel, f, 1));
841       }
842     }
843   }
844   PetscFunctionReturn(PETSC_SUCCESS);
845 }
846 
847 static PetscErrorCode DMPlexShiftTree_Internal(DM dm, PetscInt depthShift[], DM dmNew)
848 {
849   DM           refTree;
850   PetscSection pSec;
851   PetscInt    *parents, *childIDs;
852 
853   PetscFunctionBegin;
854   PetscCall(DMPlexGetReferenceTree(dm, &refTree));
855   PetscCall(DMPlexSetReferenceTree(dmNew, refTree));
856   PetscCall(DMPlexGetTree(dm, &pSec, &parents, &childIDs, NULL, NULL));
857   if (pSec) {
858     PetscInt     p, pStart, pEnd, *parentsShifted, pStartShifted, pEndShifted, depth;
859     PetscInt    *childIDsShifted;
860     PetscSection pSecShifted;
861 
862     PetscCall(PetscSectionGetChart(pSec, &pStart, &pEnd));
863     PetscCall(DMPlexGetDepth(dm, &depth));
864     pStartShifted = DMPlexShiftPoint_Internal(pStart, depth, depthShift);
865     pEndShifted   = DMPlexShiftPoint_Internal(pEnd, depth, depthShift);
866     PetscCall(PetscMalloc2(pEndShifted - pStartShifted, &parentsShifted, pEndShifted - pStartShifted, &childIDsShifted));
867     PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dmNew), &pSecShifted));
868     PetscCall(PetscSectionSetChart(pSecShifted, pStartShifted, pEndShifted));
869     for (p = pStartShifted; p < pEndShifted; p++) {
870       /* start off assuming no children */
871       PetscCall(PetscSectionSetDof(pSecShifted, p, 0));
872     }
873     for (p = pStart; p < pEnd; p++) {
874       PetscInt dof;
875       PetscInt pNew = DMPlexShiftPoint_Internal(p, depth, depthShift);
876 
877       PetscCall(PetscSectionGetDof(pSec, p, &dof));
878       PetscCall(PetscSectionSetDof(pSecShifted, pNew, dof));
879     }
880     PetscCall(PetscSectionSetUp(pSecShifted));
881     for (p = pStart; p < pEnd; p++) {
882       PetscInt dof;
883       PetscInt pNew = DMPlexShiftPoint_Internal(p, depth, depthShift);
884 
885       PetscCall(PetscSectionGetDof(pSec, p, &dof));
886       if (dof) {
887         PetscInt off, offNew;
888 
889         PetscCall(PetscSectionGetOffset(pSec, p, &off));
890         PetscCall(PetscSectionGetOffset(pSecShifted, pNew, &offNew));
891         parentsShifted[offNew]  = DMPlexShiftPoint_Internal(parents[off], depth, depthShift);
892         childIDsShifted[offNew] = childIDs[off];
893       }
894     }
895     PetscCall(DMPlexSetTree(dmNew, pSecShifted, parentsShifted, childIDsShifted));
896     PetscCall(PetscFree2(parentsShifted, childIDsShifted));
897     PetscCall(PetscSectionDestroy(&pSecShifted));
898   }
899   PetscFunctionReturn(PETSC_SUCCESS);
900 }
901 
902 static PetscErrorCode DMPlexConstructGhostCells_Internal(DM dm, DMLabel label, PetscInt *numGhostCells, DM gdm)
903 {
904   PetscSF          sf;
905   IS               valueIS;
906   const PetscInt  *values, *leaves;
907   PetscInt        *depthShift;
908   PetscInt         d, depth = 0, nleaves, loc, Ng, numFS, fs, fStart, fEnd, ghostCell, cEnd, c;
909   const PetscReal *maxCell, *Lstart, *L;
910 
911   PetscFunctionBegin;
912   PetscCall(DMGetPointSF(dm, &sf));
913   PetscCall(PetscSFGetGraph(sf, NULL, &nleaves, &leaves, NULL));
914   nleaves = PetscMax(0, nleaves);
915   PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
916   /* Count ghost cells */
917   PetscCall(DMLabelGetValueIS(label, &valueIS));
918   PetscCall(ISGetLocalSize(valueIS, &numFS));
919   PetscCall(ISGetIndices(valueIS, &values));
920   Ng = 0;
921   for (fs = 0; fs < numFS; ++fs) {
922     IS              faceIS;
923     const PetscInt *faces;
924     PetscInt        numFaces, f, numBdFaces = 0;
925 
926     PetscCall(DMLabelGetStratumIS(label, values[fs], &faceIS));
927     PetscCall(ISGetLocalSize(faceIS, &numFaces));
928     PetscCall(ISGetIndices(faceIS, &faces));
929     for (f = 0; f < numFaces; ++f) {
930       PetscInt numChildren;
931 
932       PetscCall(PetscFindInt(faces[f], nleaves, leaves, &loc));
933       PetscCall(DMPlexGetTreeChildren(dm, faces[f], &numChildren, NULL));
934       /* non-local and ancestors points don't get to register ghosts */
935       if (loc >= 0 || numChildren) continue;
936       if ((faces[f] >= fStart) && (faces[f] < fEnd)) ++numBdFaces;
937     }
938     Ng += numBdFaces;
939     PetscCall(ISRestoreIndices(faceIS, &faces));
940     PetscCall(ISDestroy(&faceIS));
941   }
942   PetscCall(DMPlexGetDepth(dm, &depth));
943   PetscCall(PetscMalloc1(2 * (depth + 1), &depthShift));
944   for (d = 0; d <= depth; d++) {
945     PetscInt dEnd;
946 
947     PetscCall(DMPlexGetDepthStratum(dm, d, NULL, &dEnd));
948     depthShift[2 * d]     = dEnd;
949     depthShift[2 * d + 1] = 0;
950   }
951   if (depth >= 0) depthShift[2 * depth + 1] = Ng;
952   PetscCall(DMPlexShiftPointSetUp_Internal(depth, depthShift));
953   PetscCall(DMPlexShiftSizes_Internal(dm, depthShift, gdm));
954   /* Step 3: Set cone/support sizes for new points */
955   PetscCall(DMPlexGetHeightStratum(dm, 0, NULL, &cEnd));
956   for (c = cEnd; c < cEnd + Ng; ++c) PetscCall(DMPlexSetConeSize(gdm, c, 1));
957   for (fs = 0; fs < numFS; ++fs) {
958     IS              faceIS;
959     const PetscInt *faces;
960     PetscInt        numFaces, f;
961 
962     PetscCall(DMLabelGetStratumIS(label, values[fs], &faceIS));
963     PetscCall(ISGetLocalSize(faceIS, &numFaces));
964     PetscCall(ISGetIndices(faceIS, &faces));
965     for (f = 0; f < numFaces; ++f) {
966       PetscInt size, numChildren;
967 
968       PetscCall(PetscFindInt(faces[f], nleaves, leaves, &loc));
969       PetscCall(DMPlexGetTreeChildren(dm, faces[f], &numChildren, NULL));
970       if (loc >= 0 || numChildren) continue;
971       if ((faces[f] < fStart) || (faces[f] >= fEnd)) continue;
972       PetscCall(DMPlexGetSupportSize(dm, faces[f], &size));
973       PetscCheck(size == 1, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM has boundary face %" PetscInt_FMT " with %" PetscInt_FMT " support cells", faces[f], size);
974       PetscCall(DMPlexSetSupportSize(gdm, faces[f] + Ng, 2));
975     }
976     PetscCall(ISRestoreIndices(faceIS, &faces));
977     PetscCall(ISDestroy(&faceIS));
978   }
979   /* Step 4: Setup ghosted DM */
980   PetscCall(DMSetUp(gdm));
981   PetscCall(DMPlexShiftPoints_Internal(dm, depthShift, gdm));
982   /* Step 6: Set cones and supports for new points */
983   ghostCell = cEnd;
984   for (fs = 0; fs < numFS; ++fs) {
985     IS              faceIS;
986     const PetscInt *faces;
987     PetscInt        numFaces, f;
988 
989     PetscCall(DMLabelGetStratumIS(label, values[fs], &faceIS));
990     PetscCall(ISGetLocalSize(faceIS, &numFaces));
991     PetscCall(ISGetIndices(faceIS, &faces));
992     for (f = 0; f < numFaces; ++f) {
993       PetscInt newFace = faces[f] + Ng, numChildren;
994 
995       PetscCall(PetscFindInt(faces[f], nleaves, leaves, &loc));
996       PetscCall(DMPlexGetTreeChildren(dm, faces[f], &numChildren, NULL));
997       if (loc >= 0 || numChildren) continue;
998       if ((faces[f] < fStart) || (faces[f] >= fEnd)) continue;
999       PetscCall(DMPlexSetCone(gdm, ghostCell, &newFace));
1000       PetscCall(DMPlexInsertSupport(gdm, newFace, 1, ghostCell));
1001       ++ghostCell;
1002     }
1003     PetscCall(ISRestoreIndices(faceIS, &faces));
1004     PetscCall(ISDestroy(&faceIS));
1005   }
1006   PetscCall(ISRestoreIndices(valueIS, &values));
1007   PetscCall(ISDestroy(&valueIS));
1008   PetscCall(DMPlexShiftCoordinates_Internal(dm, depthShift, gdm));
1009   PetscCall(DMPlexShiftSF_Internal(dm, depthShift, gdm));
1010   PetscCall(DMPlexShiftLabels_Internal(dm, depthShift, gdm));
1011   PetscCall(DMPlexCreateVTKLabel_Internal(dm, PETSC_TRUE, gdm));
1012   PetscCall(DMPlexShiftTree_Internal(dm, depthShift, gdm));
1013   PetscCall(PetscFree(depthShift));
1014   for (c = cEnd; c < cEnd + Ng; ++c) PetscCall(DMPlexSetCellType(gdm, c, DM_POLYTOPE_FV_GHOST));
1015   /* Step 7: Periodicity */
1016   PetscCall(DMGetPeriodicity(dm, &maxCell, &Lstart, &L));
1017   PetscCall(DMSetPeriodicity(gdm, maxCell, Lstart, L));
1018   if (numGhostCells) *numGhostCells = Ng;
1019   PetscFunctionReturn(PETSC_SUCCESS);
1020 }
1021 
1022 /*@C
1023   DMPlexConstructGhostCells - Construct ghost cells which connect to every boundary face
1024 
1025   Collective
1026 
1027   Input Parameters:
1028 + dm        - The original `DM`
1029 - labelName - The label specifying the boundary faces, or "Face Sets" if this is `NULL`
1030 
1031   Output Parameters:
1032 + numGhostCells - The number of ghost cells added to the `DM`
1033 - dmGhosted     - The new `DM`
1034 
1035   Level: developer
1036 
1037   Note:
1038   If no label exists of that name, one will be created marking all boundary faces
1039 
1040 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMCreate()`
1041 @*/
1042 PetscErrorCode DMPlexConstructGhostCells(DM dm, const char labelName[], PetscInt *numGhostCells, DM *dmGhosted)
1043 {
1044   DM          gdm;
1045   DMLabel     label;
1046   const char *name = labelName ? labelName : "Face Sets";
1047   PetscInt    dim, Ng = 0;
1048   PetscBool   useCone, useClosure;
1049 
1050   PetscFunctionBegin;
1051   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1052   if (numGhostCells) PetscAssertPointer(numGhostCells, 3);
1053   PetscAssertPointer(dmGhosted, 4);
1054   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &gdm));
1055   PetscCall(DMSetType(gdm, DMPLEX));
1056   PetscCall(DMGetDimension(dm, &dim));
1057   PetscCall(DMSetDimension(gdm, dim));
1058   PetscCall(DMGetBasicAdjacency(dm, &useCone, &useClosure));
1059   PetscCall(DMSetBasicAdjacency(gdm, useCone, useClosure));
1060   PetscCall(DMGetLabel(dm, name, &label));
1061   if (!label) {
1062     /* Get label for boundary faces */
1063     PetscCall(DMCreateLabel(dm, name));
1064     PetscCall(DMGetLabel(dm, name, &label));
1065     PetscCall(DMPlexMarkBoundaryFaces(dm, 1, label));
1066   }
1067   PetscCall(DMPlexConstructGhostCells_Internal(dm, label, &Ng, gdm));
1068   PetscCall(DMCopyDisc(dm, gdm));
1069   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_TRUE, gdm));
1070   gdm->setfromoptionscalled = dm->setfromoptionscalled;
1071   if (numGhostCells) *numGhostCells = Ng;
1072   *dmGhosted = gdm;
1073   PetscFunctionReturn(PETSC_SUCCESS);
1074 }
1075 
1076 static PetscErrorCode DivideCells_Private(DM dm, DMLabel label, DMPlexPointQueue queue)
1077 {
1078   PetscInt dim, d, shift = 100, *pStart, *pEnd;
1079 
1080   PetscFunctionBegin;
1081   PetscCall(DMGetDimension(dm, &dim));
1082   PetscCall(PetscMalloc2(dim, &pStart, dim, &pEnd));
1083   for (d = 0; d < dim; ++d) PetscCall(DMPlexGetDepthStratum(dm, d, &pStart[d], &pEnd[d]));
1084   while (!DMPlexPointQueueEmpty(queue)) {
1085     PetscInt  cell    = -1;
1086     PetscInt *closure = NULL;
1087     PetscInt  closureSize, cl, cval;
1088 
1089     PetscCall(DMPlexPointQueueDequeue(queue, &cell));
1090     PetscCall(DMLabelGetValue(label, cell, &cval));
1091     PetscCall(DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
1092     /* Mark points in the cell closure that touch the fault */
1093     for (d = 0; d < dim; ++d) {
1094       for (cl = 0; cl < closureSize * 2; cl += 2) {
1095         const PetscInt clp = closure[cl];
1096         PetscInt       clval;
1097 
1098         if ((clp < pStart[d]) || (clp >= pEnd[d])) continue;
1099         PetscCall(DMLabelGetValue(label, clp, &clval));
1100         if (clval == -1) {
1101           const PetscInt *cone;
1102           PetscInt        coneSize, c;
1103 
1104           /* If a cone point touches the fault, then this point touches the fault */
1105           PetscCall(DMPlexGetCone(dm, clp, &cone));
1106           PetscCall(DMPlexGetConeSize(dm, clp, &coneSize));
1107           for (c = 0; c < coneSize; ++c) {
1108             PetscInt cpval;
1109 
1110             PetscCall(DMLabelGetValue(label, cone[c], &cpval));
1111             if (cpval != -1) {
1112               PetscInt dep;
1113 
1114               PetscCall(DMPlexGetPointDepth(dm, clp, &dep));
1115               clval = cval < 0 ? -(shift + dep) : shift + dep;
1116               PetscCall(DMLabelSetValue(label, clp, clval));
1117               break;
1118             }
1119           }
1120         }
1121         /* Mark neighbor cells through marked faces (these cells must also touch the fault) */
1122         if (d == dim - 1 && clval != -1) {
1123           const PetscInt *support;
1124           PetscInt        supportSize, s, nval;
1125 
1126           PetscCall(DMPlexGetSupport(dm, clp, &support));
1127           PetscCall(DMPlexGetSupportSize(dm, clp, &supportSize));
1128           for (s = 0; s < supportSize; ++s) {
1129             PetscCall(DMLabelGetValue(label, support[s], &nval));
1130             if (nval == -1) {
1131               PetscCall(DMLabelSetValue(label, support[s], clval < 0 ? clval - 1 : clval + 1));
1132               PetscCall(DMPlexPointQueueEnqueue(queue, support[s]));
1133             }
1134           }
1135         }
1136       }
1137     }
1138     PetscCall(DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
1139   }
1140   PetscCall(PetscFree2(pStart, pEnd));
1141   PetscFunctionReturn(PETSC_SUCCESS);
1142 }
1143 
1144 typedef struct {
1145   DM               dm;
1146   DMPlexPointQueue queue;
1147 } PointDivision;
1148 
1149 static PetscErrorCode divideCell(DMLabel label, PetscInt p, PetscInt val, void *ctx)
1150 {
1151   PointDivision  *div  = (PointDivision *)ctx;
1152   PetscInt        cval = val < 0 ? val - 1 : val + 1;
1153   const PetscInt *support;
1154   PetscInt        supportSize, s;
1155 
1156   PetscFunctionBegin;
1157   PetscCall(DMPlexGetSupport(div->dm, p, &support));
1158   PetscCall(DMPlexGetSupportSize(div->dm, p, &supportSize));
1159   for (s = 0; s < supportSize; ++s) {
1160     PetscCall(DMLabelSetValue(label, support[s], cval));
1161     PetscCall(DMPlexPointQueueEnqueue(div->queue, support[s]));
1162   }
1163   PetscFunctionReturn(PETSC_SUCCESS);
1164 }
1165 
1166 /* Mark cells by label propagation */
1167 static PetscErrorCode DMPlexLabelFaultHalo(DM dm, DMLabel faultLabel)
1168 {
1169   DMPlexPointQueue queue = NULL;
1170   PointDivision    div;
1171   PetscSF          pointSF;
1172   IS               pointIS;
1173   const PetscInt  *points;
1174   PetscBool        empty;
1175   PetscInt         dim, shift = 100, n, i;
1176 
1177   PetscFunctionBegin;
1178   PetscCall(DMGetDimension(dm, &dim));
1179   PetscCall(DMPlexPointQueueCreate(1024, &queue));
1180   div.dm    = dm;
1181   div.queue = queue;
1182   /* Enqueue cells on fault */
1183   PetscCall(DMLabelGetStratumIS(faultLabel, shift + dim, &pointIS));
1184   if (pointIS) {
1185     PetscCall(ISGetLocalSize(pointIS, &n));
1186     PetscCall(ISGetIndices(pointIS, &points));
1187     for (i = 0; i < n; ++i) PetscCall(DMPlexPointQueueEnqueue(queue, points[i]));
1188     PetscCall(ISRestoreIndices(pointIS, &points));
1189     PetscCall(ISDestroy(&pointIS));
1190   }
1191   PetscCall(DMLabelGetStratumIS(faultLabel, -(shift + dim), &pointIS));
1192   if (pointIS) {
1193     PetscCall(ISGetLocalSize(pointIS, &n));
1194     PetscCall(ISGetIndices(pointIS, &points));
1195     for (i = 0; i < n; ++i) PetscCall(DMPlexPointQueueEnqueue(queue, points[i]));
1196     PetscCall(ISRestoreIndices(pointIS, &points));
1197     PetscCall(ISDestroy(&pointIS));
1198   }
1199 
1200   PetscCall(DMGetPointSF(dm, &pointSF));
1201   PetscCall(DMLabelPropagateBegin(faultLabel, pointSF));
1202   /* While edge queue is not empty: */
1203   PetscCall(DMPlexPointQueueEmptyCollective((PetscObject)dm, queue, &empty));
1204   while (!empty) {
1205     PetscCall(DivideCells_Private(dm, faultLabel, queue));
1206     PetscCall(DMLabelPropagatePush(faultLabel, pointSF, divideCell, &div));
1207     PetscCall(DMPlexPointQueueEmptyCollective((PetscObject)dm, queue, &empty));
1208   }
1209   PetscCall(DMLabelPropagateEnd(faultLabel, pointSF));
1210   PetscCall(DMPlexPointQueueDestroy(&queue));
1211   PetscFunctionReturn(PETSC_SUCCESS);
1212 }
1213 
1214 /*
1215   We are adding three kinds of points here:
1216     Replicated:     Copies of points which exist in the mesh, such as vertices identified across a fault
1217     Non-replicated: Points which exist on the fault, but are not replicated
1218     Ghost:          These are shared fault faces which are not owned by this process. These do not produce hybrid cells and do not replicate
1219     Hybrid:         Entirely new points, such as cohesive cells
1220 
1221   When creating subsequent cohesive cells, we shift the old hybrid cells to the end of the numbering at
1222   each depth so that the new split/hybrid points can be inserted as a block.
1223 */
1224 static PetscErrorCode DMPlexConstructCohesiveCells_Internal(DM dm, DMLabel label, DMLabel splitLabel, DM sdm)
1225 {
1226   MPI_Comm         comm;
1227   IS               valueIS;
1228   PetscInt         numSP = 0; /* The number of depths for which we have replicated points */
1229   const PetscInt  *values;    /* List of depths for which we have replicated points */
1230   IS              *splitIS;
1231   IS              *unsplitIS;
1232   IS               ghostIS;
1233   PetscInt        *numSplitPoints;     /* The number of replicated points at each depth */
1234   PetscInt        *numUnsplitPoints;   /* The number of non-replicated points at each depth which still give rise to hybrid points */
1235   PetscInt        *numHybridPoints;    /* The number of new hybrid points at each depth */
1236   PetscInt        *numHybridPointsOld; /* The number of existing hybrid points at each depth */
1237   PetscInt         numGhostPoints;     /* The number of unowned, shared fault faces */
1238   const PetscInt **splitPoints;        /* Replicated points for each depth */
1239   const PetscInt **unsplitPoints;      /* Non-replicated points for each depth */
1240   const PetscInt  *ghostPoints;        /* Ghost fault faces */
1241   PetscSection     coordSection;
1242   Vec              coordinates;
1243   PetscScalar     *coords;
1244   PetscInt        *depthMax;   /* The first hybrid point at each depth in the original mesh */
1245   PetscInt        *depthEnd;   /* The point limit at each depth in the original mesh */
1246   PetscInt        *depthShift; /* Number of replicated+hybrid points at each depth */
1247   PetscInt        *pMaxNew;    /* The first replicated point at each depth in the new mesh, hybrids come after this */
1248   PetscInt        *coneNew, *coneONew, *supportNew;
1249   PetscInt         shift = 100, shift2 = 200, depth = 0, dep, dim, d, sp, maxConeSize, maxSupportSize, maxConeSizeNew, maxSupportSizeNew, numLabels, vStart, vEnd, pEnd, p, v;
1250 
1251   PetscFunctionBegin;
1252   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
1253   PetscCall(DMGetDimension(dm, &dim));
1254   PetscCall(DMPlexGetDepth(dm, &depth));
1255   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
1256   /* We do not want this label automatically computed, instead we compute it here */
1257   PetscCall(DMCreateLabel(sdm, "celltype"));
1258   /* Count split points and add cohesive cells */
1259   PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize));
1260   PetscCall(PetscMalloc5(depth + 1, &depthMax, depth + 1, &depthEnd, 2 * (depth + 1), &depthShift, depth + 1, &pMaxNew, depth + 1, &numHybridPointsOld));
1261   PetscCall(PetscMalloc7(depth + 1, &splitIS, depth + 1, &unsplitIS, depth + 1, &numSplitPoints, depth + 1, &numUnsplitPoints, depth + 1, &numHybridPoints, depth + 1, &splitPoints, depth + 1, &unsplitPoints));
1262   for (d = 0; d <= depth; ++d) {
1263     PetscCall(DMPlexGetDepthStratum(dm, d, NULL, &pMaxNew[d]));
1264     PetscCall(DMPlexGetTensorPrismBounds_Internal(dm, d, &depthMax[d], NULL));
1265     depthEnd[d]           = pMaxNew[d];
1266     depthMax[d]           = depthMax[d] < 0 ? depthEnd[d] : depthMax[d];
1267     numSplitPoints[d]     = 0;
1268     numUnsplitPoints[d]   = 0;
1269     numHybridPoints[d]    = 0;
1270     numHybridPointsOld[d] = depthMax[d] < 0 ? 0 : depthEnd[d] - depthMax[d];
1271     splitPoints[d]        = NULL;
1272     unsplitPoints[d]      = NULL;
1273     splitIS[d]            = NULL;
1274     unsplitIS[d]          = NULL;
1275     /* we are shifting the existing hybrid points with the stratum behind them, so
1276      * the split comes at the end of the normal points, i.e., at depthMax[d] */
1277     depthShift[2 * d]     = depthMax[d];
1278     depthShift[2 * d + 1] = 0;
1279   }
1280   numGhostPoints = 0;
1281   ghostPoints    = NULL;
1282   if (label) {
1283     PetscCall(DMLabelGetValueIS(label, &valueIS));
1284     PetscCall(ISGetLocalSize(valueIS, &numSP));
1285     PetscCall(ISGetIndices(valueIS, &values));
1286   }
1287   for (sp = 0; sp < numSP; ++sp) {
1288     const PetscInt dep = values[sp];
1289 
1290     if ((dep < 0) || (dep > depth)) continue;
1291     PetscCall(DMLabelGetStratumIS(label, dep, &splitIS[dep]));
1292     if (splitIS[dep]) {
1293       PetscCall(ISGetLocalSize(splitIS[dep], &numSplitPoints[dep]));
1294       PetscCall(ISGetIndices(splitIS[dep], &splitPoints[dep]));
1295     }
1296     PetscCall(DMLabelGetStratumIS(label, shift2 + dep, &unsplitIS[dep]));
1297     if (unsplitIS[dep]) {
1298       PetscCall(ISGetLocalSize(unsplitIS[dep], &numUnsplitPoints[dep]));
1299       PetscCall(ISGetIndices(unsplitIS[dep], &unsplitPoints[dep]));
1300     }
1301   }
1302   PetscCall(DMLabelGetStratumIS(label, shift2 + dim - 1, &ghostIS));
1303   if (ghostIS) {
1304     PetscCall(ISGetLocalSize(ghostIS, &numGhostPoints));
1305     PetscCall(ISGetIndices(ghostIS, &ghostPoints));
1306   }
1307   /* Calculate number of hybrid points */
1308   for (d = 1; d <= depth; ++d) numHybridPoints[d] = numSplitPoints[d - 1] + numUnsplitPoints[d - 1]; /* There is a hybrid cell/face/edge for every split face/edge/vertex   */
1309   for (d = 0; d <= depth; ++d) depthShift[2 * d + 1] = numSplitPoints[d] + numHybridPoints[d];
1310   PetscCall(DMPlexShiftPointSetUp_Internal(depth, depthShift));
1311   /* the end of the points in this stratum that come before the new points:
1312    * shifting pMaxNew[d] gets the new start of the next stratum, then count back the old hybrid points and the newly
1313    * added points */
1314   for (d = 0; d <= depth; ++d) pMaxNew[d] = DMPlexShiftPoint_Internal(pMaxNew[d], depth, depthShift) - (numHybridPointsOld[d] + numSplitPoints[d] + numHybridPoints[d]);
1315   PetscCall(DMPlexShiftSizes_Internal(dm, depthShift, sdm));
1316   /* Step 3: Set cone/support sizes for new points */
1317   for (dep = 0; dep <= depth; ++dep) {
1318     for (p = 0; p < numSplitPoints[dep]; ++p) {
1319       const PetscInt  oldp   = splitPoints[dep][p];
1320       const PetscInt  newp   = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*oldp + depthOffset[dep]*/;
1321       const PetscInt  splitp = p + pMaxNew[dep];
1322       const PetscInt *support;
1323       DMPolytopeType  ct;
1324       PetscInt        coneSize, supportSize, qf, qn, qp, e;
1325 
1326       PetscCall(DMPlexGetConeSize(dm, oldp, &coneSize));
1327       PetscCall(DMPlexSetConeSize(sdm, splitp, coneSize));
1328       PetscCall(DMPlexGetSupportSize(dm, oldp, &supportSize));
1329       PetscCall(DMPlexSetSupportSize(sdm, splitp, supportSize));
1330       PetscCall(DMPlexGetCellType(dm, oldp, &ct));
1331       PetscCall(DMPlexSetCellType(sdm, splitp, ct));
1332       if (dep == depth - 1) {
1333         const PetscInt hybcell = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1];
1334 
1335         /* Add cohesive cells, they are prisms */
1336         PetscCall(DMPlexSetConeSize(sdm, hybcell, 2 + coneSize));
1337         switch (coneSize) {
1338         case 2:
1339           PetscCall(DMPlexSetCellType(sdm, hybcell, DM_POLYTOPE_SEG_PRISM_TENSOR));
1340           break;
1341         case 3:
1342           PetscCall(DMPlexSetCellType(sdm, hybcell, DM_POLYTOPE_TRI_PRISM_TENSOR));
1343           break;
1344         case 4:
1345           PetscCall(DMPlexSetCellType(sdm, hybcell, DM_POLYTOPE_QUAD_PRISM_TENSOR));
1346           break;
1347         }
1348         /* Shared fault faces with only one support cell now have two with the cohesive cell */
1349         /*   TODO Check thaat oldp has rootdegree == 1 */
1350         if (supportSize == 1) {
1351           const PetscInt *support;
1352           PetscInt        val;
1353 
1354           PetscCall(DMPlexGetSupport(dm, oldp, &support));
1355           PetscCall(DMLabelGetValue(label, support[0], &val));
1356           if (val < 0) PetscCall(DMPlexSetSupportSize(sdm, splitp, 2));
1357           else PetscCall(DMPlexSetSupportSize(sdm, newp, 2));
1358         }
1359       } else if (dep == 0) {
1360         const PetscInt hybedge = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1];
1361 
1362         PetscCall(DMPlexGetSupport(dm, oldp, &support));
1363         for (e = 0, qn = 0, qp = 0, qf = 0; e < supportSize; ++e) {
1364           PetscInt val;
1365 
1366           PetscCall(DMLabelGetValue(label, support[e], &val));
1367           if (val == 1) ++qf;
1368           if ((val == 1) || (val == (shift + 1))) ++qn;
1369           if ((val == 1) || (val == -(shift + 1))) ++qp;
1370         }
1371         /* Split old vertex: Edges into original vertex and new cohesive edge */
1372         PetscCall(DMPlexSetSupportSize(sdm, newp, qn + 1));
1373         /* Split new vertex: Edges into split vertex and new cohesive edge */
1374         PetscCall(DMPlexSetSupportSize(sdm, splitp, qp + 1));
1375         /* Add hybrid edge */
1376         PetscCall(DMPlexSetConeSize(sdm, hybedge, 2));
1377         PetscCall(DMPlexSetSupportSize(sdm, hybedge, qf));
1378         PetscCall(DMPlexSetCellType(sdm, hybedge, DM_POLYTOPE_POINT_PRISM_TENSOR));
1379       } else if (dep == dim - 2) {
1380         const PetscInt hybface = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1];
1381 
1382         PetscCall(DMPlexGetSupport(dm, oldp, &support));
1383         for (e = 0, qn = 0, qp = 0, qf = 0; e < supportSize; ++e) {
1384           PetscInt val;
1385 
1386           PetscCall(DMLabelGetValue(label, support[e], &val));
1387           if (val == dim - 1) ++qf;
1388           if ((val == dim - 1) || (val == (shift + dim - 1))) ++qn;
1389           if ((val == dim - 1) || (val == -(shift + dim - 1))) ++qp;
1390         }
1391         /* Split old edge: Faces into original edge and cohesive face (positive side?) */
1392         PetscCall(DMPlexSetSupportSize(sdm, newp, qn + 1));
1393         /* Split new edge: Faces into split edge and cohesive face (negative side?) */
1394         PetscCall(DMPlexSetSupportSize(sdm, splitp, qp + 1));
1395         /* Add hybrid face */
1396         PetscCall(DMPlexSetConeSize(sdm, hybface, 4));
1397         PetscCall(DMPlexSetSupportSize(sdm, hybface, qf));
1398         PetscCall(DMPlexSetCellType(sdm, hybface, DM_POLYTOPE_SEG_PRISM_TENSOR));
1399       }
1400     }
1401   }
1402   for (dep = 0; dep <= depth; ++dep) {
1403     for (p = 0; p < numUnsplitPoints[dep]; ++p) {
1404       const PetscInt  oldp = unsplitPoints[dep][p];
1405       const PetscInt  newp = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*oldp + depthOffset[dep]*/;
1406       const PetscInt *support;
1407       PetscInt        coneSize, supportSize, qf, e, s;
1408 
1409       PetscCall(DMPlexGetConeSize(dm, oldp, &coneSize));
1410       PetscCall(DMPlexGetSupportSize(dm, oldp, &supportSize));
1411       PetscCall(DMPlexGetSupport(dm, oldp, &support));
1412       if (dep == 0) {
1413         const PetscInt hybedge = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1] + numSplitPoints[dep];
1414 
1415         /* Unsplit vertex: Edges into original vertex, split edges, and new cohesive edge twice */
1416         for (s = 0, qf = 0; s < supportSize; ++s, ++qf) {
1417           PetscCall(PetscFindInt(support[s], numSplitPoints[dep + 1], splitPoints[dep + 1], &e));
1418           if (e >= 0) ++qf;
1419         }
1420         PetscCall(DMPlexSetSupportSize(sdm, newp, qf + 2));
1421         /* Add hybrid edge */
1422         PetscCall(DMPlexSetConeSize(sdm, hybedge, 2));
1423         for (e = 0, qf = 0; e < supportSize; ++e) {
1424           PetscInt val;
1425 
1426           PetscCall(DMLabelGetValue(label, support[e], &val));
1427           /* Split and unsplit edges produce hybrid faces */
1428           if (val == 1) ++qf;
1429           if (val == (shift2 + 1)) ++qf;
1430         }
1431         PetscCall(DMPlexSetSupportSize(sdm, hybedge, qf));
1432         PetscCall(DMPlexSetCellType(sdm, hybedge, DM_POLYTOPE_POINT_PRISM_TENSOR));
1433       } else if (dep == dim - 2) {
1434         const PetscInt hybface = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1] + numSplitPoints[dep];
1435         PetscInt       val;
1436 
1437         for (e = 0, qf = 0; e < supportSize; ++e) {
1438           PetscCall(DMLabelGetValue(label, support[e], &val));
1439           if (val == dim - 1) qf += 2;
1440           else ++qf;
1441         }
1442         /* Unsplit edge: Faces into original edge, split face, and cohesive face twice */
1443         PetscCall(DMPlexSetSupportSize(sdm, newp, qf + 2));
1444         /* Add hybrid face */
1445         for (e = 0, qf = 0; e < supportSize; ++e) {
1446           PetscCall(DMLabelGetValue(label, support[e], &val));
1447           if (val == dim - 1) ++qf;
1448         }
1449         PetscCall(DMPlexSetConeSize(sdm, hybface, 4));
1450         PetscCall(DMPlexSetSupportSize(sdm, hybface, qf));
1451         PetscCall(DMPlexSetCellType(sdm, hybface, DM_POLYTOPE_SEG_PRISM_TENSOR));
1452       }
1453     }
1454   }
1455   /* Step 4: Setup split DM */
1456   PetscCall(DMSetUp(sdm));
1457   PetscCall(DMPlexShiftPoints_Internal(dm, depthShift, sdm));
1458   PetscCall(DMPlexGetMaxSizes(sdm, &maxConeSizeNew, &maxSupportSizeNew));
1459   PetscCall(PetscMalloc3(PetscMax(maxConeSize, maxConeSizeNew) * 3, &coneNew, PetscMax(maxConeSize, maxConeSizeNew) * 3, &coneONew, PetscMax(maxSupportSize, maxSupportSizeNew), &supportNew));
1460   /* Step 6: Set cones and supports for new points */
1461   for (dep = 0; dep <= depth; ++dep) {
1462     for (p = 0; p < numSplitPoints[dep]; ++p) {
1463       const PetscInt  oldp   = splitPoints[dep][p];
1464       const PetscInt  newp   = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*oldp + depthOffset[dep]*/;
1465       const PetscInt  splitp = p + pMaxNew[dep];
1466       const PetscInt *cone, *support, *ornt;
1467       DMPolytopeType  ct;
1468       PetscInt        coneSize, supportSize, q, qf, qn, qp, v, e, s;
1469 
1470       PetscCall(DMPlexGetCellType(dm, oldp, &ct));
1471       PetscCall(DMPlexGetConeSize(dm, oldp, &coneSize));
1472       PetscCall(DMPlexGetCone(dm, oldp, &cone));
1473       PetscCall(DMPlexGetConeOrientation(dm, oldp, &ornt));
1474       PetscCall(DMPlexGetSupportSize(dm, oldp, &supportSize));
1475       PetscCall(DMPlexGetSupport(dm, oldp, &support));
1476       if (dep == depth - 1) {
1477         PetscBool       hasUnsplit = PETSC_FALSE;
1478         const PetscInt  hybcell    = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1];
1479         const PetscInt *supportF;
1480 
1481         coneONew[0] = coneONew[1] = -1000;
1482         /* Split face:       copy in old face to new face to start */
1483         PetscCall(DMPlexGetSupport(sdm, newp, &supportF));
1484         PetscCall(DMPlexSetSupport(sdm, splitp, supportF));
1485         /* Split old face:   old vertices/edges in cone so no change */
1486         /* Split new face:   new vertices/edges in cone */
1487         for (q = 0; q < coneSize; ++q) {
1488           PetscCall(PetscFindInt(cone[q], numSplitPoints[dep - 1], splitPoints[dep - 1], &v));
1489           if (v < 0) {
1490             PetscCall(PetscFindInt(cone[q], numUnsplitPoints[dep - 1], unsplitPoints[dep - 1], &v));
1491             PetscCheck(v >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %" PetscInt_FMT " in split or unsplit points of depth %" PetscInt_FMT, cone[q], dep - 1);
1492             coneNew[2 + q] = DMPlexShiftPoint_Internal(cone[q], depth, depthShift) /*cone[q] + depthOffset[dep-1]*/;
1493             hasUnsplit     = PETSC_TRUE;
1494           } else {
1495             coneNew[2 + q] = v + pMaxNew[dep - 1];
1496             if (dep > 1) {
1497               const PetscInt *econe;
1498               PetscInt        econeSize, r, vs, vu;
1499 
1500               PetscCall(DMPlexGetConeSize(dm, cone[q], &econeSize));
1501               PetscCall(DMPlexGetCone(dm, cone[q], &econe));
1502               for (r = 0; r < econeSize; ++r) {
1503                 PetscCall(PetscFindInt(econe[r], numSplitPoints[dep - 2], splitPoints[dep - 2], &vs));
1504                 PetscCall(PetscFindInt(econe[r], numUnsplitPoints[dep - 2], unsplitPoints[dep - 2], &vu));
1505                 if (vs >= 0) continue;
1506                 PetscCheck(vu >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %" PetscInt_FMT " in split or unsplit points of depth %" PetscInt_FMT, econe[r], dep - 2);
1507                 hasUnsplit = PETSC_TRUE;
1508               }
1509             }
1510           }
1511         }
1512         PetscCall(DMPlexSetCone(sdm, splitp, &coneNew[2]));
1513         PetscCall(DMPlexSetConeOrientation(sdm, splitp, ornt));
1514         /* Face support */
1515         PetscInt vals[2];
1516 
1517         PetscCall(DMLabelGetValue(label, support[0], &vals[0]));
1518         if (supportSize > 1) PetscCall(DMLabelGetValue(label, support[1], &vals[1]));
1519         else vals[1] = -vals[0];
1520         PetscCheck(vals[0] * vals[1] < 0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid support labels %" PetscInt_FMT " %" PetscInt_FMT, vals[0], vals[1]);
1521 
1522         for (s = 0; s < 2; ++s) {
1523           if (s >= supportSize) {
1524             if (vals[s] < 0) {
1525               /* Ghost old face:   Replace negative side cell with cohesive cell */
1526               PetscCall(DMPlexInsertSupport(sdm, newp, s, hybcell));
1527             } else {
1528               /* Ghost new face:   Replace positive side cell with cohesive cell */
1529               PetscCall(DMPlexInsertSupport(sdm, splitp, s, hybcell));
1530             }
1531           } else {
1532             if (vals[s] < 0) {
1533               /* Split old face:   Replace negative side cell with cohesive cell */
1534               PetscCall(DMPlexInsertSupport(sdm, newp, s, hybcell));
1535             } else {
1536               /* Split new face:   Replace positive side cell with cohesive cell */
1537               PetscCall(DMPlexInsertSupport(sdm, splitp, s, hybcell));
1538             }
1539           }
1540         }
1541         /* Get orientation for cohesive face using the positive side cell */
1542         {
1543           const PetscInt *ncone, *nconeO;
1544           PetscInt        nconeSize, nc, ocell;
1545           PetscBool       flip = PETSC_FALSE;
1546 
1547           if (supportSize > 1) {
1548             ocell = vals[0] < 0 ? support[1] : support[0];
1549           } else {
1550             ocell = support[0];
1551             flip  = vals[0] < 0 ? PETSC_TRUE : PETSC_FALSE;
1552           }
1553           PetscCall(DMPlexGetConeSize(dm, ocell, &nconeSize));
1554           PetscCall(DMPlexGetCone(dm, ocell, &ncone));
1555           PetscCall(DMPlexGetConeOrientation(dm, ocell, &nconeO));
1556           for (nc = 0; nc < nconeSize; ++nc) {
1557             if (ncone[nc] == oldp) {
1558               coneONew[0] = flip ? -(nconeO[nc] + 1) : nconeO[nc];
1559               break;
1560             }
1561           }
1562           PetscCheck(nc < nconeSize, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate face %" PetscInt_FMT " in neighboring cell %" PetscInt_FMT, oldp, ocell);
1563         }
1564         /* Cohesive cell:    Old and new split face, then new cohesive faces */
1565         {
1566           const PetscInt No = DMPolytopeTypeGetNumArrangments(ct) / 2;
1567           PetscCheck((coneONew[0] >= -No) && (coneONew[0] < No), PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid %s orientation %" PetscInt_FMT, DMPolytopeTypes[ct], coneONew[0]);
1568         }
1569         const PetscInt *arr = DMPolytopeTypeGetArrangment(ct, coneONew[0]);
1570 
1571         coneNew[0]  = newp; /* Extracted negative side orientation above */
1572         coneNew[1]  = splitp;
1573         coneONew[1] = coneONew[0];
1574         for (q = 0; q < coneSize; ++q) {
1575           /* Hybrid faces must follow order from oriented end face */
1576           const PetscInt qa = arr[q * 2 + 0];
1577           const PetscInt qo = arr[q * 2 + 1];
1578           DMPolytopeType ft = dep == 2 ? DM_POLYTOPE_SEGMENT : DM_POLYTOPE_POINT;
1579 
1580           PetscCall(PetscFindInt(cone[qa], numSplitPoints[dep - 1], splitPoints[dep - 1], &v));
1581           if (v < 0) {
1582             PetscCall(PetscFindInt(cone[qa], numUnsplitPoints[dep - 1], unsplitPoints[dep - 1], &v));
1583             coneNew[2 + q] = v + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep - 1];
1584           } else {
1585             coneNew[2 + q] = v + pMaxNew[dep] + numSplitPoints[dep];
1586           }
1587           coneONew[2 + q] = DMPolytopeTypeComposeOrientation(ft, qo, ornt[qa]);
1588         }
1589         PetscCall(DMPlexSetCone(sdm, hybcell, coneNew));
1590         PetscCall(DMPlexSetConeOrientation(sdm, hybcell, coneONew));
1591         /* Label the hybrid cells on the boundary of the split */
1592         if (hasUnsplit) PetscCall(DMLabelSetValue(label, -hybcell, dim));
1593       } else if (dep == 0) {
1594         const PetscInt hybedge = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1];
1595 
1596         /* Split old vertex: Edges in old split faces and new cohesive edge */
1597         for (e = 0, qn = 0; e < supportSize; ++e) {
1598           PetscInt val;
1599 
1600           PetscCall(DMLabelGetValue(label, support[e], &val));
1601           if ((val == 1) || (val == (shift + 1))) supportNew[qn++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1602         }
1603         supportNew[qn] = hybedge;
1604         PetscCall(DMPlexSetSupport(sdm, newp, supportNew));
1605         /* Split new vertex: Edges in new split faces and new cohesive edge */
1606         for (e = 0, qp = 0; e < supportSize; ++e) {
1607           PetscInt val, edge;
1608 
1609           PetscCall(DMLabelGetValue(label, support[e], &val));
1610           if (val == 1) {
1611             PetscCall(PetscFindInt(support[e], numSplitPoints[dep + 1], splitPoints[dep + 1], &edge));
1612             PetscCheck(edge >= 0, comm, PETSC_ERR_ARG_WRONG, "Edge %" PetscInt_FMT " is not a split edge", support[e]);
1613             supportNew[qp++] = edge + pMaxNew[dep + 1];
1614           } else if (val == -(shift + 1)) {
1615             supportNew[qp++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1616           }
1617         }
1618         supportNew[qp] = hybedge;
1619         PetscCall(DMPlexSetSupport(sdm, splitp, supportNew));
1620         /* Hybrid edge:    Old and new split vertex */
1621         coneNew[0] = newp;
1622         coneNew[1] = splitp;
1623         PetscCall(DMPlexSetCone(sdm, hybedge, coneNew));
1624         for (e = 0, qf = 0; e < supportSize; ++e) {
1625           PetscInt val, edge;
1626 
1627           PetscCall(DMLabelGetValue(label, support[e], &val));
1628           if (val == 1) {
1629             PetscCall(PetscFindInt(support[e], numSplitPoints[dep + 1], splitPoints[dep + 1], &edge));
1630             PetscCheck(edge >= 0, comm, PETSC_ERR_ARG_WRONG, "Edge %" PetscInt_FMT " is not a split edge", support[e]);
1631             supportNew[qf++] = edge + pMaxNew[dep + 2] + numSplitPoints[dep + 2];
1632           }
1633         }
1634         PetscCall(DMPlexSetSupport(sdm, hybedge, supportNew));
1635       } else if (dep == dim - 2) {
1636         const PetscInt hybface = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1];
1637 
1638         /* Split old edge:   old vertices in cone so no change */
1639         /* Split new edge:   new vertices in cone */
1640         for (q = 0; q < coneSize; ++q) {
1641           PetscCall(PetscFindInt(cone[q], numSplitPoints[dep - 1], splitPoints[dep - 1], &v));
1642           if (v < 0) {
1643             PetscCall(PetscFindInt(cone[q], numUnsplitPoints[dep - 1], unsplitPoints[dep - 1], &v));
1644             PetscCheck(v >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %" PetscInt_FMT " in split or unsplit points of depth %" PetscInt_FMT, cone[q], dep - 1);
1645             coneNew[q] = DMPlexShiftPoint_Internal(cone[q], depth, depthShift) /*cone[q] + depthOffset[dep-1]*/;
1646           } else {
1647             coneNew[q] = v + pMaxNew[dep - 1];
1648           }
1649         }
1650         PetscCall(DMPlexSetCone(sdm, splitp, coneNew));
1651         /* Split old edge: Faces in positive side cells and old split faces */
1652         for (e = 0, q = 0; e < supportSize; ++e) {
1653           PetscInt val;
1654 
1655           PetscCall(DMLabelGetValue(label, support[e], &val));
1656           if (val == dim - 1) {
1657             supportNew[q++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1658           } else if (val == (shift + dim - 1)) {
1659             supportNew[q++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1660           }
1661         }
1662         supportNew[q++] = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1];
1663         PetscCall(DMPlexSetSupport(sdm, newp, supportNew));
1664         /* Split new edge: Faces in negative side cells and new split faces */
1665         for (e = 0, q = 0; e < supportSize; ++e) {
1666           PetscInt val, face;
1667 
1668           PetscCall(DMLabelGetValue(label, support[e], &val));
1669           if (val == dim - 1) {
1670             PetscCall(PetscFindInt(support[e], numSplitPoints[dep + 1], splitPoints[dep + 1], &face));
1671             PetscCheck(face >= 0, comm, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " is not a split face", support[e]);
1672             supportNew[q++] = face + pMaxNew[dep + 1];
1673           } else if (val == -(shift + dim - 1)) {
1674             supportNew[q++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1675           }
1676         }
1677         supportNew[q++] = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1];
1678         PetscCall(DMPlexSetSupport(sdm, splitp, supportNew));
1679         /* Hybrid face */
1680         coneNew[0] = newp;
1681         coneNew[1] = splitp;
1682         for (v = 0; v < coneSize; ++v) {
1683           PetscInt vertex;
1684           PetscCall(PetscFindInt(cone[v], numSplitPoints[dep - 1], splitPoints[dep - 1], &vertex));
1685           if (vertex < 0) {
1686             PetscCall(PetscFindInt(cone[v], numUnsplitPoints[dep - 1], unsplitPoints[dep - 1], &vertex));
1687             PetscCheck(vertex >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %" PetscInt_FMT " in split or unsplit points of depth %" PetscInt_FMT, cone[v], dep - 1);
1688             coneNew[2 + v] = vertex + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep - 1];
1689           } else {
1690             coneNew[2 + v] = vertex + pMaxNew[dep] + numSplitPoints[dep];
1691           }
1692         }
1693         PetscCall(DMPlexSetCone(sdm, hybface, coneNew));
1694         for (e = 0, qf = 0; e < supportSize; ++e) {
1695           PetscInt val, face;
1696 
1697           PetscCall(DMLabelGetValue(label, support[e], &val));
1698           if (val == dim - 1) {
1699             PetscCall(PetscFindInt(support[e], numSplitPoints[dep + 1], splitPoints[dep + 1], &face));
1700             PetscCheck(face >= 0, comm, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " is not a split face", support[e]);
1701             supportNew[qf++] = face + pMaxNew[dep + 2] + numSplitPoints[dep + 2];
1702           }
1703         }
1704         PetscCall(DMPlexSetSupport(sdm, hybface, supportNew));
1705       }
1706     }
1707   }
1708   for (dep = 0; dep <= depth; ++dep) {
1709     for (p = 0; p < numUnsplitPoints[dep]; ++p) {
1710       const PetscInt  oldp = unsplitPoints[dep][p];
1711       const PetscInt  newp = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*oldp + depthOffset[dep]*/;
1712       const PetscInt *cone, *support;
1713       PetscInt        coneSize, supportSize, supportSizeNew, q, qf, e, f, s;
1714 
1715       PetscCall(DMPlexGetConeSize(dm, oldp, &coneSize));
1716       PetscCall(DMPlexGetCone(dm, oldp, &cone));
1717       PetscCall(DMPlexGetSupportSize(dm, oldp, &supportSize));
1718       PetscCall(DMPlexGetSupport(dm, oldp, &support));
1719       if (dep == 0) {
1720         const PetscInt hybedge = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1] + numSplitPoints[dep];
1721 
1722         /* Unsplit vertex */
1723         PetscCall(DMPlexGetSupportSize(sdm, newp, &supportSizeNew));
1724         for (s = 0, q = 0; s < supportSize; ++s) {
1725           supportNew[q++] = DMPlexShiftPoint_Internal(support[s], depth, depthShift) /*support[s] + depthOffset[dep+1]*/;
1726           PetscCall(PetscFindInt(support[s], numSplitPoints[dep + 1], splitPoints[dep + 1], &e));
1727           if (e >= 0) supportNew[q++] = e + pMaxNew[dep + 1];
1728         }
1729         supportNew[q++] = hybedge;
1730         supportNew[q++] = hybedge;
1731         PetscCheck(q == supportSizeNew, comm, PETSC_ERR_ARG_WRONG, "Support size %" PetscInt_FMT " != %" PetscInt_FMT " for vertex %" PetscInt_FMT, q, supportSizeNew, newp);
1732         PetscCall(DMPlexSetSupport(sdm, newp, supportNew));
1733         /* Hybrid edge */
1734         coneNew[0] = newp;
1735         coneNew[1] = newp;
1736         PetscCall(DMPlexSetCone(sdm, hybedge, coneNew));
1737         for (e = 0, qf = 0; e < supportSize; ++e) {
1738           PetscInt val, edge;
1739 
1740           PetscCall(DMLabelGetValue(label, support[e], &val));
1741           if (val == 1) {
1742             PetscCall(PetscFindInt(support[e], numSplitPoints[dep + 1], splitPoints[dep + 1], &edge));
1743             PetscCheck(edge >= 0, comm, PETSC_ERR_ARG_WRONG, "Edge %" PetscInt_FMT " is not a split edge", support[e]);
1744             supportNew[qf++] = edge + pMaxNew[dep + 2] + numSplitPoints[dep + 2];
1745           } else if (val == (shift2 + 1)) {
1746             PetscCall(PetscFindInt(support[e], numUnsplitPoints[dep + 1], unsplitPoints[dep + 1], &edge));
1747             PetscCheck(edge >= 0, comm, PETSC_ERR_ARG_WRONG, "Edge %" PetscInt_FMT " is not a unsplit edge", support[e]);
1748             supportNew[qf++] = edge + pMaxNew[dep + 2] + numSplitPoints[dep + 2] + numSplitPoints[dep + 1];
1749           }
1750         }
1751         PetscCall(DMPlexSetSupport(sdm, hybedge, supportNew));
1752       } else if (dep == dim - 2) {
1753         const PetscInt hybface = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1] + numSplitPoints[dep];
1754 
1755         /* Unsplit edge: Faces into original edge, split face, and hybrid face twice */
1756         for (f = 0, qf = 0; f < supportSize; ++f) {
1757           PetscInt val, face;
1758 
1759           PetscCall(DMLabelGetValue(label, support[f], &val));
1760           if (val == dim - 1) {
1761             PetscCall(PetscFindInt(support[f], numSplitPoints[dep + 1], splitPoints[dep + 1], &face));
1762             PetscCheck(face >= 0, comm, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " is not a split face", support[f]);
1763             supportNew[qf++] = DMPlexShiftPoint_Internal(support[f], depth, depthShift) /*support[f] + depthOffset[dep+1]*/;
1764             supportNew[qf++] = face + pMaxNew[dep + 1];
1765           } else {
1766             supportNew[qf++] = DMPlexShiftPoint_Internal(support[f], depth, depthShift) /*support[f] + depthOffset[dep+1]*/;
1767           }
1768         }
1769         supportNew[qf++] = hybface;
1770         supportNew[qf++] = hybface;
1771         PetscCall(DMPlexGetSupportSize(sdm, newp, &supportSizeNew));
1772         PetscCheck(qf == supportSizeNew, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Support size for unsplit edge %" PetscInt_FMT " is %" PetscInt_FMT " != %" PetscInt_FMT, newp, qf, supportSizeNew);
1773         PetscCall(DMPlexSetSupport(sdm, newp, supportNew));
1774         /* Add hybrid face */
1775         coneNew[0] = newp;
1776         coneNew[1] = newp;
1777         PetscCall(PetscFindInt(cone[0], numUnsplitPoints[dep - 1], unsplitPoints[dep - 1], &v));
1778         PetscCheck(v >= 0, comm, PETSC_ERR_ARG_WRONG, "Vertex %" PetscInt_FMT " is not an unsplit vertex", cone[0]);
1779         coneNew[2] = v + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep - 1];
1780         PetscCall(PetscFindInt(cone[1], numUnsplitPoints[dep - 1], unsplitPoints[dep - 1], &v));
1781         PetscCheck(v >= 0, comm, PETSC_ERR_ARG_WRONG, "Vertex %" PetscInt_FMT " is not an unsplit vertex", cone[1]);
1782         coneNew[3] = v + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep - 1];
1783         PetscCall(DMPlexSetCone(sdm, hybface, coneNew));
1784         for (f = 0, qf = 0; f < supportSize; ++f) {
1785           PetscInt val, face;
1786 
1787           PetscCall(DMLabelGetValue(label, support[f], &val));
1788           if (val == dim - 1) {
1789             PetscCall(PetscFindInt(support[f], numSplitPoints[dep + 1], splitPoints[dep + 1], &face));
1790             supportNew[qf++] = face + pMaxNew[dep + 2] + numSplitPoints[dep + 2];
1791           }
1792         }
1793         PetscCall(DMPlexGetSupportSize(sdm, hybface, &supportSizeNew));
1794         PetscCheck(qf == supportSizeNew, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Support size for hybrid face %" PetscInt_FMT " is %" PetscInt_FMT " != %" PetscInt_FMT, hybface, qf, supportSizeNew);
1795         PetscCall(DMPlexSetSupport(sdm, hybface, supportNew));
1796       }
1797     }
1798   }
1799   /* Step 6b: Replace split points in negative side cones */
1800   for (sp = 0; sp < numSP; ++sp) {
1801     PetscInt        dep = values[sp];
1802     IS              pIS;
1803     PetscInt        numPoints;
1804     const PetscInt *points;
1805 
1806     if (dep >= 0) continue;
1807     PetscCall(DMLabelGetStratumIS(label, dep, &pIS));
1808     if (!pIS) continue;
1809     dep = -dep - shift;
1810     PetscCall(ISGetLocalSize(pIS, &numPoints));
1811     PetscCall(ISGetIndices(pIS, &points));
1812     for (p = 0; p < numPoints; ++p) {
1813       const PetscInt  oldp = points[p];
1814       const PetscInt  newp = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*depthOffset[dep] + oldp*/;
1815       const PetscInt *cone;
1816       PetscInt        coneSize, c;
1817       /* PetscBool       replaced = PETSC_FALSE; */
1818 
1819       /* Negative edge: replace split vertex */
1820       /* Negative cell: replace split face */
1821       PetscCall(DMPlexGetConeSize(sdm, newp, &coneSize));
1822       PetscCall(DMPlexGetCone(sdm, newp, &cone));
1823       for (c = 0; c < coneSize; ++c) {
1824         const PetscInt coldp = DMPlexShiftPointInverse_Internal(cone[c], depth, depthShift);
1825         PetscInt       csplitp, cp, val;
1826 
1827         PetscCall(DMLabelGetValue(label, coldp, &val));
1828         if (val == dep - 1) {
1829           PetscCall(PetscFindInt(coldp, numSplitPoints[dep - 1], splitPoints[dep - 1], &cp));
1830           PetscCheck(cp >= 0, comm, PETSC_ERR_ARG_WRONG, "Point %" PetscInt_FMT " is not a split point of dimension %" PetscInt_FMT, oldp, dep - 1);
1831           csplitp = pMaxNew[dep - 1] + cp;
1832           PetscCall(DMPlexInsertCone(sdm, newp, c, csplitp));
1833           /* replaced = PETSC_TRUE; */
1834         }
1835       }
1836       /* Cells with only a vertex or edge on the submesh have no replacement */
1837       /* PetscCheck(replaced,comm, PETSC_ERR_ARG_WRONG, "The cone of point %d does not contain split points", oldp); */
1838     }
1839     PetscCall(ISRestoreIndices(pIS, &points));
1840     PetscCall(ISDestroy(&pIS));
1841   }
1842   PetscCall(DMPlexReorderCohesiveSupports(sdm));
1843   /* Step 7: Coordinates */
1844   PetscCall(DMPlexShiftCoordinates_Internal(dm, depthShift, sdm));
1845   PetscCall(DMGetCoordinateSection(sdm, &coordSection));
1846   PetscCall(DMGetCoordinatesLocal(sdm, &coordinates));
1847   PetscCall(VecGetArray(coordinates, &coords));
1848   for (v = 0; v < (numSplitPoints ? numSplitPoints[0] : 0); ++v) {
1849     const PetscInt newp   = DMPlexShiftPoint_Internal(splitPoints[0][v], depth, depthShift) /*depthOffset[0] + splitPoints[0][v]*/;
1850     const PetscInt splitp = pMaxNew[0] + v;
1851     PetscInt       dof, off, soff, d;
1852 
1853     PetscCall(PetscSectionGetDof(coordSection, newp, &dof));
1854     PetscCall(PetscSectionGetOffset(coordSection, newp, &off));
1855     PetscCall(PetscSectionGetOffset(coordSection, splitp, &soff));
1856     for (d = 0; d < dof; ++d) coords[soff + d] = coords[off + d];
1857   }
1858   PetscCall(VecRestoreArray(coordinates, &coords));
1859   /* Step 8: SF, if I can figure this out we can split the mesh in parallel */
1860   PetscCall(DMPlexShiftSF_Internal(dm, depthShift, sdm));
1861   /*   TODO We need to associate the ghost points with the correct replica */
1862   /* Step 9: Labels */
1863   PetscCall(DMPlexShiftLabels_Internal(dm, depthShift, sdm));
1864   PetscCall(DMPlexCreateVTKLabel_Internal(dm, PETSC_FALSE, sdm));
1865   PetscCall(DMGetNumLabels(sdm, &numLabels));
1866   for (dep = 0; dep <= depth; ++dep) {
1867     for (p = 0; p < numSplitPoints[dep]; ++p) {
1868       const PetscInt newp   = DMPlexShiftPoint_Internal(splitPoints[dep][p], depth, depthShift) /*depthOffset[dep] + splitPoints[dep][p]*/;
1869       const PetscInt splitp = pMaxNew[dep] + p;
1870       PetscInt       l;
1871 
1872       if (splitLabel) {
1873         const PetscInt val = 100 + dep;
1874 
1875         PetscCall(DMLabelSetValue(splitLabel, newp, val));
1876         PetscCall(DMLabelSetValue(splitLabel, splitp, -val));
1877       }
1878       for (l = 0; l < numLabels; ++l) {
1879         DMLabel     mlabel;
1880         const char *lname;
1881         PetscInt    val;
1882         PetscBool   isDepth;
1883 
1884         PetscCall(DMGetLabelName(sdm, l, &lname));
1885         PetscCall(PetscStrcmp(lname, "depth", &isDepth));
1886         if (isDepth) continue;
1887         PetscCall(DMGetLabel(sdm, lname, &mlabel));
1888         PetscCall(DMLabelGetValue(mlabel, newp, &val));
1889         if (val >= 0) PetscCall(DMLabelSetValue(mlabel, splitp, val));
1890       }
1891     }
1892   }
1893   for (sp = 0; sp < numSP; ++sp) {
1894     const PetscInt dep = values[sp];
1895 
1896     if ((dep < 0) || (dep > depth)) continue;
1897     if (splitIS[dep]) PetscCall(ISRestoreIndices(splitIS[dep], &splitPoints[dep]));
1898     PetscCall(ISDestroy(&splitIS[dep]));
1899     if (unsplitIS[dep]) PetscCall(ISRestoreIndices(unsplitIS[dep], &unsplitPoints[dep]));
1900     PetscCall(ISDestroy(&unsplitIS[dep]));
1901   }
1902   if (ghostIS) PetscCall(ISRestoreIndices(ghostIS, &ghostPoints));
1903   PetscCall(ISDestroy(&ghostIS));
1904   if (label) {
1905     PetscCall(ISRestoreIndices(valueIS, &values));
1906     PetscCall(ISDestroy(&valueIS));
1907   }
1908   for (d = 0; d <= depth; ++d) {
1909     PetscCall(DMPlexGetDepthStratum(sdm, d, NULL, &pEnd));
1910     pMaxNew[d] = pEnd - numHybridPoints[d] - numHybridPointsOld[d];
1911   }
1912   PetscCall(PetscFree3(coneNew, coneONew, supportNew));
1913   PetscCall(PetscFree5(depthMax, depthEnd, depthShift, pMaxNew, numHybridPointsOld));
1914   PetscCall(PetscFree7(splitIS, unsplitIS, numSplitPoints, numUnsplitPoints, numHybridPoints, splitPoints, unsplitPoints));
1915   PetscFunctionReturn(PETSC_SUCCESS);
1916 }
1917 
1918 /*@C
1919   DMPlexConstructCohesiveCells - Construct cohesive cells which split the face along an internal interface
1920 
1921   Collective
1922 
1923   Input Parameters:
1924 + dm    - The original `DM`
1925 - label - The `DMLabel` specifying the boundary faces (this could be auto-generated)
1926 
1927   Output Parameters:
1928 + splitLabel - The `DMLabel` containing the split points, or `NULL` if no output is desired
1929 - dmSplit    - The new `DM`
1930 
1931   Level: developer
1932 
1933 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMCreate()`, `DMPlexLabelCohesiveComplete()`
1934 @*/
1935 PetscErrorCode DMPlexConstructCohesiveCells(DM dm, DMLabel label, DMLabel splitLabel, DM *dmSplit)
1936 {
1937   DM       sdm;
1938   PetscInt dim;
1939 
1940   PetscFunctionBegin;
1941   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1942   PetscAssertPointer(dmSplit, 4);
1943   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &sdm));
1944   PetscCall(DMSetType(sdm, DMPLEX));
1945   PetscCall(DMGetDimension(dm, &dim));
1946   PetscCall(DMSetDimension(sdm, dim));
1947   switch (dim) {
1948   case 2:
1949   case 3:
1950     PetscCall(DMPlexConstructCohesiveCells_Internal(dm, label, splitLabel, sdm));
1951     break;
1952   default:
1953     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cannot construct cohesive cells for dimension %" PetscInt_FMT, dim);
1954   }
1955   *dmSplit = sdm;
1956   PetscFunctionReturn(PETSC_SUCCESS);
1957 }
1958 
1959 /* Returns the side of the surface for a given cell with a face on the surface */
1960 static PetscErrorCode GetSurfaceSide_Static(DM dm, DM subdm, PetscInt numSubpoints, const PetscInt *subpoints, PetscInt cell, PetscInt face, PetscBool *pos)
1961 {
1962   const PetscInt *cone, *ornt;
1963   PetscInt        dim, coneSize, c;
1964 
1965   PetscFunctionBegin;
1966   *pos = PETSC_TRUE;
1967   PetscCall(DMGetDimension(dm, &dim));
1968   PetscCall(DMPlexGetConeSize(dm, cell, &coneSize));
1969   PetscCall(DMPlexGetCone(dm, cell, &cone));
1970   PetscCall(DMPlexGetConeOrientation(dm, cell, &ornt));
1971   for (c = 0; c < coneSize; ++c) {
1972     if (cone[c] == face) {
1973       PetscInt o = ornt[c];
1974 
1975       if (subdm) {
1976         const PetscInt *subcone, *subornt;
1977         PetscInt        subpoint, subface, subconeSize, sc;
1978 
1979         PetscCall(PetscFindInt(cell, numSubpoints, subpoints, &subpoint));
1980         PetscCall(PetscFindInt(face, numSubpoints, subpoints, &subface));
1981         PetscCall(DMPlexGetConeSize(subdm, subpoint, &subconeSize));
1982         PetscCall(DMPlexGetCone(subdm, subpoint, &subcone));
1983         PetscCall(DMPlexGetConeOrientation(subdm, subpoint, &subornt));
1984         for (sc = 0; sc < subconeSize; ++sc) {
1985           if (subcone[sc] == subface) {
1986             o = subornt[0];
1987             break;
1988           }
1989         }
1990         PetscCheck(sc < subconeSize, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not find subpoint %" PetscInt_FMT " (%" PetscInt_FMT ") in cone for subpoint %" PetscInt_FMT " (%" PetscInt_FMT ")", subface, face, subpoint, cell);
1991       }
1992       if (o >= 0) *pos = PETSC_TRUE;
1993       else *pos = PETSC_FALSE;
1994       break;
1995     }
1996   }
1997   PetscCheck(c != coneSize, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Cell %" PetscInt_FMT " in split face %" PetscInt_FMT " support does not have it in the cone", cell, face);
1998   PetscFunctionReturn(PETSC_SUCCESS);
1999 }
2000 
2001 static PetscErrorCode CheckFaultEdge_Private(DM dm, DMLabel label)
2002 {
2003   IS              facePosIS, faceNegIS, dimIS;
2004   const PetscInt *points;
2005   PetscInt        dim, numPoints, p, shift = 100, shift2 = 200;
2006 
2007   PetscFunctionBegin;
2008   PetscCall(DMGetDimension(dm, &dim));
2009   /* If any faces touching the fault divide cells on either side, split them */
2010   PetscCall(DMLabelGetStratumIS(label, shift + dim - 1, &facePosIS));
2011   PetscCall(DMLabelGetStratumIS(label, -(shift + dim - 1), &faceNegIS));
2012   if (!facePosIS || !faceNegIS) {
2013     PetscCall(ISDestroy(&facePosIS));
2014     PetscCall(ISDestroy(&faceNegIS));
2015     PetscFunctionReturn(PETSC_SUCCESS);
2016   }
2017   PetscCall(ISExpand(facePosIS, faceNegIS, &dimIS));
2018   PetscCall(ISDestroy(&facePosIS));
2019   PetscCall(ISDestroy(&faceNegIS));
2020   PetscCall(ISGetLocalSize(dimIS, &numPoints));
2021   PetscCall(ISGetIndices(dimIS, &points));
2022   for (p = 0; p < numPoints; ++p) {
2023     const PetscInt  point = points[p];
2024     const PetscInt *support;
2025     PetscInt        supportSize, valA, valB;
2026 
2027     PetscCall(DMPlexGetSupportSize(dm, point, &supportSize));
2028     if (supportSize != 2) continue;
2029     PetscCall(DMPlexGetSupport(dm, point, &support));
2030     PetscCall(DMLabelGetValue(label, support[0], &valA));
2031     PetscCall(DMLabelGetValue(label, support[1], &valB));
2032     if ((valA == -1) || (valB == -1)) continue;
2033     if (valA * valB > 0) continue;
2034     /* Check that this face is not incident on only unsplit faces, meaning has at least one split face */
2035     {
2036       PetscInt *closure = NULL;
2037       PetscBool split   = PETSC_FALSE;
2038       PetscInt  closureSize, cl;
2039 
2040       PetscCall(DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
2041       for (cl = 0; cl < closureSize * 2; cl += 2) {
2042         PetscCall(DMLabelGetValue(label, closure[cl], &valA));
2043         if ((valA >= 0) && (valA <= dim)) {
2044           split = PETSC_TRUE;
2045           break;
2046         }
2047       }
2048       PetscCall(DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
2049       if (!split) continue;
2050     }
2051     /* Split the face */
2052     PetscCall(DMLabelGetValue(label, point, &valA));
2053     PetscCall(DMLabelClearValue(label, point, valA));
2054     PetscCall(DMLabelSetValue(label, point, dim - 1));
2055     /* Label its closure:
2056       unmarked: label as unsplit
2057       incident: relabel as split
2058       split:    do nothing
2059     */
2060     {
2061       PetscInt *closure = NULL;
2062       PetscInt  closureSize, cl, dep;
2063 
2064       PetscCall(DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
2065       for (cl = 0; cl < closureSize * 2; cl += 2) {
2066         PetscCall(DMLabelGetValue(label, closure[cl], &valA));
2067         if (valA == -1) { /* Mark as unsplit */
2068           PetscCall(DMPlexGetPointDepth(dm, closure[cl], &dep));
2069           PetscCall(DMLabelSetValue(label, closure[cl], shift2 + dep));
2070         } else if (((valA >= shift) && (valA < shift2)) || ((valA <= -shift) && (valA > -shift2))) {
2071           PetscCall(DMPlexGetPointDepth(dm, closure[cl], &dep));
2072           PetscCall(DMLabelClearValue(label, closure[cl], valA));
2073           PetscCall(DMLabelSetValue(label, closure[cl], dep));
2074         }
2075       }
2076       PetscCall(DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
2077     }
2078   }
2079   PetscCall(ISRestoreIndices(dimIS, &points));
2080   PetscCall(ISDestroy(&dimIS));
2081   PetscFunctionReturn(PETSC_SUCCESS);
2082 }
2083 
2084 /*@
2085   DMPlexLabelCohesiveComplete - Starting with a label marking points on an internal surface, we add all other mesh pieces
2086   to complete the surface
2087 
2088   Input Parameters:
2089 + dm     - The `DM`
2090 . label  - A `DMLabel` marking the surface
2091 . blabel - A `DMLabel` marking the vertices on the boundary which will not be duplicated, or `NULL` to find them automatically
2092 . bvalue - Value of `DMLabel` marking the vertices on the boundary
2093 . flip   - Flag to flip the submesh normal and replace points on the other side
2094 - subdm  - The `DM` associated with the label, or `NULL`
2095 
2096   Output Parameter:
2097 . label - A `DMLabel` marking all surface points
2098 
2099   Level: developer
2100 
2101   Note:
2102   The vertices in blabel are called "unsplit" in the terminology from hybrid cell creation.
2103 
2104 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexConstructCohesiveCells()`, `DMPlexLabelComplete()`
2105 @*/
2106 PetscErrorCode DMPlexLabelCohesiveComplete(DM dm, DMLabel label, DMLabel blabel, PetscInt bvalue, PetscBool flip, DM subdm)
2107 {
2108   DMLabel         depthLabel;
2109   IS              dimIS, subpointIS = NULL;
2110   const PetscInt *points, *subpoints;
2111   const PetscInt  rev   = flip ? -1 : 1;
2112   PetscInt        shift = 100, shift2 = 200, shift3 = 300, dim, depth, numPoints, numSubpoints, p, val;
2113 
2114   PetscFunctionBegin;
2115   PetscCall(DMPlexGetDepth(dm, &depth));
2116   PetscCall(DMGetDimension(dm, &dim));
2117   PetscCall(DMPlexGetDepthLabel(dm, &depthLabel));
2118   if (subdm) {
2119     PetscCall(DMPlexGetSubpointIS(subdm, &subpointIS));
2120     if (subpointIS) {
2121       PetscCall(ISGetLocalSize(subpointIS, &numSubpoints));
2122       PetscCall(ISGetIndices(subpointIS, &subpoints));
2123     }
2124   }
2125   /* Mark cell on the fault, and its faces which touch the fault: cell orientation for face gives the side of the fault */
2126   PetscCall(DMLabelGetStratumIS(label, dim - 1, &dimIS));
2127   if (!dimIS) goto divide;
2128   PetscCall(ISGetLocalSize(dimIS, &numPoints));
2129   PetscCall(ISGetIndices(dimIS, &points));
2130   for (p = 0; p < numPoints; ++p) { /* Loop over fault faces */
2131     const PetscInt *support;
2132     PetscInt        supportSize, s;
2133 
2134     PetscCall(DMPlexGetSupportSize(dm, points[p], &supportSize));
2135 #if 0
2136     if (supportSize != 2) {
2137       const PetscInt *lp;
2138       PetscInt        Nlp, pind;
2139 
2140       /* Check that for a cell with a single support face, that face is in the SF */
2141       /*   THis check only works for the remote side. We would need root side information */
2142       PetscCall(PetscSFGetGraph(dm->sf, NULL, &Nlp, &lp, NULL));
2143       PetscCall(PetscFindInt(points[p], Nlp, lp, &pind));
2144       PetscCheck(pind >= 0,PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Split face %" PetscInt_FMT " has %" PetscInt_FMT " != 2 supports, and the face is not shared with another process", points[p], supportSize);
2145     }
2146 #endif
2147     PetscCall(DMPlexGetSupport(dm, points[p], &support));
2148     for (s = 0; s < supportSize; ++s) {
2149       const PetscInt *cone;
2150       PetscInt        coneSize, c;
2151       PetscBool       pos;
2152 
2153       PetscCall(GetSurfaceSide_Static(dm, subdm, numSubpoints, subpoints, support[s], points[p], &pos));
2154       if (pos) PetscCall(DMLabelSetValue(label, support[s], rev * (shift + dim)));
2155       else PetscCall(DMLabelSetValue(label, support[s], -rev * (shift + dim)));
2156       if (rev < 0) pos = !pos ? PETSC_TRUE : PETSC_FALSE;
2157       /* Put faces touching the fault in the label */
2158       PetscCall(DMPlexGetConeSize(dm, support[s], &coneSize));
2159       PetscCall(DMPlexGetCone(dm, support[s], &cone));
2160       for (c = 0; c < coneSize; ++c) {
2161         const PetscInt point = cone[c];
2162 
2163         PetscCall(DMLabelGetValue(label, point, &val));
2164         if (val == -1) {
2165           PetscInt *closure = NULL;
2166           PetscInt  closureSize, cl;
2167 
2168           PetscCall(DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
2169           for (cl = 0; cl < closureSize * 2; cl += 2) {
2170             const PetscInt clp  = closure[cl];
2171             PetscInt       bval = -1;
2172 
2173             PetscCall(DMLabelGetValue(label, clp, &val));
2174             if (blabel) PetscCall(DMLabelGetValue(blabel, clp, &bval));
2175             if ((val >= 0) && (val < dim - 1) && (bval < 0)) {
2176               PetscCall(DMLabelSetValue(label, point, pos == PETSC_TRUE ? shift + dim - 1 : -(shift + dim - 1)));
2177               break;
2178             }
2179           }
2180           PetscCall(DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
2181         }
2182       }
2183     }
2184   }
2185   PetscCall(ISRestoreIndices(dimIS, &points));
2186   PetscCall(ISDestroy(&dimIS));
2187   /* Mark boundary points as unsplit */
2188   if (blabel) {
2189     IS bdIS;
2190 
2191     PetscCall(DMLabelGetStratumIS(blabel, bvalue, &bdIS));
2192     PetscCall(ISGetLocalSize(bdIS, &numPoints));
2193     PetscCall(ISGetIndices(bdIS, &points));
2194     for (p = 0; p < numPoints; ++p) {
2195       const PetscInt point = points[p];
2196       PetscInt       val, bval;
2197 
2198       PetscCall(DMLabelGetValue(blabel, point, &bval));
2199       if (bval >= 0) {
2200         PetscCall(DMLabelGetValue(label, point, &val));
2201         if ((val < 0) || (val > dim)) {
2202           /* This could be a point added from splitting a vertex on an adjacent fault, otherwise its just wrong */
2203           PetscCall(DMLabelClearValue(blabel, point, bval));
2204         }
2205       }
2206     }
2207     for (p = 0; p < numPoints; ++p) {
2208       const PetscInt point = points[p];
2209       PetscInt       val, bval;
2210 
2211       PetscCall(DMLabelGetValue(blabel, point, &bval));
2212       if (bval >= 0) {
2213         const PetscInt *cone, *support;
2214         PetscInt        coneSize, supportSize, s, valA, valB, valE;
2215 
2216         /* Mark as unsplit */
2217         PetscCall(DMLabelGetValue(label, point, &val));
2218         PetscCheck(val >= 0 && val <= dim, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %" PetscInt_FMT " has label value %" PetscInt_FMT ", should be part of the fault", point, val);
2219         PetscCall(DMLabelClearValue(label, point, val));
2220         PetscCall(DMLabelSetValue(label, point, shift2 + val));
2221         /* Check for cross-edge
2222              A cross-edge has endpoints which are both on the boundary of the surface, but the edge itself is not. */
2223         if (val != 0) continue;
2224         PetscCall(DMPlexGetSupport(dm, point, &support));
2225         PetscCall(DMPlexGetSupportSize(dm, point, &supportSize));
2226         for (s = 0; s < supportSize; ++s) {
2227           PetscCall(DMPlexGetCone(dm, support[s], &cone));
2228           PetscCall(DMPlexGetConeSize(dm, support[s], &coneSize));
2229           PetscCheck(coneSize == 2, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Edge %" PetscInt_FMT " has %" PetscInt_FMT " vertices != 2", support[s], coneSize);
2230           PetscCall(DMLabelGetValue(blabel, cone[0], &valA));
2231           PetscCall(DMLabelGetValue(blabel, cone[1], &valB));
2232           PetscCall(DMLabelGetValue(blabel, support[s], &valE));
2233           if ((valE < 0) && (valA >= 0) && (valB >= 0) && (cone[0] != cone[1])) PetscCall(DMLabelSetValue(blabel, support[s], 2));
2234         }
2235       }
2236     }
2237     PetscCall(ISRestoreIndices(bdIS, &points));
2238     PetscCall(ISDestroy(&bdIS));
2239   }
2240   /* Mark ghost fault cells */
2241   {
2242     PetscSF         sf;
2243     const PetscInt *leaves;
2244     PetscInt        Nl, l;
2245 
2246     PetscCall(DMGetPointSF(dm, &sf));
2247     PetscCall(PetscSFGetGraph(sf, NULL, &Nl, &leaves, NULL));
2248     PetscCall(DMLabelGetStratumIS(label, dim - 1, &dimIS));
2249     if (!dimIS) goto divide;
2250     PetscCall(ISGetLocalSize(dimIS, &numPoints));
2251     PetscCall(ISGetIndices(dimIS, &points));
2252     if (Nl > 0) {
2253       for (p = 0; p < numPoints; ++p) {
2254         const PetscInt point = points[p];
2255         PetscInt       val;
2256 
2257         PetscCall(PetscFindInt(point, Nl, leaves, &l));
2258         if (l >= 0) {
2259           PetscInt *closure = NULL;
2260           PetscInt  closureSize, cl;
2261 
2262           PetscCall(DMLabelGetValue(label, point, &val));
2263           PetscCheck((val == dim - 1) || (val == shift2 + dim - 1), PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %" PetscInt_FMT " has label value %" PetscInt_FMT ", should be a fault face", point, val);
2264           PetscCall(DMLabelClearValue(label, point, val));
2265           PetscCall(DMLabelSetValue(label, point, shift3 + val));
2266           PetscCall(DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
2267           for (cl = 2; cl < closureSize * 2; cl += 2) {
2268             const PetscInt clp = closure[cl];
2269 
2270             PetscCall(DMLabelGetValue(label, clp, &val));
2271             PetscCheck(val != -1, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %" PetscInt_FMT " is missing from label, but is in the closure of a fault face", point);
2272             PetscCall(DMLabelClearValue(label, clp, val));
2273             PetscCall(DMLabelSetValue(label, clp, shift3 + val));
2274           }
2275           PetscCall(DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
2276         }
2277       }
2278     }
2279     PetscCall(ISRestoreIndices(dimIS, &points));
2280     PetscCall(ISDestroy(&dimIS));
2281   }
2282 divide:
2283   if (subpointIS) PetscCall(ISRestoreIndices(subpointIS, &subpoints));
2284   PetscCall(DMPlexLabelFaultHalo(dm, label));
2285   PetscCall(CheckFaultEdge_Private(dm, label));
2286   PetscFunctionReturn(PETSC_SUCCESS);
2287 }
2288 
2289 /* Check that no cell have all vertices on the fault */
2290 static PetscErrorCode DMPlexCheckValidSubmesh_Private(DM dm, DMLabel label, DM subdm)
2291 {
2292   IS              subpointIS;
2293   const PetscInt *dmpoints;
2294   PetscInt        defaultValue, cStart, cEnd, c, vStart, vEnd;
2295 
2296   PetscFunctionBegin;
2297   if (!label) PetscFunctionReturn(PETSC_SUCCESS);
2298   PetscCall(DMLabelGetDefaultValue(label, &defaultValue));
2299   PetscCall(DMPlexGetSubpointIS(subdm, &subpointIS));
2300   if (!subpointIS) PetscFunctionReturn(PETSC_SUCCESS);
2301   PetscCall(DMPlexGetHeightStratum(subdm, 0, &cStart, &cEnd));
2302   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
2303   PetscCall(ISGetIndices(subpointIS, &dmpoints));
2304   for (c = cStart; c < cEnd; ++c) {
2305     PetscBool invalidCell = PETSC_TRUE;
2306     PetscInt *closure     = NULL;
2307     PetscInt  closureSize, cl;
2308 
2309     PetscCall(DMPlexGetTransitiveClosure(dm, dmpoints[c], PETSC_TRUE, &closureSize, &closure));
2310     for (cl = 0; cl < closureSize * 2; cl += 2) {
2311       PetscInt value = 0;
2312 
2313       if ((closure[cl] < vStart) || (closure[cl] >= vEnd)) continue;
2314       PetscCall(DMLabelGetValue(label, closure[cl], &value));
2315       if (value == defaultValue) {
2316         invalidCell = PETSC_FALSE;
2317         break;
2318       }
2319     }
2320     PetscCall(DMPlexRestoreTransitiveClosure(dm, dmpoints[c], PETSC_TRUE, &closureSize, &closure));
2321     if (invalidCell) {
2322       PetscCall(ISRestoreIndices(subpointIS, &dmpoints));
2323       PetscCall(ISDestroy(&subpointIS));
2324       PetscCall(DMDestroy(&subdm));
2325       SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Ambiguous submesh. Cell %" PetscInt_FMT " has all of its vertices on the submesh.", dmpoints[c]);
2326     }
2327   }
2328   PetscCall(ISRestoreIndices(subpointIS, &dmpoints));
2329   PetscFunctionReturn(PETSC_SUCCESS);
2330 }
2331 
2332 /*@
2333   DMPlexCreateHybridMesh - Create a mesh with hybrid cells along an internal interface
2334 
2335   Collective
2336 
2337   Input Parameters:
2338 + dm      - The original `DM`
2339 . label   - The label specifying the interface vertices
2340 . bdlabel - The optional label specifying the interface boundary vertices
2341 - bdvalue - Value of optional label specifying the interface boundary vertices
2342 
2343   Output Parameters:
2344 + hybridLabel - The label fully marking the interface, or `NULL` if no output is desired
2345 . splitLabel  - The label containing the split points, or `NULL` if no output is desired
2346 . dmInterface - The new interface `DM`, or `NULL`
2347 - dmHybrid    - The new `DM` with cohesive cells
2348 
2349   Level: developer
2350 
2351   Note:
2352   The hybridLabel indicates what parts of the original mesh impinged on the division surface. For points
2353   directly on the division surface, they are labeled with their dimension, so an edge 7 on the division surface would be
2354   7 (1) in hybridLabel. For points that impinge from the positive side, they are labeled with 100+dim, so an edge 6 with
2355   one vertex 3 on the surface would be 6 (101) and 3 (0) in hybridLabel. If an edge 9 from the negative side of the
2356   surface also hits vertex 3, it would be 9 (-101) in hybridLabel.
2357 
2358   The splitLabel indicates what points in the new hybrid mesh were the result of splitting points in the original
2359   mesh. The label value is $\pm 100+dim$ for each point. For example, if two edges 10 and 14 in the hybrid resulting from
2360   splitting an edge in the original mesh, you would have 10 (101) and 14 (-101) in the splitLabel.
2361 
2362   The dmInterface is a `DM` built from the original division surface. It has a label which can be retrieved using
2363   `DMPlexGetSubpointMap()` which maps each point back to the point in the surface of the original mesh.
2364 
2365 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexConstructCohesiveCells()`, `DMPlexLabelCohesiveComplete()`, `DMPlexGetSubpointMap()`, `DMCreate()`
2366 @*/
2367 PetscErrorCode DMPlexCreateHybridMesh(DM dm, DMLabel label, DMLabel bdlabel, PetscInt bdvalue, DMLabel *hybridLabel, DMLabel *splitLabel, DM *dmInterface, DM *dmHybrid)
2368 {
2369   DM       idm;
2370   DMLabel  subpointMap, hlabel, slabel = NULL;
2371   PetscInt dim;
2372 
2373   PetscFunctionBegin;
2374   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2375   if (label) PetscAssertPointer(label, 2);
2376   if (bdlabel) PetscAssertPointer(bdlabel, 3);
2377   if (hybridLabel) PetscAssertPointer(hybridLabel, 5);
2378   if (splitLabel) PetscAssertPointer(splitLabel, 6);
2379   if (dmInterface) PetscAssertPointer(dmInterface, 7);
2380   PetscAssertPointer(dmHybrid, 8);
2381   PetscCall(DMGetDimension(dm, &dim));
2382   PetscCall(DMPlexCreateSubmesh(dm, label, 1, PETSC_FALSE, &idm));
2383   PetscCall(DMPlexCheckValidSubmesh_Private(dm, label, idm));
2384   PetscCall(DMPlexOrient(idm));
2385   PetscCall(DMPlexGetSubpointMap(idm, &subpointMap));
2386   PetscCall(DMLabelDuplicate(subpointMap, &hlabel));
2387   PetscCall(DMLabelClearStratum(hlabel, dim));
2388   if (splitLabel) {
2389     const char *name;
2390     char        sname[PETSC_MAX_PATH_LEN];
2391 
2392     PetscCall(PetscObjectGetName((PetscObject)hlabel, &name));
2393     PetscCall(PetscStrncpy(sname, name, sizeof(sname)));
2394     PetscCall(PetscStrlcat(sname, " split", sizeof(sname)));
2395     PetscCall(DMLabelCreate(PETSC_COMM_SELF, sname, &slabel));
2396   }
2397   PetscCall(DMPlexLabelCohesiveComplete(dm, hlabel, bdlabel, bdvalue, PETSC_FALSE, idm));
2398   if (dmInterface) {
2399     *dmInterface = idm;
2400   } else PetscCall(DMDestroy(&idm));
2401   PetscCall(DMPlexConstructCohesiveCells(dm, hlabel, slabel, dmHybrid));
2402   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_TRUE, *dmHybrid));
2403   if (hybridLabel) *hybridLabel = hlabel;
2404   else PetscCall(DMLabelDestroy(&hlabel));
2405   if (splitLabel) *splitLabel = slabel;
2406   {
2407     DM      cdm;
2408     DMLabel ctLabel;
2409 
2410     /* We need to somehow share the celltype label with the coordinate dm */
2411     PetscCall(DMGetCoordinateDM(*dmHybrid, &cdm));
2412     PetscCall(DMPlexGetCellTypeLabel(*dmHybrid, &ctLabel));
2413     PetscCall(DMSetLabel(cdm, ctLabel));
2414   }
2415   PetscFunctionReturn(PETSC_SUCCESS);
2416 }
2417 
2418 /* Here we need the explicit assumption that:
2419 
2420      For any marked cell, the marked vertices constitute a single face
2421 */
2422 static PetscErrorCode DMPlexMarkSubmesh_Uninterpolated(DM dm, DMLabel vertexLabel, PetscInt value, DMLabel subpointMap, PetscInt *numFaces, PetscInt *nFV, DM subdm)
2423 {
2424   IS              subvertexIS = NULL;
2425   const PetscInt *subvertices;
2426   PetscInt       *pStart, *pEnd, pSize;
2427   PetscInt        depth, dim, d, numSubVerticesInitial = 0, v;
2428 
2429   PetscFunctionBegin;
2430   *numFaces = 0;
2431   *nFV      = 0;
2432   PetscCall(DMPlexGetDepth(dm, &depth));
2433   PetscCall(DMGetDimension(dm, &dim));
2434   pSize = PetscMax(depth, dim) + 1;
2435   PetscCall(PetscMalloc2(pSize, &pStart, pSize, &pEnd));
2436   for (d = 0; d <= depth; ++d) PetscCall(DMPlexGetSimplexOrBoxCells(dm, depth - d, &pStart[d], &pEnd[d]));
2437   /* Loop over initial vertices and mark all faces in the collective star() */
2438   if (vertexLabel) PetscCall(DMLabelGetStratumIS(vertexLabel, value, &subvertexIS));
2439   if (subvertexIS) {
2440     PetscCall(ISGetSize(subvertexIS, &numSubVerticesInitial));
2441     PetscCall(ISGetIndices(subvertexIS, &subvertices));
2442   }
2443   for (v = 0; v < numSubVerticesInitial; ++v) {
2444     const PetscInt vertex = subvertices[v];
2445     PetscInt      *star   = NULL;
2446     PetscInt       starSize, s, numCells = 0, c;
2447 
2448     PetscCall(DMPlexGetTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star));
2449     for (s = 0; s < starSize * 2; s += 2) {
2450       const PetscInt point = star[s];
2451       if ((point >= pStart[depth]) && (point < pEnd[depth])) star[numCells++] = point;
2452     }
2453     for (c = 0; c < numCells; ++c) {
2454       const PetscInt cell    = star[c];
2455       PetscInt      *closure = NULL;
2456       PetscInt       closureSize, cl;
2457       PetscInt       cellLoc, numCorners = 0, faceSize = 0;
2458 
2459       PetscCall(DMLabelGetValue(subpointMap, cell, &cellLoc));
2460       if (cellLoc == 2) continue;
2461       PetscCheck(cellLoc < 0, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Cell %" PetscInt_FMT " has dimension %" PetscInt_FMT " in the surface label", cell, cellLoc);
2462       PetscCall(DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
2463       for (cl = 0; cl < closureSize * 2; cl += 2) {
2464         const PetscInt point = closure[cl];
2465         PetscInt       vertexLoc;
2466 
2467         if ((point >= pStart[0]) && (point < pEnd[0])) {
2468           ++numCorners;
2469           PetscCall(DMLabelGetValue(vertexLabel, point, &vertexLoc));
2470           if (vertexLoc == value) closure[faceSize++] = point;
2471         }
2472       }
2473       if (!(*nFV)) PetscCall(DMPlexGetNumFaceVertices(dm, dim, numCorners, nFV));
2474       PetscCheck(faceSize <= *nFV, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Invalid submesh: Too many vertices %" PetscInt_FMT " of an element on the surface", faceSize);
2475       if (faceSize == *nFV) {
2476         const PetscInt *cells = NULL;
2477         PetscInt        numCells, nc;
2478 
2479         ++(*numFaces);
2480         for (cl = 0; cl < faceSize; ++cl) PetscCall(DMLabelSetValue(subpointMap, closure[cl], 0));
2481         PetscCall(DMPlexGetJoin(dm, faceSize, closure, &numCells, &cells));
2482         for (nc = 0; nc < numCells; ++nc) PetscCall(DMLabelSetValue(subpointMap, cells[nc], 2));
2483         PetscCall(DMPlexRestoreJoin(dm, faceSize, closure, &numCells, &cells));
2484       }
2485       PetscCall(DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
2486     }
2487     PetscCall(DMPlexRestoreTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star));
2488   }
2489   if (subvertexIS) PetscCall(ISRestoreIndices(subvertexIS, &subvertices));
2490   PetscCall(ISDestroy(&subvertexIS));
2491   PetscCall(PetscFree2(pStart, pEnd));
2492   PetscFunctionReturn(PETSC_SUCCESS);
2493 }
2494 
2495 static PetscErrorCode DMPlexMarkSubmesh_Interpolated(DM dm, DMLabel vertexLabel, PetscInt value, PetscBool markedFaces, DMLabel subpointMap, DM subdm)
2496 {
2497   IS              subvertexIS = NULL;
2498   const PetscInt *subvertices;
2499   PetscInt       *pStart, *pEnd;
2500   PetscInt        dim, d, numSubVerticesInitial = 0, v;
2501 
2502   PetscFunctionBegin;
2503   PetscCall(DMGetDimension(dm, &dim));
2504   PetscCall(PetscMalloc2(dim + 1, &pStart, dim + 1, &pEnd));
2505   for (d = 0; d <= dim; ++d) PetscCall(DMPlexGetSimplexOrBoxCells(dm, dim - d, &pStart[d], &pEnd[d]));
2506   /* Loop over initial vertices and mark all faces in the collective star() */
2507   if (vertexLabel) {
2508     PetscCall(DMLabelGetStratumIS(vertexLabel, value, &subvertexIS));
2509     if (subvertexIS) {
2510       PetscCall(ISGetSize(subvertexIS, &numSubVerticesInitial));
2511       PetscCall(ISGetIndices(subvertexIS, &subvertices));
2512     }
2513   }
2514   for (v = 0; v < numSubVerticesInitial; ++v) {
2515     const PetscInt vertex = subvertices[v];
2516     PetscInt      *star   = NULL;
2517     PetscInt       starSize, s, numFaces = 0, f;
2518 
2519     PetscCall(DMPlexGetTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star));
2520     for (s = 0; s < starSize * 2; s += 2) {
2521       const PetscInt point = star[s];
2522       PetscInt       faceLoc;
2523 
2524       if ((point >= pStart[dim - 1]) && (point < pEnd[dim - 1])) {
2525         if (markedFaces) {
2526           PetscCall(DMLabelGetValue(vertexLabel, point, &faceLoc));
2527           if (faceLoc < 0) continue;
2528         }
2529         star[numFaces++] = point;
2530       }
2531     }
2532     for (f = 0; f < numFaces; ++f) {
2533       const PetscInt face    = star[f];
2534       PetscInt      *closure = NULL;
2535       PetscInt       closureSize, c;
2536       PetscInt       faceLoc;
2537 
2538       PetscCall(DMLabelGetValue(subpointMap, face, &faceLoc));
2539       if (faceLoc == dim - 1) continue;
2540       PetscCheck(faceLoc < 0, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Face %" PetscInt_FMT " has dimension %" PetscInt_FMT " in the surface label", face, faceLoc);
2541       PetscCall(DMPlexGetTransitiveClosure(dm, face, PETSC_TRUE, &closureSize, &closure));
2542       for (c = 0; c < closureSize * 2; c += 2) {
2543         const PetscInt point = closure[c];
2544         PetscInt       vertexLoc;
2545 
2546         if ((point >= pStart[0]) && (point < pEnd[0])) {
2547           PetscCall(DMLabelGetValue(vertexLabel, point, &vertexLoc));
2548           if (vertexLoc != value) break;
2549         }
2550       }
2551       if (c == closureSize * 2) {
2552         const PetscInt *support;
2553         PetscInt        supportSize, s;
2554 
2555         for (c = 0; c < closureSize * 2; c += 2) {
2556           const PetscInt point = closure[c];
2557 
2558           for (d = 0; d < dim; ++d) {
2559             if ((point >= pStart[d]) && (point < pEnd[d])) {
2560               PetscCall(DMLabelSetValue(subpointMap, point, d));
2561               break;
2562             }
2563           }
2564         }
2565         PetscCall(DMPlexGetSupportSize(dm, face, &supportSize));
2566         PetscCall(DMPlexGetSupport(dm, face, &support));
2567         for (s = 0; s < supportSize; ++s) PetscCall(DMLabelSetValue(subpointMap, support[s], dim));
2568       }
2569       PetscCall(DMPlexRestoreTransitiveClosure(dm, face, PETSC_TRUE, &closureSize, &closure));
2570     }
2571     PetscCall(DMPlexRestoreTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star));
2572   }
2573   if (subvertexIS) PetscCall(ISRestoreIndices(subvertexIS, &subvertices));
2574   PetscCall(ISDestroy(&subvertexIS));
2575   PetscCall(PetscFree2(pStart, pEnd));
2576   PetscFunctionReturn(PETSC_SUCCESS);
2577 }
2578 
2579 static PetscErrorCode DMPlexMarkCohesiveSubmesh_Uninterpolated(DM dm, PetscBool hasLagrange, const char labelname[], PetscInt value, DMLabel subpointMap, PetscInt *numFaces, PetscInt *nFV, PetscInt *subCells[], DM subdm)
2580 {
2581   DMLabel         label = NULL;
2582   const PetscInt *cone;
2583   PetscInt        dim, cMax, cEnd, c, subc = 0, p, coneSize = -1;
2584 
2585   PetscFunctionBegin;
2586   *numFaces = 0;
2587   *nFV      = 0;
2588   if (labelname) PetscCall(DMGetLabel(dm, labelname, &label));
2589   *subCells = NULL;
2590   PetscCall(DMGetDimension(dm, &dim));
2591   PetscCall(DMPlexGetTensorPrismBounds_Internal(dm, dim, &cMax, &cEnd));
2592   if (cMax < 0) PetscFunctionReturn(PETSC_SUCCESS);
2593   if (label) {
2594     for (c = cMax; c < cEnd; ++c) {
2595       PetscInt val;
2596 
2597       PetscCall(DMLabelGetValue(label, c, &val));
2598       if (val == value) {
2599         ++(*numFaces);
2600         PetscCall(DMPlexGetConeSize(dm, c, &coneSize));
2601       }
2602     }
2603   } else {
2604     *numFaces = cEnd - cMax;
2605     PetscCall(DMPlexGetConeSize(dm, cMax, &coneSize));
2606   }
2607   PetscCall(PetscMalloc1(*numFaces * 2, subCells));
2608   if (!(*numFaces)) PetscFunctionReturn(PETSC_SUCCESS);
2609   *nFV = hasLagrange ? coneSize / 3 : coneSize / 2;
2610   for (c = cMax; c < cEnd; ++c) {
2611     const PetscInt *cells;
2612     PetscInt        numCells;
2613 
2614     if (label) {
2615       PetscInt val;
2616 
2617       PetscCall(DMLabelGetValue(label, c, &val));
2618       if (val != value) continue;
2619     }
2620     PetscCall(DMPlexGetCone(dm, c, &cone));
2621     for (p = 0; p < *nFV; ++p) PetscCall(DMLabelSetValue(subpointMap, cone[p], 0));
2622     /* Negative face */
2623     PetscCall(DMPlexGetJoin(dm, *nFV, cone, &numCells, &cells));
2624     /* Not true in parallel
2625     PetscCheck(numCells == 2,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive cells should separate two cells"); */
2626     for (p = 0; p < numCells; ++p) {
2627       PetscCall(DMLabelSetValue(subpointMap, cells[p], 2));
2628       (*subCells)[subc++] = cells[p];
2629     }
2630     PetscCall(DMPlexRestoreJoin(dm, *nFV, cone, &numCells, &cells));
2631     /* Positive face is not included */
2632   }
2633   PetscFunctionReturn(PETSC_SUCCESS);
2634 }
2635 
2636 static PetscErrorCode DMPlexMarkCohesiveSubmesh_Interpolated(DM dm, DMLabel label, PetscInt value, DMLabel subpointMap, DM subdm)
2637 {
2638   PetscInt *pStart, *pEnd;
2639   PetscInt  dim, cMax, cEnd, c, d;
2640 
2641   PetscFunctionBegin;
2642   PetscCall(DMGetDimension(dm, &dim));
2643   PetscCall(DMPlexGetTensorPrismBounds_Internal(dm, dim, &cMax, &cEnd));
2644   if (cMax < 0) PetscFunctionReturn(PETSC_SUCCESS);
2645   PetscCall(PetscMalloc2(dim + 1, &pStart, dim + 1, &pEnd));
2646   for (d = 0; d <= dim; ++d) PetscCall(DMPlexGetDepthStratum(dm, d, &pStart[d], &pEnd[d]));
2647   for (c = cMax; c < cEnd; ++c) {
2648     const PetscInt *cone;
2649     PetscInt       *closure = NULL;
2650     PetscInt        fconeSize, coneSize, closureSize, cl, val;
2651 
2652     if (label) {
2653       PetscCall(DMLabelGetValue(label, c, &val));
2654       if (val != value) continue;
2655     }
2656     PetscCall(DMPlexGetConeSize(dm, c, &coneSize));
2657     PetscCall(DMPlexGetCone(dm, c, &cone));
2658     PetscCall(DMPlexGetConeSize(dm, cone[0], &fconeSize));
2659     PetscCheck(coneSize == (fconeSize ? fconeSize : 1) + 2, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive cells should separate two cells");
2660     /* Negative face */
2661     PetscCall(DMPlexGetTransitiveClosure(dm, cone[0], PETSC_TRUE, &closureSize, &closure));
2662     for (cl = 0; cl < closureSize * 2; cl += 2) {
2663       const PetscInt point = closure[cl];
2664 
2665       for (d = 0; d <= dim; ++d) {
2666         if ((point >= pStart[d]) && (point < pEnd[d])) {
2667           PetscCall(DMLabelSetValue(subpointMap, point, d));
2668           break;
2669         }
2670       }
2671     }
2672     PetscCall(DMPlexRestoreTransitiveClosure(dm, cone[0], PETSC_TRUE, &closureSize, &closure));
2673     /* Cells -- positive face is not included */
2674     for (cl = 0; cl < 1; ++cl) {
2675       const PetscInt *support;
2676       PetscInt        supportSize, s;
2677 
2678       PetscCall(DMPlexGetSupportSize(dm, cone[cl], &supportSize));
2679       /* PetscCheck(supportSize == 2,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive faces should separate two cells"); */
2680       PetscCall(DMPlexGetSupport(dm, cone[cl], &support));
2681       for (s = 0; s < supportSize; ++s) PetscCall(DMLabelSetValue(subpointMap, support[s], dim));
2682     }
2683   }
2684   PetscCall(PetscFree2(pStart, pEnd));
2685   PetscFunctionReturn(PETSC_SUCCESS);
2686 }
2687 
2688 static PetscErrorCode DMPlexGetFaceOrientation(DM dm, PetscInt cell, PetscInt numCorners, PetscInt indices[], PetscInt oppositeVertex, PetscInt origVertices[], PetscInt faceVertices[], PetscBool *posOriented)
2689 {
2690   MPI_Comm       comm;
2691   PetscBool      posOrient = PETSC_FALSE;
2692   const PetscInt debug     = 0;
2693   PetscInt       cellDim, faceSize, f;
2694 
2695   PetscFunctionBegin;
2696   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
2697   PetscCall(DMGetDimension(dm, &cellDim));
2698   if (debug) PetscCall(PetscPrintf(comm, "cellDim: %" PetscInt_FMT " numCorners: %" PetscInt_FMT "\n", cellDim, numCorners));
2699 
2700   if (cellDim == 1 && numCorners == 2) {
2701     /* Triangle */
2702     faceSize  = numCorners - 1;
2703     posOrient = !(oppositeVertex % 2) ? PETSC_TRUE : PETSC_FALSE;
2704   } else if (cellDim == 2 && numCorners == 3) {
2705     /* Triangle */
2706     faceSize  = numCorners - 1;
2707     posOrient = !(oppositeVertex % 2) ? PETSC_TRUE : PETSC_FALSE;
2708   } else if (cellDim == 3 && numCorners == 4) {
2709     /* Tetrahedron */
2710     faceSize  = numCorners - 1;
2711     posOrient = (oppositeVertex % 2) ? PETSC_TRUE : PETSC_FALSE;
2712   } else if (cellDim == 1 && numCorners == 3) {
2713     /* Quadratic line */
2714     faceSize  = 1;
2715     posOrient = PETSC_TRUE;
2716   } else if (cellDim == 2 && numCorners == 4) {
2717     /* Quads */
2718     faceSize = 2;
2719     if ((indices[1] > indices[0]) && (indices[1] - indices[0] == 1)) {
2720       posOrient = PETSC_TRUE;
2721     } else if ((indices[0] == 3) && (indices[1] == 0)) {
2722       posOrient = PETSC_TRUE;
2723     } else {
2724       if (((indices[0] > indices[1]) && (indices[0] - indices[1] == 1)) || ((indices[0] == 0) && (indices[1] == 3))) {
2725         posOrient = PETSC_FALSE;
2726       } else SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid quad crossedge");
2727     }
2728   } else if (cellDim == 2 && numCorners == 6) {
2729     /* Quadratic triangle (I hate this) */
2730     /* Edges are determined by the first 2 vertices (corners of edges) */
2731     const PetscInt faceSizeTri = 3;
2732     PetscInt       sortedIndices[3], i, iFace;
2733     PetscBool      found                    = PETSC_FALSE;
2734     PetscInt       faceVerticesTriSorted[9] = {
2735       0, 3, 4, /* bottom */
2736       1, 4, 5, /* right */
2737       2, 3, 5, /* left */
2738     };
2739     PetscInt faceVerticesTri[9] = {
2740       0, 3, 4, /* bottom */
2741       1, 4, 5, /* right */
2742       2, 5, 3, /* left */
2743     };
2744 
2745     for (i = 0; i < faceSizeTri; ++i) sortedIndices[i] = indices[i];
2746     PetscCall(PetscSortInt(faceSizeTri, sortedIndices));
2747     for (iFace = 0; iFace < 3; ++iFace) {
2748       const PetscInt ii = iFace * faceSizeTri;
2749       PetscInt       fVertex, cVertex;
2750 
2751       if ((sortedIndices[0] == faceVerticesTriSorted[ii + 0]) && (sortedIndices[1] == faceVerticesTriSorted[ii + 1])) {
2752         for (fVertex = 0; fVertex < faceSizeTri; ++fVertex) {
2753           for (cVertex = 0; cVertex < faceSizeTri; ++cVertex) {
2754             if (indices[cVertex] == faceVerticesTri[ii + fVertex]) {
2755               faceVertices[fVertex] = origVertices[cVertex];
2756               break;
2757             }
2758           }
2759         }
2760         found = PETSC_TRUE;
2761         break;
2762       }
2763     }
2764     PetscCheck(found, comm, PETSC_ERR_ARG_WRONG, "Invalid tri crossface");
2765     if (posOriented) *posOriented = PETSC_TRUE;
2766     PetscFunctionReturn(PETSC_SUCCESS);
2767   } else if (cellDim == 2 && numCorners == 9) {
2768     /* Quadratic quad (I hate this) */
2769     /* Edges are determined by the first 2 vertices (corners of edges) */
2770     const PetscInt faceSizeQuad = 3;
2771     PetscInt       sortedIndices[3], i, iFace;
2772     PetscBool      found                      = PETSC_FALSE;
2773     PetscInt       faceVerticesQuadSorted[12] = {
2774       0, 1, 4, /* bottom */
2775       1, 2, 5, /* right */
2776       2, 3, 6, /* top */
2777       0, 3, 7, /* left */
2778     };
2779     PetscInt faceVerticesQuad[12] = {
2780       0, 1, 4, /* bottom */
2781       1, 2, 5, /* right */
2782       2, 3, 6, /* top */
2783       3, 0, 7, /* left */
2784     };
2785 
2786     for (i = 0; i < faceSizeQuad; ++i) sortedIndices[i] = indices[i];
2787     PetscCall(PetscSortInt(faceSizeQuad, sortedIndices));
2788     for (iFace = 0; iFace < 4; ++iFace) {
2789       const PetscInt ii = iFace * faceSizeQuad;
2790       PetscInt       fVertex, cVertex;
2791 
2792       if ((sortedIndices[0] == faceVerticesQuadSorted[ii + 0]) && (sortedIndices[1] == faceVerticesQuadSorted[ii + 1])) {
2793         for (fVertex = 0; fVertex < faceSizeQuad; ++fVertex) {
2794           for (cVertex = 0; cVertex < faceSizeQuad; ++cVertex) {
2795             if (indices[cVertex] == faceVerticesQuad[ii + fVertex]) {
2796               faceVertices[fVertex] = origVertices[cVertex];
2797               break;
2798             }
2799           }
2800         }
2801         found = PETSC_TRUE;
2802         break;
2803       }
2804     }
2805     PetscCheck(found, comm, PETSC_ERR_ARG_WRONG, "Invalid quad crossface");
2806     if (posOriented) *posOriented = PETSC_TRUE;
2807     PetscFunctionReturn(PETSC_SUCCESS);
2808   } else if (cellDim == 3 && numCorners == 8) {
2809     /* Hexes
2810        A hex is two oriented quads with the normal of the first
2811        pointing up at the second.
2812 
2813           7---6
2814          /|  /|
2815         4---5 |
2816         | 1-|-2
2817         |/  |/
2818         0---3
2819 
2820         Faces are determined by the first 4 vertices (corners of faces) */
2821     const PetscInt faceSizeHex = 4;
2822     PetscInt       sortedIndices[4], i, iFace;
2823     PetscBool      found                     = PETSC_FALSE;
2824     PetscInt       faceVerticesHexSorted[24] = {
2825       0, 1, 2, 3, /* bottom */
2826       4, 5, 6, 7, /* top */
2827       0, 3, 4, 5, /* front */
2828       2, 3, 5, 6, /* right */
2829       1, 2, 6, 7, /* back */
2830       0, 1, 4, 7, /* left */
2831     };
2832     PetscInt faceVerticesHex[24] = {
2833       1, 2, 3, 0, /* bottom */
2834       4, 5, 6, 7, /* top */
2835       0, 3, 5, 4, /* front */
2836       3, 2, 6, 5, /* right */
2837       2, 1, 7, 6, /* back */
2838       1, 0, 4, 7, /* left */
2839     };
2840 
2841     for (i = 0; i < faceSizeHex; ++i) sortedIndices[i] = indices[i];
2842     PetscCall(PetscSortInt(faceSizeHex, sortedIndices));
2843     for (iFace = 0; iFace < 6; ++iFace) {
2844       const PetscInt ii = iFace * faceSizeHex;
2845       PetscInt       fVertex, cVertex;
2846 
2847       if ((sortedIndices[0] == faceVerticesHexSorted[ii + 0]) && (sortedIndices[1] == faceVerticesHexSorted[ii + 1]) && (sortedIndices[2] == faceVerticesHexSorted[ii + 2]) && (sortedIndices[3] == faceVerticesHexSorted[ii + 3])) {
2848         for (fVertex = 0; fVertex < faceSizeHex; ++fVertex) {
2849           for (cVertex = 0; cVertex < faceSizeHex; ++cVertex) {
2850             if (indices[cVertex] == faceVerticesHex[ii + fVertex]) {
2851               faceVertices[fVertex] = origVertices[cVertex];
2852               break;
2853             }
2854           }
2855         }
2856         found = PETSC_TRUE;
2857         break;
2858       }
2859     }
2860     PetscCheck(found, comm, PETSC_ERR_ARG_WRONG, "Invalid hex crossface");
2861     if (posOriented) *posOriented = PETSC_TRUE;
2862     PetscFunctionReturn(PETSC_SUCCESS);
2863   } else if (cellDim == 3 && numCorners == 10) {
2864     /* Quadratic tet */
2865     /* Faces are determined by the first 3 vertices (corners of faces) */
2866     const PetscInt faceSizeTet = 6;
2867     PetscInt       sortedIndices[6], i, iFace;
2868     PetscBool      found                     = PETSC_FALSE;
2869     PetscInt       faceVerticesTetSorted[24] = {
2870       0, 1, 2, 6, 7, 8, /* bottom */
2871       0, 3, 4, 6, 7, 9, /* front */
2872       1, 4, 5, 7, 8, 9, /* right */
2873       2, 3, 5, 6, 8, 9, /* left */
2874     };
2875     PetscInt faceVerticesTet[24] = {
2876       0, 1, 2, 6, 7, 8, /* bottom */
2877       0, 4, 3, 6, 7, 9, /* front */
2878       1, 5, 4, 7, 8, 9, /* right */
2879       2, 3, 5, 8, 6, 9, /* left */
2880     };
2881 
2882     for (i = 0; i < faceSizeTet; ++i) sortedIndices[i] = indices[i];
2883     PetscCall(PetscSortInt(faceSizeTet, sortedIndices));
2884     for (iFace = 0; iFace < 4; ++iFace) {
2885       const PetscInt ii = iFace * faceSizeTet;
2886       PetscInt       fVertex, cVertex;
2887 
2888       if ((sortedIndices[0] == faceVerticesTetSorted[ii + 0]) && (sortedIndices[1] == faceVerticesTetSorted[ii + 1]) && (sortedIndices[2] == faceVerticesTetSorted[ii + 2]) && (sortedIndices[3] == faceVerticesTetSorted[ii + 3])) {
2889         for (fVertex = 0; fVertex < faceSizeTet; ++fVertex) {
2890           for (cVertex = 0; cVertex < faceSizeTet; ++cVertex) {
2891             if (indices[cVertex] == faceVerticesTet[ii + fVertex]) {
2892               faceVertices[fVertex] = origVertices[cVertex];
2893               break;
2894             }
2895           }
2896         }
2897         found = PETSC_TRUE;
2898         break;
2899       }
2900     }
2901     PetscCheck(found, comm, PETSC_ERR_ARG_WRONG, "Invalid tet crossface");
2902     if (posOriented) *posOriented = PETSC_TRUE;
2903     PetscFunctionReturn(PETSC_SUCCESS);
2904   } else if (cellDim == 3 && numCorners == 27) {
2905     /* Quadratic hexes (I hate this)
2906        A hex is two oriented quads with the normal of the first
2907        pointing up at the second.
2908 
2909          7---6
2910         /|  /|
2911        4---5 |
2912        | 3-|-2
2913        |/  |/
2914        0---1
2915 
2916        Faces are determined by the first 4 vertices (corners of faces) */
2917     const PetscInt faceSizeQuadHex = 9;
2918     PetscInt       sortedIndices[9], i, iFace;
2919     PetscBool      found                         = PETSC_FALSE;
2920     PetscInt       faceVerticesQuadHexSorted[54] = {
2921       0, 1, 2, 3, 8,  9,  10, 11, 24, /* bottom */
2922       4, 5, 6, 7, 12, 13, 14, 15, 25, /* top */
2923       0, 1, 4, 5, 8,  12, 16, 17, 22, /* front */
2924       1, 2, 5, 6, 9,  13, 17, 18, 21, /* right */
2925       2, 3, 6, 7, 10, 14, 18, 19, 23, /* back */
2926       0, 3, 4, 7, 11, 15, 16, 19, 20, /* left */
2927     };
2928     PetscInt faceVerticesQuadHex[54] = {
2929       3, 2, 1, 0, 10, 9,  8,  11, 24, /* bottom */
2930       4, 5, 6, 7, 12, 13, 14, 15, 25, /* top */
2931       0, 1, 5, 4, 8,  17, 12, 16, 22, /* front */
2932       1, 2, 6, 5, 9,  18, 13, 17, 21, /* right */
2933       2, 3, 7, 6, 10, 19, 14, 18, 23, /* back */
2934       3, 0, 4, 7, 11, 16, 15, 19, 20  /* left */
2935     };
2936 
2937     for (i = 0; i < faceSizeQuadHex; ++i) sortedIndices[i] = indices[i];
2938     PetscCall(PetscSortInt(faceSizeQuadHex, sortedIndices));
2939     for (iFace = 0; iFace < 6; ++iFace) {
2940       const PetscInt ii = iFace * faceSizeQuadHex;
2941       PetscInt       fVertex, cVertex;
2942 
2943       if ((sortedIndices[0] == faceVerticesQuadHexSorted[ii + 0]) && (sortedIndices[1] == faceVerticesQuadHexSorted[ii + 1]) && (sortedIndices[2] == faceVerticesQuadHexSorted[ii + 2]) && (sortedIndices[3] == faceVerticesQuadHexSorted[ii + 3])) {
2944         for (fVertex = 0; fVertex < faceSizeQuadHex; ++fVertex) {
2945           for (cVertex = 0; cVertex < faceSizeQuadHex; ++cVertex) {
2946             if (indices[cVertex] == faceVerticesQuadHex[ii + fVertex]) {
2947               faceVertices[fVertex] = origVertices[cVertex];
2948               break;
2949             }
2950           }
2951         }
2952         found = PETSC_TRUE;
2953         break;
2954       }
2955     }
2956     PetscCheck(found, comm, PETSC_ERR_ARG_WRONG, "Invalid hex crossface");
2957     if (posOriented) *posOriented = PETSC_TRUE;
2958     PetscFunctionReturn(PETSC_SUCCESS);
2959   } else SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Unknown cell type for faceOrientation().");
2960   if (!posOrient) {
2961     if (debug) PetscCall(PetscPrintf(comm, "  Reversing initial face orientation\n"));
2962     for (f = 0; f < faceSize; ++f) faceVertices[f] = origVertices[faceSize - 1 - f];
2963   } else {
2964     if (debug) PetscCall(PetscPrintf(comm, "  Keeping initial face orientation\n"));
2965     for (f = 0; f < faceSize; ++f) faceVertices[f] = origVertices[f];
2966   }
2967   if (posOriented) *posOriented = posOrient;
2968   PetscFunctionReturn(PETSC_SUCCESS);
2969 }
2970 
2971 /*@
2972   DMPlexGetOrientedFace - Given a cell and a face, as a set of vertices, return the oriented face, as a set of vertices,
2973   in faceVertices. The orientation is such that the face normal points out of the cell
2974 
2975   Not Collective
2976 
2977   Input Parameters:
2978 + dm           - The original mesh
2979 . cell         - The cell mesh point
2980 . faceSize     - The number of vertices on the face
2981 . face         - The face vertices
2982 . numCorners   - The number of vertices on the cell
2983 . indices      - Local numbering of face vertices in cell cone
2984 - origVertices - Original face vertices
2985 
2986   Output Parameters:
2987 + faceVertices - The face vertices properly oriented
2988 - posOriented  - `PETSC_TRUE` if the face was oriented with outward normal
2989 
2990   Level: developer
2991 
2992 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetCone()`
2993 @*/
2994 PetscErrorCode DMPlexGetOrientedFace(DM dm, PetscInt cell, PetscInt faceSize, const PetscInt face[], PetscInt numCorners, PetscInt indices[], PetscInt origVertices[], PetscInt faceVertices[], PetscBool *posOriented)
2995 {
2996   const PetscInt *cone = NULL;
2997   PetscInt        coneSize, v, f, v2;
2998   PetscInt        oppositeVertex = -1;
2999 
3000   PetscFunctionBegin;
3001   PetscCall(DMPlexGetConeSize(dm, cell, &coneSize));
3002   PetscCall(DMPlexGetCone(dm, cell, &cone));
3003   for (v = 0, v2 = 0; v < coneSize; ++v) {
3004     PetscBool found = PETSC_FALSE;
3005 
3006     for (f = 0; f < faceSize; ++f) {
3007       if (face[f] == cone[v]) {
3008         found = PETSC_TRUE;
3009         break;
3010       }
3011     }
3012     if (found) {
3013       indices[v2]      = v;
3014       origVertices[v2] = cone[v];
3015       ++v2;
3016     } else {
3017       oppositeVertex = v;
3018     }
3019   }
3020   PetscCall(DMPlexGetFaceOrientation(dm, cell, numCorners, indices, oppositeVertex, origVertices, faceVertices, posOriented));
3021   PetscFunctionReturn(PETSC_SUCCESS);
3022 }
3023 
3024 /*
3025   DMPlexInsertFace_Internal - Puts a face into the mesh
3026 
3027   Not Collective
3028 
3029   Input Parameters:
3030   + dm              - The `DMPLEX`
3031   . numFaceVertex   - The number of vertices in the face
3032   . faceVertices    - The vertices in the face for `dm`
3033   . subfaceVertices - The vertices in the face for subdm
3034   . numCorners      - The number of vertices in the `cell`
3035   . cell            - A cell in `dm` containing the face
3036   . subcell         - A cell in subdm containing the face
3037   . firstFace       - First face in the mesh
3038   - newFacePoint    - Next face in the mesh
3039 
3040   Output Parameter:
3041   . newFacePoint - Contains next face point number on input, updated on output
3042 
3043   Level: developer
3044 */
3045 static PetscErrorCode DMPlexInsertFace_Internal(DM dm, DM subdm, PetscInt numFaceVertices, const PetscInt faceVertices[], const PetscInt subfaceVertices[], PetscInt numCorners, PetscInt cell, PetscInt subcell, PetscInt firstFace, PetscInt *newFacePoint)
3046 {
3047   MPI_Comm        comm;
3048   DM_Plex        *submesh = (DM_Plex *)subdm->data;
3049   const PetscInt *faces;
3050   PetscInt        numFaces, coneSize;
3051 
3052   PetscFunctionBegin;
3053   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
3054   PetscCall(DMPlexGetConeSize(subdm, subcell, &coneSize));
3055   PetscCheck(coneSize == 1, comm, PETSC_ERR_ARG_OUTOFRANGE, "Cone size of cell %" PetscInt_FMT " is %" PetscInt_FMT " != 1", cell, coneSize);
3056 #if 0
3057   /* Cannot use this because support() has not been constructed yet */
3058   PetscCall(DMPlexGetJoin(subdm, numFaceVertices, subfaceVertices, &numFaces, &faces));
3059 #else
3060   {
3061     PetscInt f;
3062 
3063     numFaces = 0;
3064     PetscCall(DMGetWorkArray(subdm, 1, MPIU_INT, (void **)&faces));
3065     for (f = firstFace; f < *newFacePoint; ++f) {
3066       PetscInt dof, off, d;
3067 
3068       PetscCall(PetscSectionGetDof(submesh->coneSection, f, &dof));
3069       PetscCall(PetscSectionGetOffset(submesh->coneSection, f, &off));
3070       /* Yes, I know this is quadratic, but I expect the sizes to be <5 */
3071       for (d = 0; d < dof; ++d) {
3072         const PetscInt p = submesh->cones[off + d];
3073         PetscInt       v;
3074 
3075         for (v = 0; v < numFaceVertices; ++v) {
3076           if (subfaceVertices[v] == p) break;
3077         }
3078         if (v == numFaceVertices) break;
3079       }
3080       if (d == dof) {
3081         numFaces               = 1;
3082         ((PetscInt *)faces)[0] = f;
3083       }
3084     }
3085   }
3086 #endif
3087   PetscCheck(numFaces <= 1, comm, PETSC_ERR_ARG_WRONG, "Vertex set had %" PetscInt_FMT " faces, not one", numFaces);
3088   if (numFaces == 1) {
3089     /* Add the other cell neighbor for this face */
3090     PetscCall(DMPlexSetCone(subdm, subcell, faces));
3091   } else {
3092     PetscInt *indices, *origVertices, *orientedVertices, *orientedSubVertices, v, ov;
3093     PetscBool posOriented;
3094 
3095     PetscCall(DMGetWorkArray(subdm, 4 * numFaceVertices * sizeof(PetscInt), MPIU_INT, &orientedVertices));
3096     origVertices        = &orientedVertices[numFaceVertices];
3097     indices             = &orientedVertices[numFaceVertices * 2];
3098     orientedSubVertices = &orientedVertices[numFaceVertices * 3];
3099     PetscCall(DMPlexGetOrientedFace(dm, cell, numFaceVertices, faceVertices, numCorners, indices, origVertices, orientedVertices, &posOriented));
3100     /* TODO: I know that routine should return a permutation, not the indices */
3101     for (v = 0; v < numFaceVertices; ++v) {
3102       const PetscInt vertex = faceVertices[v], subvertex = subfaceVertices[v];
3103       for (ov = 0; ov < numFaceVertices; ++ov) {
3104         if (orientedVertices[ov] == vertex) {
3105           orientedSubVertices[ov] = subvertex;
3106           break;
3107         }
3108       }
3109       PetscCheck(ov != numFaceVertices, comm, PETSC_ERR_PLIB, "Could not find face vertex %" PetscInt_FMT " in orientated set", vertex);
3110     }
3111     PetscCall(DMPlexSetCone(subdm, *newFacePoint, orientedSubVertices));
3112     PetscCall(DMPlexSetCone(subdm, subcell, newFacePoint));
3113     PetscCall(DMRestoreWorkArray(subdm, 4 * numFaceVertices * sizeof(PetscInt), MPIU_INT, &orientedVertices));
3114     ++(*newFacePoint);
3115   }
3116 #if 0
3117   PetscCall(DMPlexRestoreJoin(subdm, numFaceVertices, subfaceVertices, &numFaces, &faces));
3118 #else
3119   PetscCall(DMRestoreWorkArray(subdm, 1, MPIU_INT, (void **)&faces));
3120 #endif
3121   PetscFunctionReturn(PETSC_SUCCESS);
3122 }
3123 
3124 static PetscErrorCode DMPlexCreateSubmesh_Uninterpolated(DM dm, DMLabel vertexLabel, PetscInt value, DM subdm)
3125 {
3126   MPI_Comm        comm;
3127   DMLabel         subpointMap;
3128   IS              subvertexIS, subcellIS;
3129   const PetscInt *subVertices, *subCells;
3130   PetscInt        numSubVertices, firstSubVertex, numSubCells;
3131   PetscInt       *subface, maxConeSize, numSubFaces = 0, firstSubFace, newFacePoint, nFV = 0;
3132   PetscInt        vStart, vEnd, c, f;
3133 
3134   PetscFunctionBegin;
3135   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
3136   /* Create subpointMap which marks the submesh */
3137   PetscCall(DMLabelCreate(PETSC_COMM_SELF, "subpoint_map", &subpointMap));
3138   PetscCall(DMPlexSetSubpointMap(subdm, subpointMap));
3139   if (vertexLabel) PetscCall(DMPlexMarkSubmesh_Uninterpolated(dm, vertexLabel, value, subpointMap, &numSubFaces, &nFV, subdm));
3140   /* Setup chart */
3141   PetscCall(DMLabelGetStratumSize(subpointMap, 0, &numSubVertices));
3142   PetscCall(DMLabelGetStratumSize(subpointMap, 2, &numSubCells));
3143   PetscCall(DMPlexSetChart(subdm, 0, numSubCells + numSubFaces + numSubVertices));
3144   PetscCall(DMPlexSetVTKCellHeight(subdm, 1));
3145   /* Set cone sizes */
3146   firstSubVertex = numSubCells;
3147   firstSubFace   = numSubCells + numSubVertices;
3148   newFacePoint   = firstSubFace;
3149   PetscCall(DMLabelGetStratumIS(subpointMap, 0, &subvertexIS));
3150   if (subvertexIS) PetscCall(ISGetIndices(subvertexIS, &subVertices));
3151   PetscCall(DMLabelGetStratumIS(subpointMap, 2, &subcellIS));
3152   if (subcellIS) PetscCall(ISGetIndices(subcellIS, &subCells));
3153   for (c = 0; c < numSubCells; ++c) PetscCall(DMPlexSetConeSize(subdm, c, 1));
3154   for (f = firstSubFace; f < firstSubFace + numSubFaces; ++f) PetscCall(DMPlexSetConeSize(subdm, f, nFV));
3155   PetscCall(DMSetUp(subdm));
3156   PetscCall(DMLabelDestroy(&subpointMap));
3157   /* Create face cones */
3158   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
3159   PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, NULL));
3160   PetscCall(DMGetWorkArray(subdm, maxConeSize, MPIU_INT, (void **)&subface));
3161   for (c = 0; c < numSubCells; ++c) {
3162     const PetscInt cell    = subCells[c];
3163     const PetscInt subcell = c;
3164     PetscInt      *closure = NULL;
3165     PetscInt       closureSize, cl, numCorners = 0, faceSize = 0;
3166 
3167     PetscCall(DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
3168     for (cl = 0; cl < closureSize * 2; cl += 2) {
3169       const PetscInt point = closure[cl];
3170       PetscInt       subVertex;
3171 
3172       if ((point >= vStart) && (point < vEnd)) {
3173         ++numCorners;
3174         PetscCall(PetscFindInt(point, numSubVertices, subVertices, &subVertex));
3175         if (subVertex >= 0) {
3176           closure[faceSize] = point;
3177           subface[faceSize] = firstSubVertex + subVertex;
3178           ++faceSize;
3179         }
3180       }
3181     }
3182     PetscCheck(faceSize <= nFV, comm, PETSC_ERR_ARG_WRONG, "Invalid submesh: Too many vertices %" PetscInt_FMT " of an element on the surface", faceSize);
3183     if (faceSize == nFV) PetscCall(DMPlexInsertFace_Internal(dm, subdm, faceSize, closure, subface, numCorners, cell, subcell, firstSubFace, &newFacePoint));
3184     PetscCall(DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
3185   }
3186   PetscCall(DMRestoreWorkArray(subdm, maxConeSize, MPIU_INT, (void **)&subface));
3187   PetscCall(DMPlexSymmetrize(subdm));
3188   PetscCall(DMPlexStratify(subdm));
3189   /* Build coordinates */
3190   {
3191     PetscSection coordSection, subCoordSection;
3192     Vec          coordinates, subCoordinates;
3193     PetscScalar *coords, *subCoords;
3194     PetscInt     numComp, coordSize, v;
3195     const char  *name;
3196 
3197     PetscCall(DMGetCoordinateSection(dm, &coordSection));
3198     PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
3199     PetscCall(DMGetCoordinateSection(subdm, &subCoordSection));
3200     PetscCall(PetscSectionSetNumFields(subCoordSection, 1));
3201     PetscCall(PetscSectionGetFieldComponents(coordSection, 0, &numComp));
3202     PetscCall(PetscSectionSetFieldComponents(subCoordSection, 0, numComp));
3203     PetscCall(PetscSectionSetChart(subCoordSection, firstSubVertex, firstSubVertex + numSubVertices));
3204     for (v = 0; v < numSubVertices; ++v) {
3205       const PetscInt vertex    = subVertices[v];
3206       const PetscInt subvertex = firstSubVertex + v;
3207       PetscInt       dof;
3208 
3209       PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3210       PetscCall(PetscSectionSetDof(subCoordSection, subvertex, dof));
3211       PetscCall(PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof));
3212     }
3213     PetscCall(PetscSectionSetUp(subCoordSection));
3214     PetscCall(PetscSectionGetStorageSize(subCoordSection, &coordSize));
3215     PetscCall(VecCreate(PETSC_COMM_SELF, &subCoordinates));
3216     PetscCall(PetscObjectGetName((PetscObject)coordinates, &name));
3217     PetscCall(PetscObjectSetName((PetscObject)subCoordinates, name));
3218     PetscCall(VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE));
3219     PetscCall(VecSetType(subCoordinates, VECSTANDARD));
3220     if (coordSize) {
3221       PetscCall(VecGetArray(coordinates, &coords));
3222       PetscCall(VecGetArray(subCoordinates, &subCoords));
3223       for (v = 0; v < numSubVertices; ++v) {
3224         const PetscInt vertex    = subVertices[v];
3225         const PetscInt subvertex = firstSubVertex + v;
3226         PetscInt       dof, off, sdof, soff, d;
3227 
3228         PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3229         PetscCall(PetscSectionGetOffset(coordSection, vertex, &off));
3230         PetscCall(PetscSectionGetDof(subCoordSection, subvertex, &sdof));
3231         PetscCall(PetscSectionGetOffset(subCoordSection, subvertex, &soff));
3232         PetscCheck(dof == sdof, comm, PETSC_ERR_PLIB, "Coordinate dimension %" PetscInt_FMT " on subvertex %" PetscInt_FMT ", vertex %" PetscInt_FMT " should be %" PetscInt_FMT, sdof, subvertex, vertex, dof);
3233         for (d = 0; d < dof; ++d) subCoords[soff + d] = coords[off + d];
3234       }
3235       PetscCall(VecRestoreArray(coordinates, &coords));
3236       PetscCall(VecRestoreArray(subCoordinates, &subCoords));
3237     }
3238     PetscCall(DMSetCoordinatesLocal(subdm, subCoordinates));
3239     PetscCall(VecDestroy(&subCoordinates));
3240   }
3241   /* Cleanup */
3242   if (subvertexIS) PetscCall(ISRestoreIndices(subvertexIS, &subVertices));
3243   PetscCall(ISDestroy(&subvertexIS));
3244   if (subcellIS) PetscCall(ISRestoreIndices(subcellIS, &subCells));
3245   PetscCall(ISDestroy(&subcellIS));
3246   PetscFunctionReturn(PETSC_SUCCESS);
3247 }
3248 
3249 /* TODO: Fix this to properly propagate up error conditions it may find */
3250 static inline PetscInt DMPlexFilterPoint_Internal(PetscInt point, PetscInt firstSubPoint, PetscInt numSubPoints, const PetscInt subPoints[])
3251 {
3252   PetscInt       subPoint;
3253   PetscErrorCode ierr;
3254 
3255   ierr = PetscFindInt(point, numSubPoints, subPoints, &subPoint);
3256   if (ierr) return -1;
3257   return subPoint < 0 ? subPoint : firstSubPoint + subPoint;
3258 }
3259 
3260 /* TODO: Fix this to properly propagate up error conditions it may find */
3261 static inline PetscInt DMPlexFilterPointPerm_Internal(PetscInt point, PetscInt firstSubPoint, PetscInt numSubPoints, const PetscInt subPoints[], const PetscInt subIndices[])
3262 {
3263   PetscInt       subPoint;
3264   PetscErrorCode ierr;
3265 
3266   ierr = PetscFindInt(point, numSubPoints, subPoints, &subPoint);
3267   if (ierr) return -1;
3268   return subPoint < 0 ? subPoint : firstSubPoint + (subIndices ? subIndices[subPoint] : subPoint);
3269 }
3270 
3271 static PetscErrorCode DMPlexFilterLabels_Internal(DM dm, const PetscInt numSubPoints[], const PetscInt *subpoints[], const PetscInt firstSubPoint[], DM subdm)
3272 {
3273   DMLabel  depthLabel;
3274   PetscInt Nl, l, d;
3275 
3276   PetscFunctionBegin;
3277   // Reset depth label for fast lookup
3278   PetscCall(DMPlexGetDepthLabel(dm, &depthLabel));
3279   PetscCall(DMLabelMakeAllInvalid_Internal(depthLabel));
3280   PetscCall(DMGetNumLabels(dm, &Nl));
3281   for (l = 0; l < Nl; ++l) {
3282     DMLabel         label, newlabel;
3283     const char     *lname;
3284     PetscBool       isDepth, isDim, isCelltype, isVTK;
3285     IS              valueIS;
3286     const PetscInt *values;
3287     PetscInt        Nv, v;
3288 
3289     PetscCall(DMGetLabelName(dm, l, &lname));
3290     PetscCall(PetscStrcmp(lname, "depth", &isDepth));
3291     PetscCall(PetscStrcmp(lname, "dim", &isDim));
3292     PetscCall(PetscStrcmp(lname, "celltype", &isCelltype));
3293     PetscCall(PetscStrcmp(lname, "vtk", &isVTK));
3294     if (isDepth || isDim || isCelltype || isVTK) continue;
3295     PetscCall(DMCreateLabel(subdm, lname));
3296     PetscCall(DMGetLabel(dm, lname, &label));
3297     PetscCall(DMGetLabel(subdm, lname, &newlabel));
3298     PetscCall(DMLabelGetDefaultValue(label, &v));
3299     PetscCall(DMLabelSetDefaultValue(newlabel, v));
3300     PetscCall(DMLabelGetValueIS(label, &valueIS));
3301     PetscCall(ISGetLocalSize(valueIS, &Nv));
3302     PetscCall(ISGetIndices(valueIS, &values));
3303     for (v = 0; v < Nv; ++v) {
3304       IS              pointIS;
3305       const PetscInt *points;
3306       PetscInt        Np, p;
3307 
3308       PetscCall(DMLabelGetStratumIS(label, values[v], &pointIS));
3309       PetscCall(ISGetLocalSize(pointIS, &Np));
3310       PetscCall(ISGetIndices(pointIS, &points));
3311       for (p = 0; p < Np; ++p) {
3312         const PetscInt point = points[p];
3313         PetscInt       subp;
3314 
3315         PetscCall(DMPlexGetPointDepth(dm, point, &d));
3316         subp = DMPlexFilterPoint_Internal(point, firstSubPoint[d], numSubPoints[d], subpoints[d]);
3317         if (subp >= 0) PetscCall(DMLabelSetValue(newlabel, subp, values[v]));
3318       }
3319       PetscCall(ISRestoreIndices(pointIS, &points));
3320       PetscCall(ISDestroy(&pointIS));
3321     }
3322     PetscCall(ISRestoreIndices(valueIS, &values));
3323     PetscCall(ISDestroy(&valueIS));
3324   }
3325   PetscFunctionReturn(PETSC_SUCCESS);
3326 }
3327 
3328 static PetscErrorCode DMPlexCreateSubmeshGeneric_Interpolated(DM dm, DMLabel label, PetscInt value, PetscBool markedFaces, PetscBool isCohesive, PetscInt cellHeight, DM subdm)
3329 {
3330   MPI_Comm         comm;
3331   DMLabel          subpointMap;
3332   IS              *subpointIS;
3333   const PetscInt **subpoints;
3334   PetscInt        *numSubPoints, *firstSubPoint, *coneNew, *orntNew;
3335   PetscInt         totSubPoints = 0, maxConeSize, dim, sdim, cdim, p, d, v;
3336   PetscMPIInt      rank;
3337 
3338   PetscFunctionBegin;
3339   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
3340   PetscCallMPI(MPI_Comm_rank(comm, &rank));
3341   /* Create subpointMap which marks the submesh */
3342   PetscCall(DMLabelCreate(PETSC_COMM_SELF, "subpoint_map", &subpointMap));
3343   PetscCall(DMPlexSetSubpointMap(subdm, subpointMap));
3344   if (cellHeight) {
3345     if (isCohesive) PetscCall(DMPlexMarkCohesiveSubmesh_Interpolated(dm, label, value, subpointMap, subdm));
3346     else PetscCall(DMPlexMarkSubmesh_Interpolated(dm, label, value, markedFaces, subpointMap, subdm));
3347   } else {
3348     DMLabel         depth;
3349     IS              pointIS;
3350     const PetscInt *points;
3351     PetscInt        numPoints = 0;
3352 
3353     PetscCall(DMPlexGetDepthLabel(dm, &depth));
3354     PetscCall(DMLabelGetStratumIS(label, value, &pointIS));
3355     if (pointIS) {
3356       PetscCall(ISGetIndices(pointIS, &points));
3357       PetscCall(ISGetLocalSize(pointIS, &numPoints));
3358     }
3359     for (p = 0; p < numPoints; ++p) {
3360       PetscInt *closure = NULL;
3361       PetscInt  closureSize, c, pdim;
3362 
3363       PetscCall(DMPlexGetTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closure));
3364       for (c = 0; c < closureSize * 2; c += 2) {
3365         PetscCall(DMLabelGetValue(depth, closure[c], &pdim));
3366         PetscCall(DMLabelSetValue(subpointMap, closure[c], pdim));
3367       }
3368       PetscCall(DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closure));
3369     }
3370     if (pointIS) PetscCall(ISRestoreIndices(pointIS, &points));
3371     PetscCall(ISDestroy(&pointIS));
3372   }
3373   /* Setup chart */
3374   PetscCall(DMGetDimension(dm, &dim));
3375   PetscCall(DMGetCoordinateDim(dm, &cdim));
3376   PetscCall(PetscMalloc4(dim + 1, &numSubPoints, dim + 1, &firstSubPoint, dim + 1, &subpointIS, dim + 1, &subpoints));
3377   for (d = 0; d <= dim; ++d) {
3378     PetscCall(DMLabelGetStratumSize(subpointMap, d, &numSubPoints[d]));
3379     totSubPoints += numSubPoints[d];
3380   }
3381   // Determine submesh dimension
3382   PetscCall(DMGetDimension(subdm, &sdim));
3383   if (sdim > 0) {
3384     // Calling function knows what dimension to use, and we include neighboring cells as well
3385     sdim = dim;
3386   } else {
3387     // We reset the subdimension based on what is being selected
3388     PetscInt lsdim;
3389     for (lsdim = dim; lsdim >= 0; --lsdim)
3390       if (numSubPoints[lsdim]) break;
3391     PetscCall(MPIU_Allreduce(&lsdim, &sdim, 1, MPIU_INT, MPI_MAX, comm));
3392     PetscCall(DMSetDimension(subdm, sdim));
3393     PetscCall(DMSetCoordinateDim(subdm, cdim));
3394   }
3395   PetscCall(DMPlexSetChart(subdm, 0, totSubPoints));
3396   PetscCall(DMPlexSetVTKCellHeight(subdm, cellHeight));
3397   /* Set cone sizes */
3398   firstSubPoint[sdim] = 0;
3399   firstSubPoint[0]    = firstSubPoint[sdim] + numSubPoints[sdim];
3400   if (sdim > 1) firstSubPoint[sdim - 1] = firstSubPoint[0] + numSubPoints[0];
3401   if (sdim > 2) firstSubPoint[sdim - 2] = firstSubPoint[sdim - 1] + numSubPoints[sdim - 1];
3402   for (d = 0; d <= sdim; ++d) {
3403     PetscCall(DMLabelGetStratumIS(subpointMap, d, &subpointIS[d]));
3404     if (subpointIS[d]) PetscCall(ISGetIndices(subpointIS[d], &subpoints[d]));
3405   }
3406   /* We do not want this label automatically computed, instead we compute it here */
3407   PetscCall(DMCreateLabel(subdm, "celltype"));
3408   for (d = 0; d <= sdim; ++d) {
3409     for (p = 0; p < numSubPoints[d]; ++p) {
3410       const PetscInt  point    = subpoints[d][p];
3411       const PetscInt  subpoint = firstSubPoint[d] + p;
3412       const PetscInt *cone;
3413       PetscInt        coneSize;
3414 
3415       PetscCall(DMPlexGetConeSize(dm, point, &coneSize));
3416       if (cellHeight && (d == sdim)) {
3417         PetscInt coneSizeNew, c, val;
3418 
3419         PetscCall(DMPlexGetCone(dm, point, &cone));
3420         for (c = 0, coneSizeNew = 0; c < coneSize; ++c) {
3421           PetscCall(DMLabelGetValue(subpointMap, cone[c], &val));
3422           if (val >= 0) coneSizeNew++;
3423         }
3424         PetscCall(DMPlexSetConeSize(subdm, subpoint, coneSizeNew));
3425         PetscCall(DMPlexSetCellType(subdm, subpoint, DM_POLYTOPE_FV_GHOST));
3426       } else {
3427         DMPolytopeType ct;
3428 
3429         PetscCall(DMPlexSetConeSize(subdm, subpoint, coneSize));
3430         PetscCall(DMPlexGetCellType(dm, point, &ct));
3431         PetscCall(DMPlexSetCellType(subdm, subpoint, ct));
3432       }
3433     }
3434   }
3435   PetscCall(DMLabelDestroy(&subpointMap));
3436   PetscCall(DMSetUp(subdm));
3437   /* Set cones */
3438   PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, NULL));
3439   PetscCall(PetscMalloc2(maxConeSize, &coneNew, maxConeSize, &orntNew));
3440   for (d = 0; d <= sdim; ++d) {
3441     for (p = 0; p < numSubPoints[d]; ++p) {
3442       const PetscInt  point    = subpoints[d][p];
3443       const PetscInt  subpoint = firstSubPoint[d] + p;
3444       const PetscInt *cone, *ornt;
3445       PetscInt        coneSize, subconeSize, coneSizeNew, c, subc, fornt = 0;
3446 
3447       if (d == sdim - 1) {
3448         const PetscInt *support, *cone, *ornt;
3449         PetscInt        supportSize, coneSize, s, subc;
3450 
3451         PetscCall(DMPlexGetSupport(dm, point, &support));
3452         PetscCall(DMPlexGetSupportSize(dm, point, &supportSize));
3453         for (s = 0; s < supportSize; ++s) {
3454           PetscBool isHybrid = PETSC_FALSE;
3455 
3456           PetscCall(DMPlexCellIsHybrid_Internal(dm, support[s], &isHybrid));
3457           if (!isHybrid) continue;
3458           PetscCall(PetscFindInt(support[s], numSubPoints[d + 1], subpoints[d + 1], &subc));
3459           if (subc >= 0) {
3460             const PetscInt ccell = subpoints[d + 1][subc];
3461 
3462             PetscCall(DMPlexGetCone(dm, ccell, &cone));
3463             PetscCall(DMPlexGetConeSize(dm, ccell, &coneSize));
3464             PetscCall(DMPlexGetConeOrientation(dm, ccell, &ornt));
3465             for (c = 0; c < coneSize; ++c) {
3466               if (cone[c] == point) {
3467                 fornt = ornt[c];
3468                 break;
3469               }
3470             }
3471             break;
3472           }
3473         }
3474       }
3475       PetscCall(DMPlexGetConeSize(dm, point, &coneSize));
3476       PetscCall(DMPlexGetConeSize(subdm, subpoint, &subconeSize));
3477       PetscCall(DMPlexGetCone(dm, point, &cone));
3478       PetscCall(DMPlexGetConeOrientation(dm, point, &ornt));
3479       for (c = 0, coneSizeNew = 0; c < coneSize; ++c) {
3480         PetscCall(PetscFindInt(cone[c], numSubPoints[d - 1], subpoints[d - 1], &subc));
3481         if (subc >= 0) {
3482           coneNew[coneSizeNew] = firstSubPoint[d - 1] + subc;
3483           orntNew[coneSizeNew] = ornt[c];
3484           ++coneSizeNew;
3485         }
3486       }
3487       PetscCheck(coneSizeNew == subconeSize, comm, PETSC_ERR_PLIB, "Number of cone points located %" PetscInt_FMT " does not match subcone size %" PetscInt_FMT, coneSizeNew, subconeSize);
3488       PetscCall(DMPlexSetCone(subdm, subpoint, coneNew));
3489       PetscCall(DMPlexSetConeOrientation(subdm, subpoint, orntNew));
3490       if (fornt < 0) PetscCall(DMPlexOrientPoint(subdm, subpoint, fornt));
3491     }
3492   }
3493   PetscCall(PetscFree2(coneNew, orntNew));
3494   PetscCall(DMPlexSymmetrize(subdm));
3495   PetscCall(DMPlexStratify(subdm));
3496   /* Build coordinates */
3497   {
3498     PetscSection coordSection, subCoordSection;
3499     Vec          coordinates, subCoordinates;
3500     PetscScalar *coords, *subCoords;
3501     PetscInt     cdim, numComp, coordSize;
3502     const char  *name;
3503 
3504     PetscCall(DMGetCoordinateDim(dm, &cdim));
3505     PetscCall(DMGetCoordinateSection(dm, &coordSection));
3506     PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
3507     PetscCall(DMGetCoordinateSection(subdm, &subCoordSection));
3508     PetscCall(PetscSectionSetNumFields(subCoordSection, 1));
3509     PetscCall(PetscSectionGetFieldComponents(coordSection, 0, &numComp));
3510     PetscCall(PetscSectionSetFieldComponents(subCoordSection, 0, numComp));
3511     PetscCall(PetscSectionSetChart(subCoordSection, firstSubPoint[0], firstSubPoint[0] + numSubPoints[0]));
3512     for (v = 0; v < numSubPoints[0]; ++v) {
3513       const PetscInt vertex    = subpoints[0][v];
3514       const PetscInt subvertex = firstSubPoint[0] + v;
3515       PetscInt       dof;
3516 
3517       PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3518       PetscCall(PetscSectionSetDof(subCoordSection, subvertex, dof));
3519       PetscCall(PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof));
3520     }
3521     PetscCall(PetscSectionSetUp(subCoordSection));
3522     PetscCall(PetscSectionGetStorageSize(subCoordSection, &coordSize));
3523     PetscCall(VecCreate(PETSC_COMM_SELF, &subCoordinates));
3524     PetscCall(PetscObjectGetName((PetscObject)coordinates, &name));
3525     PetscCall(PetscObjectSetName((PetscObject)subCoordinates, name));
3526     PetscCall(VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE));
3527     PetscCall(VecSetBlockSize(subCoordinates, cdim));
3528     PetscCall(VecSetType(subCoordinates, VECSTANDARD));
3529     PetscCall(VecGetArray(coordinates, &coords));
3530     PetscCall(VecGetArray(subCoordinates, &subCoords));
3531     for (v = 0; v < numSubPoints[0]; ++v) {
3532       const PetscInt vertex    = subpoints[0][v];
3533       const PetscInt subvertex = firstSubPoint[0] + v;
3534       PetscInt       dof, off, sdof, soff, d;
3535 
3536       PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3537       PetscCall(PetscSectionGetOffset(coordSection, vertex, &off));
3538       PetscCall(PetscSectionGetDof(subCoordSection, subvertex, &sdof));
3539       PetscCall(PetscSectionGetOffset(subCoordSection, subvertex, &soff));
3540       PetscCheck(dof == sdof, comm, PETSC_ERR_PLIB, "Coordinate dimension %" PetscInt_FMT " on subvertex %" PetscInt_FMT ", vertex %" PetscInt_FMT " should be %" PetscInt_FMT, sdof, subvertex, vertex, dof);
3541       for (d = 0; d < dof; ++d) subCoords[soff + d] = coords[off + d];
3542     }
3543     PetscCall(VecRestoreArray(coordinates, &coords));
3544     PetscCall(VecRestoreArray(subCoordinates, &subCoords));
3545     PetscCall(DMSetCoordinatesLocal(subdm, subCoordinates));
3546     PetscCall(VecDestroy(&subCoordinates));
3547   }
3548   /* Build SF: We need this complexity because subpoints might not be selected on the owning process */
3549   {
3550     PetscSF            sfPoint, sfPointSub;
3551     IS                 subpIS;
3552     const PetscSFNode *remotePoints;
3553     PetscSFNode       *sremotePoints = NULL, *newLocalPoints = NULL, *newOwners = NULL;
3554     const PetscInt    *localPoints, *subpoints, *rootdegree;
3555     PetscInt          *slocalPoints = NULL, *sortedPoints = NULL, *sortedIndices = NULL;
3556     PetscInt           numRoots, numLeaves, numSubpoints = 0, numSubroots, numSubleaves = 0, l, sl = 0, ll = 0, pStart, pEnd, p;
3557     PetscMPIInt        rank, size;
3558 
3559     PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
3560     PetscCallMPI(MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size));
3561     PetscCall(DMGetPointSF(dm, &sfPoint));
3562     PetscCall(DMGetPointSF(subdm, &sfPointSub));
3563     PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
3564     PetscCall(DMPlexGetChart(subdm, NULL, &numSubroots));
3565     PetscCall(DMPlexGetSubpointIS(subdm, &subpIS));
3566     if (subpIS) {
3567       PetscBool sorted = PETSC_TRUE;
3568 
3569       PetscCall(ISGetIndices(subpIS, &subpoints));
3570       PetscCall(ISGetLocalSize(subpIS, &numSubpoints));
3571       for (p = 1; p < numSubpoints; ++p) sorted = sorted && (subpoints[p] >= subpoints[p - 1]) ? PETSC_TRUE : PETSC_FALSE;
3572       if (!sorted) {
3573         PetscCall(PetscMalloc2(numSubpoints, &sortedPoints, numSubpoints, &sortedIndices));
3574         for (p = 0; p < numSubpoints; ++p) sortedIndices[p] = p;
3575         PetscCall(PetscArraycpy(sortedPoints, subpoints, numSubpoints));
3576         PetscCall(PetscSortIntWithArray(numSubpoints, sortedPoints, sortedIndices));
3577       }
3578     }
3579     PetscCall(PetscSFGetGraph(sfPoint, &numRoots, &numLeaves, &localPoints, &remotePoints));
3580     if (numRoots >= 0) {
3581       PetscCall(PetscSFComputeDegreeBegin(sfPoint, &rootdegree));
3582       PetscCall(PetscSFComputeDegreeEnd(sfPoint, &rootdegree));
3583       PetscCall(PetscMalloc2(pEnd - pStart, &newLocalPoints, numRoots, &newOwners));
3584       for (p = 0; p < pEnd - pStart; ++p) {
3585         newLocalPoints[p].rank  = -2;
3586         newLocalPoints[p].index = -2;
3587       }
3588       /* Set subleaves */
3589       for (l = 0; l < numLeaves; ++l) {
3590         const PetscInt point    = localPoints[l];
3591         const PetscInt subpoint = DMPlexFilterPointPerm_Internal(point, 0, numSubpoints, sortedPoints ? sortedPoints : subpoints, sortedIndices);
3592 
3593         if (subpoint < 0) continue;
3594         newLocalPoints[point - pStart].rank  = rank;
3595         newLocalPoints[point - pStart].index = subpoint;
3596         ++numSubleaves;
3597       }
3598       /* Must put in owned subpoints */
3599       for (p = pStart; p < pEnd; ++p) {
3600         newOwners[p - pStart].rank  = -3;
3601         newOwners[p - pStart].index = -3;
3602       }
3603       for (p = 0; p < numSubpoints; ++p) {
3604         /* Hold on to currently owned points */
3605         if (rootdegree[subpoints[p] - pStart]) newOwners[subpoints[p] - pStart].rank = rank + size;
3606         else newOwners[subpoints[p] - pStart].rank = rank;
3607         newOwners[subpoints[p] - pStart].index = p;
3608       }
3609       PetscCall(PetscSFReduceBegin(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC));
3610       PetscCall(PetscSFReduceEnd(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC));
3611       for (p = pStart; p < pEnd; ++p)
3612         if (newOwners[p - pStart].rank >= size) newOwners[p - pStart].rank -= size;
3613       PetscCall(PetscSFBcastBegin(sfPoint, MPIU_2INT, newOwners, newLocalPoints, MPI_REPLACE));
3614       PetscCall(PetscSFBcastEnd(sfPoint, MPIU_2INT, newOwners, newLocalPoints, MPI_REPLACE));
3615       PetscCall(PetscMalloc1(numSubleaves, &slocalPoints));
3616       PetscCall(PetscMalloc1(numSubleaves, &sremotePoints));
3617       for (l = 0; l < numLeaves; ++l) {
3618         const PetscInt point    = localPoints[l];
3619         const PetscInt subpoint = DMPlexFilterPointPerm_Internal(point, 0, numSubpoints, sortedPoints ? sortedPoints : subpoints, sortedIndices);
3620 
3621         if (subpoint < 0) continue;
3622         if (newLocalPoints[point].rank == rank) {
3623           ++ll;
3624           continue;
3625         }
3626         slocalPoints[sl]        = subpoint;
3627         sremotePoints[sl].rank  = newLocalPoints[point].rank;
3628         sremotePoints[sl].index = newLocalPoints[point].index;
3629         PetscCheck(sremotePoints[sl].rank >= 0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote rank for local point %" PetscInt_FMT, point);
3630         PetscCheck(sremotePoints[sl].index >= 0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote subpoint for local point %" PetscInt_FMT, point);
3631         ++sl;
3632       }
3633       PetscCheck(sl + ll == numSubleaves, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatch in number of subleaves %" PetscInt_FMT " + %" PetscInt_FMT " != %" PetscInt_FMT, sl, ll, numSubleaves);
3634       PetscCall(PetscFree2(newLocalPoints, newOwners));
3635       PetscCall(PetscSFSetGraph(sfPointSub, numSubroots, sl, slocalPoints, PETSC_OWN_POINTER, sremotePoints, PETSC_OWN_POINTER));
3636     }
3637     if (subpIS) PetscCall(ISRestoreIndices(subpIS, &subpoints));
3638     PetscCall(PetscFree2(sortedPoints, sortedIndices));
3639   }
3640   /* Filter labels */
3641   PetscCall(DMPlexFilterLabels_Internal(dm, numSubPoints, subpoints, firstSubPoint, subdm));
3642   /* Cleanup */
3643   for (d = 0; d <= sdim; ++d) {
3644     if (subpointIS[d]) PetscCall(ISRestoreIndices(subpointIS[d], &subpoints[d]));
3645     PetscCall(ISDestroy(&subpointIS[d]));
3646   }
3647   PetscCall(PetscFree4(numSubPoints, firstSubPoint, subpointIS, subpoints));
3648   PetscFunctionReturn(PETSC_SUCCESS);
3649 }
3650 
3651 static PetscErrorCode DMPlexCreateSubmesh_Interpolated(DM dm, DMLabel vertexLabel, PetscInt value, PetscBool markedFaces, DM subdm)
3652 {
3653   PetscFunctionBegin;
3654   PetscCall(DMPlexCreateSubmeshGeneric_Interpolated(dm, vertexLabel, value, markedFaces, PETSC_FALSE, 1, subdm));
3655   PetscFunctionReturn(PETSC_SUCCESS);
3656 }
3657 
3658 /*@
3659   DMPlexCreateSubmesh - Extract a hypersurface from the mesh using vertices defined by a label
3660 
3661   Input Parameters:
3662 + dm          - The original mesh
3663 . vertexLabel - The `DMLabel` marking points contained in the surface
3664 . value       - The label value to use
3665 - markedFaces - `PETSC_TRUE` if surface faces are marked in addition to vertices, `PETSC_FALSE` if only vertices are marked
3666 
3667   Output Parameter:
3668 . subdm - The surface mesh
3669 
3670   Level: developer
3671 
3672   Note:
3673   This function produces a `DMLabel` mapping original points in the submesh to their depth. This can be obtained using `DMPlexGetSubpointMap()`.
3674 
3675 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetSubpointMap()`, `DMGetLabel()`, `DMLabelSetValue()`
3676 @*/
3677 PetscErrorCode DMPlexCreateSubmesh(DM dm, DMLabel vertexLabel, PetscInt value, PetscBool markedFaces, DM *subdm)
3678 {
3679   DMPlexInterpolatedFlag interpolated;
3680   PetscInt               dim, cdim;
3681 
3682   PetscFunctionBegin;
3683   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3684   PetscAssertPointer(subdm, 5);
3685   PetscCall(DMGetDimension(dm, &dim));
3686   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), subdm));
3687   PetscCall(DMSetType(*subdm, DMPLEX));
3688   PetscCall(DMSetDimension(*subdm, dim - 1));
3689   PetscCall(DMGetCoordinateDim(dm, &cdim));
3690   PetscCall(DMSetCoordinateDim(*subdm, cdim));
3691   PetscCall(DMPlexIsInterpolated(dm, &interpolated));
3692   PetscCheck(interpolated != DMPLEX_INTERPOLATED_PARTIAL, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Not for partially interpolated meshes");
3693   if (interpolated) {
3694     PetscCall(DMPlexCreateSubmesh_Interpolated(dm, vertexLabel, value, markedFaces, *subdm));
3695   } else {
3696     PetscCall(DMPlexCreateSubmesh_Uninterpolated(dm, vertexLabel, value, *subdm));
3697   }
3698   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_TRUE, *subdm));
3699   PetscFunctionReturn(PETSC_SUCCESS);
3700 }
3701 
3702 static PetscErrorCode DMPlexCreateCohesiveSubmesh_Uninterpolated(DM dm, PetscBool hasLagrange, const char label[], PetscInt value, DM subdm)
3703 {
3704   MPI_Comm        comm;
3705   DMLabel         subpointMap;
3706   IS              subvertexIS;
3707   const PetscInt *subVertices;
3708   PetscInt        numSubVertices, firstSubVertex, numSubCells, *subCells = NULL;
3709   PetscInt       *subface, maxConeSize, numSubFaces, firstSubFace, newFacePoint, nFV;
3710   PetscInt        c, f;
3711 
3712   PetscFunctionBegin;
3713   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
3714   /* Create subpointMap which marks the submesh */
3715   PetscCall(DMLabelCreate(PETSC_COMM_SELF, "subpoint_map", &subpointMap));
3716   PetscCall(DMPlexSetSubpointMap(subdm, subpointMap));
3717   PetscCall(DMPlexMarkCohesiveSubmesh_Uninterpolated(dm, hasLagrange, label, value, subpointMap, &numSubFaces, &nFV, &subCells, subdm));
3718   /* Setup chart */
3719   PetscCall(DMLabelGetStratumSize(subpointMap, 0, &numSubVertices));
3720   PetscCall(DMLabelGetStratumSize(subpointMap, 2, &numSubCells));
3721   PetscCall(DMPlexSetChart(subdm, 0, numSubCells + numSubFaces + numSubVertices));
3722   PetscCall(DMPlexSetVTKCellHeight(subdm, 1));
3723   /* Set cone sizes */
3724   firstSubVertex = numSubCells;
3725   firstSubFace   = numSubCells + numSubVertices;
3726   newFacePoint   = firstSubFace;
3727   PetscCall(DMLabelGetStratumIS(subpointMap, 0, &subvertexIS));
3728   if (subvertexIS) PetscCall(ISGetIndices(subvertexIS, &subVertices));
3729   for (c = 0; c < numSubCells; ++c) PetscCall(DMPlexSetConeSize(subdm, c, 1));
3730   for (f = firstSubFace; f < firstSubFace + numSubFaces; ++f) PetscCall(DMPlexSetConeSize(subdm, f, nFV));
3731   PetscCall(DMSetUp(subdm));
3732   PetscCall(DMLabelDestroy(&subpointMap));
3733   /* Create face cones */
3734   PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, NULL));
3735   PetscCall(DMGetWorkArray(subdm, maxConeSize, MPIU_INT, (void **)&subface));
3736   for (c = 0; c < numSubCells; ++c) {
3737     const PetscInt  cell    = subCells[c];
3738     const PetscInt  subcell = c;
3739     const PetscInt *cone, *cells;
3740     PetscBool       isHybrid = PETSC_FALSE;
3741     PetscInt        numCells, subVertex, p, v;
3742 
3743     PetscCall(DMPlexCellIsHybrid_Internal(dm, cell, &isHybrid));
3744     if (!isHybrid) continue;
3745     PetscCall(DMPlexGetCone(dm, cell, &cone));
3746     for (v = 0; v < nFV; ++v) {
3747       PetscCall(PetscFindInt(cone[v], numSubVertices, subVertices, &subVertex));
3748       subface[v] = firstSubVertex + subVertex;
3749     }
3750     PetscCall(DMPlexSetCone(subdm, newFacePoint, subface));
3751     PetscCall(DMPlexSetCone(subdm, subcell, &newFacePoint));
3752     PetscCall(DMPlexGetJoin(dm, nFV, cone, &numCells, &cells));
3753     /* Not true in parallel
3754     PetscCheck(numCells == 2,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive cells should separate two cells"); */
3755     for (p = 0; p < numCells; ++p) {
3756       PetscInt  negsubcell;
3757       PetscBool isHybrid = PETSC_FALSE;
3758 
3759       PetscCall(DMPlexCellIsHybrid_Internal(dm, cells[p], &isHybrid));
3760       if (isHybrid) continue;
3761       /* I know this is a crap search */
3762       for (negsubcell = 0; negsubcell < numSubCells; ++negsubcell) {
3763         if (subCells[negsubcell] == cells[p]) break;
3764       }
3765       PetscCheck(negsubcell != numSubCells, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not find negative face neighbor for cohesive cell %" PetscInt_FMT, cell);
3766       PetscCall(DMPlexSetCone(subdm, negsubcell, &newFacePoint));
3767     }
3768     PetscCall(DMPlexRestoreJoin(dm, nFV, cone, &numCells, &cells));
3769     ++newFacePoint;
3770   }
3771   PetscCall(DMRestoreWorkArray(subdm, maxConeSize, MPIU_INT, (void **)&subface));
3772   PetscCall(DMPlexSymmetrize(subdm));
3773   PetscCall(DMPlexStratify(subdm));
3774   /* Build coordinates */
3775   {
3776     PetscSection coordSection, subCoordSection;
3777     Vec          coordinates, subCoordinates;
3778     PetscScalar *coords, *subCoords;
3779     PetscInt     cdim, numComp, coordSize, v;
3780     const char  *name;
3781 
3782     PetscCall(DMGetCoordinateDim(dm, &cdim));
3783     PetscCall(DMGetCoordinateSection(dm, &coordSection));
3784     PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
3785     PetscCall(DMGetCoordinateSection(subdm, &subCoordSection));
3786     PetscCall(PetscSectionSetNumFields(subCoordSection, 1));
3787     PetscCall(PetscSectionGetFieldComponents(coordSection, 0, &numComp));
3788     PetscCall(PetscSectionSetFieldComponents(subCoordSection, 0, numComp));
3789     PetscCall(PetscSectionSetChart(subCoordSection, firstSubVertex, firstSubVertex + numSubVertices));
3790     for (v = 0; v < numSubVertices; ++v) {
3791       const PetscInt vertex    = subVertices[v];
3792       const PetscInt subvertex = firstSubVertex + v;
3793       PetscInt       dof;
3794 
3795       PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3796       PetscCall(PetscSectionSetDof(subCoordSection, subvertex, dof));
3797       PetscCall(PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof));
3798     }
3799     PetscCall(PetscSectionSetUp(subCoordSection));
3800     PetscCall(PetscSectionGetStorageSize(subCoordSection, &coordSize));
3801     PetscCall(VecCreate(PETSC_COMM_SELF, &subCoordinates));
3802     PetscCall(PetscObjectGetName((PetscObject)coordinates, &name));
3803     PetscCall(PetscObjectSetName((PetscObject)subCoordinates, name));
3804     PetscCall(VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE));
3805     PetscCall(VecSetBlockSize(subCoordinates, cdim));
3806     PetscCall(VecSetType(subCoordinates, VECSTANDARD));
3807     PetscCall(VecGetArray(coordinates, &coords));
3808     PetscCall(VecGetArray(subCoordinates, &subCoords));
3809     for (v = 0; v < numSubVertices; ++v) {
3810       const PetscInt vertex    = subVertices[v];
3811       const PetscInt subvertex = firstSubVertex + v;
3812       PetscInt       dof, off, sdof, soff, d;
3813 
3814       PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3815       PetscCall(PetscSectionGetOffset(coordSection, vertex, &off));
3816       PetscCall(PetscSectionGetDof(subCoordSection, subvertex, &sdof));
3817       PetscCall(PetscSectionGetOffset(subCoordSection, subvertex, &soff));
3818       PetscCheck(dof == sdof, comm, PETSC_ERR_PLIB, "Coordinate dimension %" PetscInt_FMT " on subvertex %" PetscInt_FMT ", vertex %" PetscInt_FMT " should be %" PetscInt_FMT, sdof, subvertex, vertex, dof);
3819       for (d = 0; d < dof; ++d) subCoords[soff + d] = coords[off + d];
3820     }
3821     PetscCall(VecRestoreArray(coordinates, &coords));
3822     PetscCall(VecRestoreArray(subCoordinates, &subCoords));
3823     PetscCall(DMSetCoordinatesLocal(subdm, subCoordinates));
3824     PetscCall(VecDestroy(&subCoordinates));
3825   }
3826   /* Build SF */
3827   CHKMEMQ;
3828   {
3829     PetscSF            sfPoint, sfPointSub;
3830     const PetscSFNode *remotePoints;
3831     PetscSFNode       *sremotePoints, *newLocalPoints, *newOwners;
3832     const PetscInt    *localPoints;
3833     PetscInt          *slocalPoints;
3834     PetscInt           numRoots, numLeaves, numSubRoots = numSubCells + numSubFaces + numSubVertices, numSubLeaves = 0, l, sl, ll, pStart, pEnd, p, vStart, vEnd;
3835     PetscMPIInt        rank;
3836 
3837     PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
3838     PetscCall(DMGetPointSF(dm, &sfPoint));
3839     PetscCall(DMGetPointSF(subdm, &sfPointSub));
3840     PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
3841     PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
3842     PetscCall(PetscSFGetGraph(sfPoint, &numRoots, &numLeaves, &localPoints, &remotePoints));
3843     if (numRoots >= 0) {
3844       /* Only vertices should be shared */
3845       PetscCall(PetscMalloc2(pEnd - pStart, &newLocalPoints, numRoots, &newOwners));
3846       for (p = 0; p < pEnd - pStart; ++p) {
3847         newLocalPoints[p].rank  = -2;
3848         newLocalPoints[p].index = -2;
3849       }
3850       /* Set subleaves */
3851       for (l = 0; l < numLeaves; ++l) {
3852         const PetscInt point    = localPoints[l];
3853         const PetscInt subPoint = DMPlexFilterPoint_Internal(point, firstSubVertex, numSubVertices, subVertices);
3854 
3855         PetscCheck(!(point < vStart) || !(point >= vEnd), PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Should not be mapping anything but vertices, %" PetscInt_FMT, point);
3856         if (subPoint < 0) continue;
3857         newLocalPoints[point - pStart].rank  = rank;
3858         newLocalPoints[point - pStart].index = subPoint;
3859         ++numSubLeaves;
3860       }
3861       /* Must put in owned subpoints */
3862       for (p = pStart; p < pEnd; ++p) {
3863         const PetscInt subPoint = DMPlexFilterPoint_Internal(p, firstSubVertex, numSubVertices, subVertices);
3864 
3865         if (subPoint < 0) {
3866           newOwners[p - pStart].rank  = -3;
3867           newOwners[p - pStart].index = -3;
3868         } else {
3869           newOwners[p - pStart].rank  = rank;
3870           newOwners[p - pStart].index = subPoint;
3871         }
3872       }
3873       PetscCall(PetscSFReduceBegin(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC));
3874       PetscCall(PetscSFReduceEnd(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC));
3875       PetscCall(PetscSFBcastBegin(sfPoint, MPIU_2INT, newOwners, newLocalPoints, MPI_REPLACE));
3876       PetscCall(PetscSFBcastEnd(sfPoint, MPIU_2INT, newOwners, newLocalPoints, MPI_REPLACE));
3877       PetscCall(PetscMalloc1(numSubLeaves, &slocalPoints));
3878       PetscCall(PetscMalloc1(numSubLeaves, &sremotePoints));
3879       for (l = 0, sl = 0, ll = 0; l < numLeaves; ++l) {
3880         const PetscInt point    = localPoints[l];
3881         const PetscInt subPoint = DMPlexFilterPoint_Internal(point, firstSubVertex, numSubVertices, subVertices);
3882 
3883         if (subPoint < 0) continue;
3884         if (newLocalPoints[point].rank == rank) {
3885           ++ll;
3886           continue;
3887         }
3888         slocalPoints[sl]        = subPoint;
3889         sremotePoints[sl].rank  = newLocalPoints[point].rank;
3890         sremotePoints[sl].index = newLocalPoints[point].index;
3891         PetscCheck(sremotePoints[sl].rank >= 0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote rank for local point %" PetscInt_FMT, point);
3892         PetscCheck(sremotePoints[sl].index >= 0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote subpoint for local point %" PetscInt_FMT, point);
3893         ++sl;
3894       }
3895       PetscCall(PetscFree2(newLocalPoints, newOwners));
3896       PetscCheck(sl + ll == numSubLeaves, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatch in number of subleaves %" PetscInt_FMT " + %" PetscInt_FMT " != %" PetscInt_FMT, sl, ll, numSubLeaves);
3897       PetscCall(PetscSFSetGraph(sfPointSub, numSubRoots, sl, slocalPoints, PETSC_OWN_POINTER, sremotePoints, PETSC_OWN_POINTER));
3898     }
3899   }
3900   CHKMEMQ;
3901   /* Cleanup */
3902   if (subvertexIS) PetscCall(ISRestoreIndices(subvertexIS, &subVertices));
3903   PetscCall(ISDestroy(&subvertexIS));
3904   PetscCall(PetscFree(subCells));
3905   PetscFunctionReturn(PETSC_SUCCESS);
3906 }
3907 
3908 static PetscErrorCode DMPlexCreateCohesiveSubmesh_Interpolated(DM dm, const char labelname[], PetscInt value, DM subdm)
3909 {
3910   DMLabel label = NULL;
3911 
3912   PetscFunctionBegin;
3913   if (labelname) PetscCall(DMGetLabel(dm, labelname, &label));
3914   PetscCall(DMPlexCreateSubmeshGeneric_Interpolated(dm, label, value, PETSC_FALSE, PETSC_TRUE, 1, subdm));
3915   PetscFunctionReturn(PETSC_SUCCESS);
3916 }
3917 
3918 /*@C
3919   DMPlexCreateCohesiveSubmesh - Extract from a mesh with cohesive cells the hypersurface defined by one face of the cells. Optionally, a label can be given to restrict the cells.
3920 
3921   Input Parameters:
3922 + dm          - The original mesh
3923 . hasLagrange - The mesh has Lagrange unknowns in the cohesive cells
3924 . label       - A label name, or `NULL`
3925 - value       - A label value
3926 
3927   Output Parameter:
3928 . subdm - The surface mesh
3929 
3930   Level: developer
3931 
3932   Note:
3933   This function produces a `DMLabel` mapping original points in the submesh to their depth. This can be obtained using `DMPlexGetSubpointMap()`.
3934 
3935 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetSubpointMap()`, `DMPlexCreateSubmesh()`
3936 @*/
3937 PetscErrorCode DMPlexCreateCohesiveSubmesh(DM dm, PetscBool hasLagrange, const char label[], PetscInt value, DM *subdm)
3938 {
3939   PetscInt dim, cdim, depth;
3940 
3941   PetscFunctionBegin;
3942   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3943   PetscAssertPointer(subdm, 5);
3944   PetscCall(DMGetDimension(dm, &dim));
3945   PetscCall(DMPlexGetDepth(dm, &depth));
3946   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), subdm));
3947   PetscCall(DMSetType(*subdm, DMPLEX));
3948   PetscCall(DMSetDimension(*subdm, dim - 1));
3949   PetscCall(DMGetCoordinateDim(dm, &cdim));
3950   PetscCall(DMSetCoordinateDim(*subdm, cdim));
3951   if (depth == dim) {
3952     PetscCall(DMPlexCreateCohesiveSubmesh_Interpolated(dm, label, value, *subdm));
3953   } else {
3954     PetscCall(DMPlexCreateCohesiveSubmesh_Uninterpolated(dm, hasLagrange, label, value, *subdm));
3955   }
3956   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_TRUE, *subdm));
3957   PetscFunctionReturn(PETSC_SUCCESS);
3958 }
3959 
3960 /*@
3961   DMPlexReorderCohesiveSupports - Ensure that face supports for cohesive end caps are ordered
3962 
3963   Not Collective
3964 
3965   Input Parameter:
3966 . dm - The `DM` containing cohesive cells
3967 
3968   Level: developer
3969 
3970   Note:
3971   For the negative size (first) face, the cohesive cell should be first in the support, and for
3972   the positive side (second) face, the cohesive cell should be second in the support.
3973 
3974 .seealso: `DMPlexConstructCohesiveCells()`, `DMPlexCreateCohesiveSubmesh()`
3975 @*/
3976 PetscErrorCode DMPlexReorderCohesiveSupports(DM dm)
3977 {
3978   PetscInt dim, cStart, cEnd;
3979 
3980   PetscFunctionBegin;
3981   PetscCall(DMGetDimension(dm, &dim));
3982   PetscCall(DMPlexGetTensorPrismBounds_Internal(dm, dim, &cStart, &cEnd));
3983   for (PetscInt c = cStart; c < cEnd; ++c) {
3984     const PetscInt *cone;
3985     PetscInt        coneSize;
3986 
3987     PetscCall(DMPlexGetConeSize(dm, c, &coneSize));
3988     PetscCall(DMPlexGetCone(dm, c, &cone));
3989     PetscCheck(coneSize >= 2, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Hybrid cell %" PetscInt_FMT " cone size %" PetscInt_FMT " must be >= 2", c, coneSize);
3990     for (PetscInt s = 0; s < 2; ++s) {
3991       const PetscInt *supp;
3992       PetscInt        suppSize, newsupp[2];
3993 
3994       PetscCall(DMPlexGetSupportSize(dm, cone[s], &suppSize));
3995       PetscCall(DMPlexGetSupport(dm, cone[s], &supp));
3996       if (suppSize == 2) {
3997         /* Reorder hybrid end cap support */
3998         if (supp[s] == c) {
3999           newsupp[0] = supp[1];
4000           newsupp[1] = supp[0];
4001         } else {
4002           newsupp[0] = supp[0];
4003           newsupp[1] = supp[1];
4004         }
4005         PetscCheck(newsupp[1 - s] == c, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Hybrid end cap %" PetscInt_FMT " support entry %" PetscInt_FMT " != %" PetscInt_FMT " hybrid cell", cone[s], newsupp[1 - s], c);
4006         PetscCall(DMPlexSetSupport(dm, cone[s], newsupp));
4007       }
4008     }
4009   }
4010   PetscFunctionReturn(PETSC_SUCCESS);
4011 }
4012 
4013 /*@
4014   DMPlexFilter - Extract a subset of mesh cells defined by a label as a separate mesh
4015 
4016   Input Parameters:
4017 + dm        - The original mesh
4018 . cellLabel - The `DMLabel` marking cells contained in the new mesh
4019 - value     - The label value to use
4020 
4021   Output Parameter:
4022 . subdm - The new mesh
4023 
4024   Level: developer
4025 
4026   Note:
4027   This function produces a `DMLabel` mapping original points in the submesh to their depth. This can be obtained using `DMPlexGetSubpointMap()`.
4028 
4029 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetSubpointMap()`, `DMGetLabel()`, `DMLabelSetValue()`, `DMPlexCreateSubmesh()`
4030 @*/
4031 PetscErrorCode DMPlexFilter(DM dm, DMLabel cellLabel, PetscInt value, DM *subdm)
4032 {
4033   PetscInt dim, overlap;
4034 
4035   PetscFunctionBegin;
4036   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4037   PetscAssertPointer(subdm, 4);
4038   PetscCall(DMGetDimension(dm, &dim));
4039   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), subdm));
4040   PetscCall(DMSetType(*subdm, DMPLEX));
4041   /* Extract submesh in place, could be empty on some procs, could have inconsistency if procs do not both extract a shared cell */
4042   PetscCall(DMPlexCreateSubmeshGeneric_Interpolated(dm, cellLabel, value, PETSC_FALSE, PETSC_FALSE, 0, *subdm));
4043   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_TRUE, *subdm));
4044   // It is possible to obtain a surface mesh where some faces are in SF
4045   //   We should either mark the mesh as having an overlap, or delete these from the SF
4046   PetscCall(DMPlexGetOverlap(dm, &overlap));
4047   if (!overlap) {
4048     PetscSF         sf;
4049     const PetscInt *leaves;
4050     PetscInt        cStart, cEnd, Nl;
4051     PetscBool       hasSubcell = PETSC_FALSE, ghasSubcell;
4052 
4053     PetscCall(DMPlexGetHeightStratum(*subdm, 0, &cStart, &cEnd));
4054     PetscCall(DMGetPointSF(*subdm, &sf));
4055     PetscCall(PetscSFGetGraph(sf, NULL, &Nl, &leaves, NULL));
4056     for (PetscInt l = 0; l < Nl; ++l) {
4057       const PetscInt point = leaves ? leaves[l] : l;
4058 
4059       if (point >= cStart && point < cEnd) {
4060         hasSubcell = PETSC_TRUE;
4061         break;
4062       }
4063     }
4064     PetscCall(MPIU_Allreduce(&hasSubcell, &ghasSubcell, 1, MPIU_BOOL, MPI_LOR, PetscObjectComm((PetscObject)dm)));
4065     if (ghasSubcell) PetscCall(DMPlexSetOverlap(*subdm, NULL, 1));
4066   }
4067   PetscFunctionReturn(PETSC_SUCCESS);
4068 }
4069 
4070 /*@
4071   DMPlexGetSubpointMap - Returns a `DMLabel` with point dimension as values
4072 
4073   Input Parameter:
4074 . dm - The submesh `DM`
4075 
4076   Output Parameter:
4077 . subpointMap - The `DMLabel` of all the points from the original mesh in this submesh, or `NULL` if this is not a submesh
4078 
4079   Level: developer
4080 
4081 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateSubmesh()`, `DMPlexGetSubpointIS()`
4082 @*/
4083 PetscErrorCode DMPlexGetSubpointMap(DM dm, DMLabel *subpointMap)
4084 {
4085   PetscFunctionBegin;
4086   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4087   PetscAssertPointer(subpointMap, 2);
4088   *subpointMap = ((DM_Plex *)dm->data)->subpointMap;
4089   PetscFunctionReturn(PETSC_SUCCESS);
4090 }
4091 
4092 /*@
4093   DMPlexSetSubpointMap - Sets the `DMLabel` with point dimension as values
4094 
4095   Input Parameters:
4096 + dm          - The submesh `DM`
4097 - subpointMap - The `DMLabel` of all the points from the original mesh in this submesh
4098 
4099   Level: developer
4100 
4101   Note:
4102   Should normally not be called by the user, since it is set in `DMPlexCreateSubmesh()`
4103 
4104 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateSubmesh()`, `DMPlexGetSubpointIS()`
4105 @*/
4106 PetscErrorCode DMPlexSetSubpointMap(DM dm, DMLabel subpointMap)
4107 {
4108   DM_Plex *mesh = (DM_Plex *)dm->data;
4109   DMLabel  tmp;
4110 
4111   PetscFunctionBegin;
4112   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4113   tmp               = mesh->subpointMap;
4114   mesh->subpointMap = subpointMap;
4115   PetscCall(PetscObjectReference((PetscObject)mesh->subpointMap));
4116   PetscCall(DMLabelDestroy(&tmp));
4117   PetscFunctionReturn(PETSC_SUCCESS);
4118 }
4119 
4120 static PetscErrorCode DMPlexCreateSubpointIS_Internal(DM dm, IS *subpointIS)
4121 {
4122   DMLabel  spmap;
4123   PetscInt depth, d;
4124 
4125   PetscFunctionBegin;
4126   PetscCall(DMPlexGetSubpointMap(dm, &spmap));
4127   PetscCall(DMPlexGetDepth(dm, &depth));
4128   if (spmap && depth >= 0) {
4129     DM_Plex  *mesh = (DM_Plex *)dm->data;
4130     PetscInt *points, *depths;
4131     PetscInt  pStart, pEnd, p, off;
4132 
4133     PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
4134     PetscCheck(!pStart, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Submeshes must start the point numbering at 0, not %" PetscInt_FMT, pStart);
4135     PetscCall(PetscMalloc1(pEnd, &points));
4136     PetscCall(DMGetWorkArray(dm, depth + 1, MPIU_INT, &depths));
4137     depths[0] = depth;
4138     depths[1] = 0;
4139     for (d = 2; d <= depth; ++d) depths[d] = depth + 1 - d;
4140     for (d = 0, off = 0; d <= depth; ++d) {
4141       const PetscInt dep = depths[d];
4142       PetscInt       depStart, depEnd, n;
4143 
4144       PetscCall(DMPlexGetDepthStratum(dm, dep, &depStart, &depEnd));
4145       PetscCall(DMLabelGetStratumSize(spmap, dep, &n));
4146       if (((d < 2) && (depth > 1)) || (d == 1)) { /* Only check vertices and cells for now since the map is broken for others */
4147         PetscCheck(n == depEnd - depStart, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "The number of mapped submesh points %" PetscInt_FMT " at depth %" PetscInt_FMT " should be %" PetscInt_FMT, n, dep, depEnd - depStart);
4148       } else {
4149         if (!n) {
4150           if (d == 0) {
4151             /* Missing cells */
4152             for (p = 0; p < depEnd - depStart; ++p, ++off) points[off] = -1;
4153           } else {
4154             /* Missing faces */
4155             for (p = 0; p < depEnd - depStart; ++p, ++off) points[off] = PETSC_MAX_INT;
4156           }
4157         }
4158       }
4159       if (n) {
4160         IS              is;
4161         const PetscInt *opoints;
4162 
4163         PetscCall(DMLabelGetStratumIS(spmap, dep, &is));
4164         PetscCall(ISGetIndices(is, &opoints));
4165         for (p = 0; p < n; ++p, ++off) points[off] = opoints[p];
4166         PetscCall(ISRestoreIndices(is, &opoints));
4167         PetscCall(ISDestroy(&is));
4168       }
4169     }
4170     PetscCall(DMRestoreWorkArray(dm, depth + 1, MPIU_INT, &depths));
4171     PetscCheck(off == pEnd, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "The number of mapped submesh points %" PetscInt_FMT " should be %" PetscInt_FMT, off, pEnd);
4172     PetscCall(ISCreateGeneral(PETSC_COMM_SELF, pEnd, points, PETSC_OWN_POINTER, subpointIS));
4173     PetscCall(PetscObjectStateGet((PetscObject)spmap, &mesh->subpointState));
4174   }
4175   PetscFunctionReturn(PETSC_SUCCESS);
4176 }
4177 
4178 /*@
4179   DMPlexGetSubpointIS - Returns an `IS` covering the entire subdm chart with the original points as data
4180 
4181   Input Parameter:
4182 . dm - The submesh `DM`
4183 
4184   Output Parameter:
4185 . subpointIS - The `IS` of all the points from the original mesh in this submesh, or `NULL` if this is not a submesh
4186 
4187   Level: developer
4188 
4189   Note:
4190   This `IS` is guaranteed to be sorted by the construction of the submesh
4191 
4192 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateSubmesh()`, `DMPlexGetSubpointMap()`
4193 @*/
4194 PetscErrorCode DMPlexGetSubpointIS(DM dm, IS *subpointIS)
4195 {
4196   DM_Plex         *mesh = (DM_Plex *)dm->data;
4197   DMLabel          spmap;
4198   PetscObjectState state;
4199 
4200   PetscFunctionBegin;
4201   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4202   PetscAssertPointer(subpointIS, 2);
4203   PetscCall(DMPlexGetSubpointMap(dm, &spmap));
4204   PetscCall(PetscObjectStateGet((PetscObject)spmap, &state));
4205   if (state != mesh->subpointState || !mesh->subpointIS) PetscCall(DMPlexCreateSubpointIS_Internal(dm, &mesh->subpointIS));
4206   *subpointIS = mesh->subpointIS;
4207   PetscFunctionReturn(PETSC_SUCCESS);
4208 }
4209 
4210 /*@
4211   DMGetEnclosureRelation - Get the relationship between `dmA` and `dmB`
4212 
4213   Input Parameters:
4214 + dmA - The first `DM`
4215 - dmB - The second `DM`
4216 
4217   Output Parameter:
4218 . rel - The relation of `dmA` to `dmB`
4219 
4220   Level: intermediate
4221 
4222 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMGetEnclosurePoint()`
4223 @*/
4224 PetscErrorCode DMGetEnclosureRelation(DM dmA, DM dmB, DMEnclosureType *rel)
4225 {
4226   DM       plexA, plexB, sdm;
4227   DMLabel  spmap;
4228   PetscInt pStartA, pEndA, pStartB, pEndB, NpA, NpB;
4229 
4230   PetscFunctionBegin;
4231   PetscAssertPointer(rel, 3);
4232   *rel = DM_ENC_NONE;
4233   if (!dmA || !dmB) PetscFunctionReturn(PETSC_SUCCESS);
4234   PetscValidHeaderSpecific(dmA, DM_CLASSID, 1);
4235   PetscValidHeaderSpecific(dmB, DM_CLASSID, 2);
4236   if (dmA == dmB) {
4237     *rel = DM_ENC_EQUALITY;
4238     PetscFunctionReturn(PETSC_SUCCESS);
4239   }
4240   PetscCall(DMConvert(dmA, DMPLEX, &plexA));
4241   PetscCall(DMConvert(dmB, DMPLEX, &plexB));
4242   PetscCall(DMPlexGetChart(plexA, &pStartA, &pEndA));
4243   PetscCall(DMPlexGetChart(plexB, &pStartB, &pEndB));
4244   /* Assumption 1: subDMs have smaller charts than the DMs that they originate from
4245     - The degenerate case of a subdomain which includes all of the domain on some process can be treated as equality */
4246   if ((pStartA == pStartB) && (pEndA == pEndB)) {
4247     *rel = DM_ENC_EQUALITY;
4248     goto end;
4249   }
4250   NpA = pEndA - pStartA;
4251   NpB = pEndB - pStartB;
4252   if (NpA == NpB) goto end;
4253   sdm = NpA > NpB ? plexB : plexA; /* The other is the original, enclosing dm */
4254   PetscCall(DMPlexGetSubpointMap(sdm, &spmap));
4255   if (!spmap) goto end;
4256   /* TODO Check the space mapped to by subpointMap is same size as dm */
4257   if (NpA > NpB) {
4258     *rel = DM_ENC_SUPERMESH;
4259   } else {
4260     *rel = DM_ENC_SUBMESH;
4261   }
4262 end:
4263   PetscCall(DMDestroy(&plexA));
4264   PetscCall(DMDestroy(&plexB));
4265   PetscFunctionReturn(PETSC_SUCCESS);
4266 }
4267 
4268 /*@
4269   DMGetEnclosurePoint - Get the point `pA` in `dmA` which corresponds to the point `pB` in `dmB`
4270 
4271   Input Parameters:
4272 + dmA   - The first `DM`
4273 . dmB   - The second `DM`
4274 . etype - The type of enclosure relation that `dmA` has to `dmB`
4275 - pB    - A point of `dmB`
4276 
4277   Output Parameter:
4278 . pA - The corresponding point of `dmA`
4279 
4280   Level: intermediate
4281 
4282 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMGetEnclosureRelation()`
4283 @*/
4284 PetscErrorCode DMGetEnclosurePoint(DM dmA, DM dmB, DMEnclosureType etype, PetscInt pB, PetscInt *pA)
4285 {
4286   DM              sdm;
4287   IS              subpointIS;
4288   const PetscInt *subpoints;
4289   PetscInt        numSubpoints;
4290 
4291   PetscFunctionBegin;
4292   /* TODO Cache the IS, making it look like an index */
4293   switch (etype) {
4294   case DM_ENC_SUPERMESH:
4295     sdm = dmB;
4296     PetscCall(DMPlexGetSubpointIS(sdm, &subpointIS));
4297     PetscCall(ISGetIndices(subpointIS, &subpoints));
4298     *pA = subpoints[pB];
4299     PetscCall(ISRestoreIndices(subpointIS, &subpoints));
4300     break;
4301   case DM_ENC_SUBMESH:
4302     sdm = dmA;
4303     PetscCall(DMPlexGetSubpointIS(sdm, &subpointIS));
4304     PetscCall(ISGetLocalSize(subpointIS, &numSubpoints));
4305     PetscCall(ISGetIndices(subpointIS, &subpoints));
4306     PetscCall(PetscFindInt(pB, numSubpoints, subpoints, pA));
4307     if (*pA < 0) {
4308       PetscCall(DMViewFromOptions(dmA, NULL, "-dm_enc_A_view"));
4309       PetscCall(DMViewFromOptions(dmB, NULL, "-dm_enc_B_view"));
4310       SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Point %" PetscInt_FMT " not found in submesh", pB);
4311     }
4312     PetscCall(ISRestoreIndices(subpointIS, &subpoints));
4313     break;
4314   case DM_ENC_EQUALITY:
4315   case DM_ENC_NONE:
4316     *pA = pB;
4317     break;
4318   case DM_ENC_UNKNOWN: {
4319     DMEnclosureType enc;
4320 
4321     PetscCall(DMGetEnclosureRelation(dmA, dmB, &enc));
4322     PetscCall(DMGetEnclosurePoint(dmA, dmB, enc, pB, pA));
4323   } break;
4324   default:
4325     SETERRQ(PetscObjectComm((PetscObject)dmA), PETSC_ERR_ARG_OUTOFRANGE, "Invalid enclosure type %d", (int)etype);
4326   }
4327   PetscFunctionReturn(PETSC_SUCCESS);
4328 }
4329