xref: /petsc/src/dm/impls/plex/plexsubmesh.c (revision 34c645fd3b0199e05bec2fcc32d3597bfeb7f4f2)
1 #include <petsc/private/dmpleximpl.h>  /*I      "petscdmplex.h"    I*/
2 #include <petsc/private/dmlabelimpl.h> /*I      "petscdmlabel.h"   I*/
3 #include <petscsf.h>
4 
5 static PetscErrorCode DMPlexCellIsHybrid_Internal(DM dm, PetscInt p, PetscBool *isHybrid)
6 {
7   DMPolytopeType ct;
8 
9   PetscFunctionBegin;
10   PetscCall(DMPlexGetCellType(dm, p, &ct));
11   switch (ct) {
12   case DM_POLYTOPE_POINT_PRISM_TENSOR:
13   case DM_POLYTOPE_SEG_PRISM_TENSOR:
14   case DM_POLYTOPE_TRI_PRISM_TENSOR:
15   case DM_POLYTOPE_QUAD_PRISM_TENSOR:
16     *isHybrid = PETSC_TRUE;
17     break;
18   default:
19     *isHybrid = PETSC_FALSE;
20     break;
21   }
22   PetscFunctionReturn(PETSC_SUCCESS);
23 }
24 
25 static PetscErrorCode DMPlexGetTensorPrismBounds_Internal(DM dm, PetscInt dim, PetscInt *cStart, PetscInt *cEnd)
26 {
27   DMLabel ctLabel;
28 
29   PetscFunctionBegin;
30   if (cStart) *cStart = -1;
31   if (cEnd) *cEnd = -1;
32   PetscCall(DMPlexGetCellTypeLabel(dm, &ctLabel));
33   switch (dim) {
34   case 1:
35     PetscCall(DMLabelGetStratumBounds(ctLabel, DM_POLYTOPE_POINT_PRISM_TENSOR, cStart, cEnd));
36     break;
37   case 2:
38     PetscCall(DMLabelGetStratumBounds(ctLabel, DM_POLYTOPE_SEG_PRISM_TENSOR, cStart, cEnd));
39     break;
40   case 3:
41     PetscCall(DMLabelGetStratumBounds(ctLabel, DM_POLYTOPE_TRI_PRISM_TENSOR, cStart, cEnd));
42     if (*cStart < 0) PetscCall(DMLabelGetStratumBounds(ctLabel, DM_POLYTOPE_QUAD_PRISM_TENSOR, cStart, cEnd));
43     break;
44   default:
45     PetscFunctionReturn(PETSC_SUCCESS);
46   }
47   PetscFunctionReturn(PETSC_SUCCESS);
48 }
49 
50 PetscErrorCode DMPlexMarkBoundaryFaces_Internal(DM dm, PetscInt val, PetscInt cellHeight, DMLabel label, PetscBool missing_only)
51 {
52   PetscInt           depth, pStart, pEnd, fStart, fEnd, f, supportSize, nroots = -1, nleaves = -1, defval;
53   PetscSF            sf;
54   const PetscSFNode *iremote = NULL;
55   const PetscInt    *ilocal  = NULL;
56   PetscInt          *leafData;
57 
58   PetscFunctionBegin;
59   PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
60   PetscCall(DMPlexGetDepth(dm, &depth));
61   if (depth >= cellHeight + 1) {
62     PetscCall(DMPlexGetHeightStratum(dm, cellHeight + 1, &fStart, &fEnd));
63   } else {
64     /* Note DMPlexGetHeightStratum() returns fStart, fEnd = pStart, pEnd */
65     /* if height > depth, which is not what we want here.                */
66     fStart = 0;
67     fEnd   = 0;
68   }
69   PetscCall(DMLabelGetDefaultValue(label, &defval));
70   PetscCall(PetscCalloc1(pEnd - pStart, &leafData));
71   leafData = PetscSafePointerPlusOffset(leafData, -pStart);
72   PetscCall(DMGetPointSF(dm, &sf));
73   if (sf) PetscCall(PetscSFGetGraph(sf, &nroots, &nleaves, &ilocal, &iremote));
74   if (sf && nroots >= 0) {
75     PetscInt        cStart, cEnd, c, i;
76     PetscInt       *rootData, *rootData1, *cellOwners, hasTwoSupportCells = -2;
77     const PetscInt *support;
78     PetscMPIInt     rank;
79 
80     PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
81     PetscCall(DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd));
82     PetscCall(PetscCalloc3(pEnd - pStart, &rootData, pEnd - pStart, &rootData1, cEnd - cStart, &cellOwners));
83     rootData -= pStart;
84     rootData1 -= pStart;
85     for (c = cStart; c < cEnd; ++c) cellOwners[c - cStart] = (PetscInt)rank;
86     for (i = 0; i < nleaves; ++i) {
87       c = ilocal ? ilocal[i] : i;
88       if (c >= cStart && c < cEnd) cellOwners[c - cStart] = iremote[i].rank;
89     }
90     for (f = fStart; f < fEnd; ++f) {
91       PetscCall(DMPlexGetSupportSize(dm, f, &supportSize));
92       if (supportSize == 1) {
93         PetscCall(DMPlexGetSupport(dm, f, &support));
94         leafData[f] = cellOwners[support[0] - cStart];
95       } else {
96         /* TODO: When using DMForest, we could have a parent facet (a coarse facet)     */
97         /*       supportSize of which alone does not tell us if it is an interior       */
98         /*       facet or an exterior facet. Those facets can be identified by checking */
99         /*       if they are in the parent tree. We should probably skip those parent   */
100         /*       facets here, which will allow for including the following check, and   */
101         /*       see if they are exterior facets or not afterwards by checking if the   */
102         /*       children are exterior or not.                                          */
103         /* PetscCheck(supportSize == 2, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid facet support size (%" PetscInt_FMT ") on facet (%" PetscInt_FMT ")", supportSize, f); */
104         leafData[f] = hasTwoSupportCells; /* some negative PetscInt */
105       }
106       rootData[f]  = leafData[f];
107       rootData1[f] = leafData[f];
108     }
109     PetscCall(PetscSFReduceBegin(sf, MPIU_INT, leafData, rootData, MPI_MIN));
110     PetscCall(PetscSFReduceBegin(sf, MPIU_INT, leafData, rootData1, MPI_MAX));
111     PetscCall(PetscSFReduceEnd(sf, MPIU_INT, leafData, rootData, MPI_MIN));
112     PetscCall(PetscSFReduceEnd(sf, MPIU_INT, leafData, rootData1, MPI_MAX));
113     for (f = fStart; f < fEnd; ++f) {
114       /* Store global support size of f.                                                    */
115       /* Facet f is an interior facet if and only if one of the following two is satisfied: */
116       /* 1. supportSize is 2 on some rank.                                                  */
117       /* 2. supportSize is 1 on any rank that can see f, but f is on a partition boundary;  */
118       /*    i.e., rootData[f] < rootData1[f].                                               */
119       rootData[f] = (rootData[f] == hasTwoSupportCells || (rootData[f] < rootData1[f])) ? 2 : 1;
120       leafData[f] = rootData[f];
121     }
122     PetscCall(PetscSFBcastBegin(sf, MPIU_INT, rootData, leafData, MPI_REPLACE));
123     PetscCall(PetscSFBcastEnd(sf, MPIU_INT, rootData, leafData, MPI_REPLACE));
124     rootData += pStart;
125     rootData1 += pStart;
126     PetscCall(PetscFree3(rootData, rootData1, cellOwners));
127   } else {
128     for (f = fStart; f < fEnd; ++f) {
129       PetscCall(DMPlexGetSupportSize(dm, f, &supportSize));
130       leafData[f] = supportSize;
131     }
132   }
133   for (f = fStart; f < fEnd; ++f) {
134     if (leafData[f] == 1) {
135       if (val < 0) {
136         PetscInt *closure = NULL;
137         PetscInt  clSize, cl, cval;
138 
139         PetscAssert(!missing_only, PETSC_COMM_SELF, PETSC_ERR_SUP, "Not implemented");
140         PetscCall(DMPlexGetTransitiveClosure(dm, f, PETSC_TRUE, &clSize, &closure));
141         for (cl = 0; cl < clSize * 2; cl += 2) {
142           PetscCall(DMLabelGetValue(label, closure[cl], &cval));
143           if (cval < 0) continue;
144           PetscCall(DMLabelSetValue(label, f, cval));
145           break;
146         }
147         if (cl == clSize * 2) PetscCall(DMLabelSetValue(label, f, 1));
148         PetscCall(DMPlexRestoreTransitiveClosure(dm, f, PETSC_TRUE, &clSize, &closure));
149       } else {
150         if (missing_only) {
151           PetscInt fval;
152           PetscCall(DMLabelGetValue(label, f, &fval));
153           if (fval != defval) PetscCall(DMLabelClearValue(label, f, fval));
154           else PetscCall(DMLabelSetValue(label, f, val));
155         } else {
156           PetscCall(DMLabelSetValue(label, f, val));
157         }
158       }
159     } else {
160       /* TODO: See the above comment on DMForest */
161       /* PetscCheck(leafData[f] == 2, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid facet support size (%" PetscInt_FMT ") on facet (%" PetscInt_FMT ")", leafData[f], f); */
162     }
163   }
164   leafData = PetscSafePointerPlusOffset(leafData, pStart);
165   PetscCall(PetscFree(leafData));
166   PetscFunctionReturn(PETSC_SUCCESS);
167 }
168 
169 /*@
170   DMPlexMarkBoundaryFaces - Mark all faces on the boundary
171 
172   Collective
173 
174   Input Parameters:
175 + dm  - The original `DM`
176 - val - The marker value, or `PETSC_DETERMINE` to use some value in the closure (or 1 if none are found)
177 
178   Output Parameter:
179 . label - The `DMLabel` marking boundary faces with the given value
180 
181   Level: developer
182 
183   Note:
184   This function will use the point `PetscSF` from the input `DM` and the ownership of the support cells to exclude points on the partition boundary from being marked. If you also wish to mark the partition boundary, you can use `DMSetPointSF()` to temporarily set it to `NULL`, and then reset it to the original object after the call.
185 
186   In DMForest there can be facets support sizes of which alone can not determine whether they are on the boundary. Currently, this function is not guaranteed to produce the correct result in such case.
187 
188 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMLabelCreate()`, `DMCreateLabel()`
189 @*/
190 PetscErrorCode DMPlexMarkBoundaryFaces(DM dm, PetscInt val, DMLabel label)
191 {
192   DM                     plex;
193   DMPlexInterpolatedFlag flg;
194 
195   PetscFunctionBegin;
196   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
197   PetscCall(DMConvert(dm, DMPLEX, &plex));
198   PetscCall(DMPlexIsInterpolated(plex, &flg));
199   PetscCheck(flg == DMPLEX_INTERPOLATED_FULL, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM is not fully interpolated on this rank");
200   PetscCall(DMPlexMarkBoundaryFaces_Internal(plex, val, 0, label, PETSC_FALSE));
201   PetscCall(DMDestroy(&plex));
202   PetscFunctionReturn(PETSC_SUCCESS);
203 }
204 
205 static PetscErrorCode DMPlexLabelComplete_Internal(DM dm, DMLabel label, PetscBool completeCells)
206 {
207   IS              valueIS;
208   PetscSF         sfPoint;
209   const PetscInt *values;
210   PetscInt        numValues, v, cStart, cEnd, nroots;
211 
212   PetscFunctionBegin;
213   PetscCall(DMLabelGetNumValues(label, &numValues));
214   PetscCall(DMLabelGetValueIS(label, &valueIS));
215   PetscCall(DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd));
216   PetscCall(ISGetIndices(valueIS, &values));
217   for (v = 0; v < numValues; ++v) {
218     IS              pointIS;
219     const PetscInt *points;
220     PetscInt        numPoints, p;
221 
222     PetscCall(DMLabelGetStratumSize(label, values[v], &numPoints));
223     PetscCall(DMLabelGetStratumIS(label, values[v], &pointIS));
224     PetscCall(ISGetIndices(pointIS, &points));
225     for (p = 0; p < numPoints; ++p) {
226       PetscInt  q       = points[p];
227       PetscInt *closure = NULL;
228       PetscInt  closureSize, c;
229 
230       if (cStart <= q && q < cEnd && !completeCells) { /* skip cells */
231         continue;
232       }
233       PetscCall(DMPlexGetTransitiveClosure(dm, q, PETSC_TRUE, &closureSize, &closure));
234       for (c = 0; c < closureSize * 2; c += 2) PetscCall(DMLabelSetValue(label, closure[c], values[v]));
235       PetscCall(DMPlexRestoreTransitiveClosure(dm, q, PETSC_TRUE, &closureSize, &closure));
236     }
237     PetscCall(ISRestoreIndices(pointIS, &points));
238     PetscCall(ISDestroy(&pointIS));
239   }
240   PetscCall(ISRestoreIndices(valueIS, &values));
241   PetscCall(ISDestroy(&valueIS));
242   PetscCall(DMGetPointSF(dm, &sfPoint));
243   PetscCall(PetscSFGetGraph(sfPoint, &nroots, NULL, NULL, NULL));
244   if (nroots >= 0) {
245     DMLabel         lblRoots, lblLeaves;
246     IS              valueIS, pointIS;
247     const PetscInt *values;
248     PetscInt        numValues, v;
249 
250     /* Pull point contributions from remote leaves into local roots */
251     PetscCall(DMLabelGather(label, sfPoint, &lblLeaves));
252     PetscCall(DMLabelGetValueIS(lblLeaves, &valueIS));
253     PetscCall(ISGetLocalSize(valueIS, &numValues));
254     PetscCall(ISGetIndices(valueIS, &values));
255     for (v = 0; v < numValues; ++v) {
256       const PetscInt value = values[v];
257 
258       PetscCall(DMLabelGetStratumIS(lblLeaves, value, &pointIS));
259       PetscCall(DMLabelInsertIS(label, pointIS, value));
260       PetscCall(ISDestroy(&pointIS));
261     }
262     PetscCall(ISRestoreIndices(valueIS, &values));
263     PetscCall(ISDestroy(&valueIS));
264     PetscCall(DMLabelDestroy(&lblLeaves));
265     /* Push point contributions from roots into remote leaves */
266     PetscCall(DMLabelDistribute(label, sfPoint, &lblRoots));
267     PetscCall(DMLabelGetValueIS(lblRoots, &valueIS));
268     PetscCall(ISGetLocalSize(valueIS, &numValues));
269     PetscCall(ISGetIndices(valueIS, &values));
270     for (v = 0; v < numValues; ++v) {
271       const PetscInt value = values[v];
272 
273       PetscCall(DMLabelGetStratumIS(lblRoots, value, &pointIS));
274       PetscCall(DMLabelInsertIS(label, pointIS, value));
275       PetscCall(ISDestroy(&pointIS));
276     }
277     PetscCall(ISRestoreIndices(valueIS, &values));
278     PetscCall(ISDestroy(&valueIS));
279     PetscCall(DMLabelDestroy(&lblRoots));
280   }
281   PetscFunctionReturn(PETSC_SUCCESS);
282 }
283 
284 /*@
285   DMPlexLabelComplete - Starting with a label marking points on a surface, we add the transitive closure to the surface
286 
287   Input Parameters:
288 + dm    - The `DM`
289 - label - A `DMLabel` marking the surface points
290 
291   Output Parameter:
292 . label - A `DMLabel` marking all surface points in the transitive closure
293 
294   Level: developer
295 
296 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexLabelCohesiveComplete()`
297 @*/
298 PetscErrorCode DMPlexLabelComplete(DM dm, DMLabel label)
299 {
300   PetscFunctionBegin;
301   PetscCall(DMPlexLabelComplete_Internal(dm, label, PETSC_TRUE));
302   PetscFunctionReturn(PETSC_SUCCESS);
303 }
304 
305 /*@
306   DMPlexLabelAddCells - Starting with a label marking points on a surface, we add a cell for each point
307 
308   Input Parameters:
309 + dm    - The `DM`
310 - label - A `DMLabel` marking the surface points
311 
312   Output Parameter:
313 . label - A `DMLabel` incorporating cells
314 
315   Level: developer
316 
317   Note:
318   The cells allow FEM boundary conditions to be applied using the cell geometry
319 
320 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexLabelAddFaceCells()`, `DMPlexLabelComplete()`, `DMPlexLabelCohesiveComplete()`
321 @*/
322 PetscErrorCode DMPlexLabelAddCells(DM dm, DMLabel label)
323 {
324   IS              valueIS;
325   const PetscInt *values;
326   PetscInt        numValues, v, csStart, csEnd, chStart, chEnd;
327 
328   PetscFunctionBegin;
329   PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &csStart, &csEnd));
330   PetscCall(DMPlexGetHeightStratum(dm, 0, &chStart, &chEnd));
331   PetscCall(DMLabelGetNumValues(label, &numValues));
332   PetscCall(DMLabelGetValueIS(label, &valueIS));
333   PetscCall(ISGetIndices(valueIS, &values));
334   for (v = 0; v < numValues; ++v) {
335     IS              pointIS;
336     const PetscInt *points;
337     PetscInt        numPoints, p;
338 
339     PetscCall(DMLabelGetStratumSize(label, values[v], &numPoints));
340     PetscCall(DMLabelGetStratumIS(label, values[v], &pointIS));
341     PetscCall(ISGetIndices(pointIS, &points));
342     for (p = 0; p < numPoints; ++p) {
343       const PetscInt point   = points[p];
344       PetscInt      *closure = NULL;
345       PetscInt       closureSize, cl, h, cStart, cEnd;
346       DMPolytopeType ct;
347 
348       // If the point is a hybrid, allow hybrid cells
349       PetscCall(DMPlexGetCellType(dm, point, &ct));
350       PetscCall(DMPlexGetPointHeight(dm, point, &h));
351       if (DMPolytopeTypeIsHybrid(ct)) {
352         cStart = chStart;
353         cEnd   = chEnd;
354       } else {
355         cStart = csStart;
356         cEnd   = csEnd;
357       }
358 
359       PetscCall(DMPlexGetTransitiveClosure(dm, points[p], PETSC_FALSE, &closureSize, &closure));
360       for (cl = closureSize - 1; cl > 0; --cl) {
361         const PetscInt cell = closure[cl * 2];
362         if ((cell >= cStart) && (cell < cEnd)) {
363           PetscCall(DMLabelSetValue(label, cell, values[v]));
364           break;
365         }
366       }
367       PetscCall(DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_FALSE, &closureSize, &closure));
368     }
369     PetscCall(ISRestoreIndices(pointIS, &points));
370     PetscCall(ISDestroy(&pointIS));
371   }
372   PetscCall(ISRestoreIndices(valueIS, &values));
373   PetscCall(ISDestroy(&valueIS));
374   PetscFunctionReturn(PETSC_SUCCESS);
375 }
376 
377 /*@
378   DMPlexLabelAddFaceCells - Starting with a label marking faces on a surface, we add a cell for each face
379 
380   Input Parameters:
381 + dm    - The `DM`
382 - label - A `DMLabel` marking the surface points
383 
384   Output Parameter:
385 . label - A `DMLabel` incorporating cells
386 
387   Level: developer
388 
389   Note:
390   The cells allow FEM boundary conditions to be applied using the cell geometry
391 
392 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexLabelAddCells()`, `DMPlexLabelComplete()`, `DMPlexLabelCohesiveComplete()`
393 @*/
394 PetscErrorCode DMPlexLabelAddFaceCells(DM dm, DMLabel label)
395 {
396   IS              valueIS;
397   const PetscInt *values;
398   PetscInt        numValues, v, cStart, cEnd, fStart, fEnd;
399 
400   PetscFunctionBegin;
401   PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd));
402   PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
403   PetscCall(DMLabelGetNumValues(label, &numValues));
404   PetscCall(DMLabelGetValueIS(label, &valueIS));
405   PetscCall(ISGetIndices(valueIS, &values));
406   for (v = 0; v < numValues; ++v) {
407     IS              pointIS;
408     const PetscInt *points;
409     PetscInt        numPoints, p;
410 
411     PetscCall(DMLabelGetStratumSize(label, values[v], &numPoints));
412     PetscCall(DMLabelGetStratumIS(label, values[v], &pointIS));
413     PetscCall(ISGetIndices(pointIS, &points));
414     for (p = 0; p < numPoints; ++p) {
415       const PetscInt face    = points[p];
416       PetscInt      *closure = NULL;
417       PetscInt       closureSize, cl;
418 
419       if ((face < fStart) || (face >= fEnd)) continue;
420       PetscCall(DMPlexGetTransitiveClosure(dm, face, PETSC_FALSE, &closureSize, &closure));
421       for (cl = closureSize - 1; cl > 0; --cl) {
422         const PetscInt cell = closure[cl * 2];
423         if ((cell >= cStart) && (cell < cEnd)) {
424           PetscCall(DMLabelSetValue(label, cell, values[v]));
425           break;
426         }
427       }
428       PetscCall(DMPlexRestoreTransitiveClosure(dm, face, PETSC_FALSE, &closureSize, &closure));
429     }
430     PetscCall(ISRestoreIndices(pointIS, &points));
431     PetscCall(ISDestroy(&pointIS));
432   }
433   PetscCall(ISRestoreIndices(valueIS, &values));
434   PetscCall(ISDestroy(&valueIS));
435   PetscFunctionReturn(PETSC_SUCCESS);
436 }
437 
438 /*@
439   DMPlexLabelClearCells - Remove cells from a label
440 
441   Input Parameters:
442 + dm    - The `DM`
443 - label - A `DMLabel` marking surface points and their adjacent cells
444 
445   Output Parameter:
446 . label - A `DMLabel` without cells
447 
448   Level: developer
449 
450   Note:
451   This undoes `DMPlexLabelAddCells()` or `DMPlexLabelAddFaceCells()`
452 
453 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexLabelComplete()`, `DMPlexLabelCohesiveComplete()`, `DMPlexLabelAddCells()`
454 @*/
455 PetscErrorCode DMPlexLabelClearCells(DM dm, DMLabel label)
456 {
457   IS              valueIS;
458   const PetscInt *values;
459   PetscInt        numValues, v, cStart, cEnd;
460 
461   PetscFunctionBegin;
462   PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd));
463   PetscCall(DMLabelGetNumValues(label, &numValues));
464   PetscCall(DMLabelGetValueIS(label, &valueIS));
465   PetscCall(ISGetIndices(valueIS, &values));
466   for (v = 0; v < numValues; ++v) {
467     IS              pointIS;
468     const PetscInt *points;
469     PetscInt        numPoints, p;
470 
471     PetscCall(DMLabelGetStratumSize(label, values[v], &numPoints));
472     PetscCall(DMLabelGetStratumIS(label, values[v], &pointIS));
473     PetscCall(ISGetIndices(pointIS, &points));
474     for (p = 0; p < numPoints; ++p) {
475       PetscInt point = points[p];
476 
477       if (point >= cStart && point < cEnd) PetscCall(DMLabelClearValue(label, point, values[v]));
478     }
479     PetscCall(ISRestoreIndices(pointIS, &points));
480     PetscCall(ISDestroy(&pointIS));
481   }
482   PetscCall(ISRestoreIndices(valueIS, &values));
483   PetscCall(ISDestroy(&valueIS));
484   PetscFunctionReturn(PETSC_SUCCESS);
485 }
486 
487 /* take (oldEnd, added) pairs, ordered by height and convert them to (oldstart, newstart) pairs, ordered by ascending
488  * index (skipping first, which is (0,0)) */
489 static inline PetscErrorCode DMPlexShiftPointSetUp_Internal(PetscInt depth, PetscInt depthShift[])
490 {
491   PetscInt d, off = 0;
492 
493   PetscFunctionBegin;
494   /* sort by (oldend): yes this is an O(n^2) sort, we expect depth <= 3 */
495   for (d = 0; d < depth; d++) {
496     PetscInt firstd     = d;
497     PetscInt firstStart = depthShift[2 * d];
498     PetscInt e;
499 
500     for (e = d + 1; e <= depth; e++) {
501       if (depthShift[2 * e] < firstStart) {
502         firstd     = e;
503         firstStart = depthShift[2 * d];
504       }
505     }
506     if (firstd != d) {
507       PetscInt swap[2];
508 
509       e                     = firstd;
510       swap[0]               = depthShift[2 * d];
511       swap[1]               = depthShift[2 * d + 1];
512       depthShift[2 * d]     = depthShift[2 * e];
513       depthShift[2 * d + 1] = depthShift[2 * e + 1];
514       depthShift[2 * e]     = swap[0];
515       depthShift[2 * e + 1] = swap[1];
516     }
517   }
518   /* convert (oldstart, added) to (oldstart, newstart) */
519   for (d = 0; d <= depth; d++) {
520     off += depthShift[2 * d + 1];
521     depthShift[2 * d + 1] = depthShift[2 * d] + off;
522   }
523   PetscFunctionReturn(PETSC_SUCCESS);
524 }
525 
526 /* depthShift is a list of (old, new) pairs */
527 static inline PetscInt DMPlexShiftPoint_Internal(PetscInt p, PetscInt depth, PetscInt depthShift[])
528 {
529   PetscInt d;
530   PetscInt newOff = 0;
531 
532   for (d = 0; d <= depth; d++) {
533     if (p < depthShift[2 * d]) return p + newOff;
534     else newOff = depthShift[2 * d + 1] - depthShift[2 * d];
535   }
536   return p + newOff;
537 }
538 
539 /* depthShift is a list of (old, new) pairs */
540 static inline PetscInt DMPlexShiftPointInverse_Internal(PetscInt p, PetscInt depth, PetscInt depthShift[])
541 {
542   PetscInt d;
543   PetscInt newOff = 0;
544 
545   for (d = 0; d <= depth; d++) {
546     if (p < depthShift[2 * d + 1]) return p + newOff;
547     else newOff = depthShift[2 * d] - depthShift[2 * d + 1];
548   }
549   return p + newOff;
550 }
551 
552 static PetscErrorCode DMPlexShiftSizes_Internal(DM dm, PetscInt depthShift[], DM dmNew)
553 {
554   PetscInt depth = 0, d, pStart, pEnd, p;
555   DMLabel  depthLabel;
556 
557   PetscFunctionBegin;
558   PetscCall(DMPlexGetDepth(dm, &depth));
559   if (depth < 0) PetscFunctionReturn(PETSC_SUCCESS);
560   /* Step 1: Expand chart */
561   PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
562   pEnd = DMPlexShiftPoint_Internal(pEnd, depth, depthShift);
563   PetscCall(DMPlexSetChart(dmNew, pStart, pEnd));
564   PetscCall(DMCreateLabel(dmNew, "depth"));
565   PetscCall(DMPlexGetDepthLabel(dmNew, &depthLabel));
566   PetscCall(DMCreateLabel(dmNew, "celltype"));
567   /* Step 2: Set cone and support sizes */
568   for (d = 0; d <= depth; ++d) {
569     PetscInt pStartNew, pEndNew;
570     IS       pIS;
571 
572     PetscCall(DMPlexGetDepthStratum(dm, d, &pStart, &pEnd));
573     pStartNew = DMPlexShiftPoint_Internal(pStart, depth, depthShift);
574     pEndNew   = DMPlexShiftPoint_Internal(pEnd, depth, depthShift);
575     PetscCall(ISCreateStride(PETSC_COMM_SELF, pEndNew - pStartNew, pStartNew, 1, &pIS));
576     PetscCall(DMLabelSetStratumIS(depthLabel, d, pIS));
577     PetscCall(ISDestroy(&pIS));
578     for (p = pStart; p < pEnd; ++p) {
579       PetscInt       newp = DMPlexShiftPoint_Internal(p, depth, depthShift);
580       PetscInt       size;
581       DMPolytopeType ct;
582 
583       PetscCall(DMPlexGetConeSize(dm, p, &size));
584       PetscCall(DMPlexSetConeSize(dmNew, newp, size));
585       PetscCall(DMPlexGetSupportSize(dm, p, &size));
586       PetscCall(DMPlexSetSupportSize(dmNew, newp, size));
587       PetscCall(DMPlexGetCellType(dm, p, &ct));
588       PetscCall(DMPlexSetCellType(dmNew, newp, ct));
589     }
590   }
591   PetscFunctionReturn(PETSC_SUCCESS);
592 }
593 
594 static PetscErrorCode DMPlexShiftPoints_Internal(DM dm, PetscInt depthShift[], DM dmNew)
595 {
596   PetscInt *newpoints;
597   PetscInt  depth = 0, maxConeSize, maxSupportSize, maxConeSizeNew, maxSupportSizeNew, pStart, pEnd, p;
598 
599   PetscFunctionBegin;
600   PetscCall(DMPlexGetDepth(dm, &depth));
601   if (depth < 0) PetscFunctionReturn(PETSC_SUCCESS);
602   PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize));
603   PetscCall(DMPlexGetMaxSizes(dmNew, &maxConeSizeNew, &maxSupportSizeNew));
604   PetscCall(PetscMalloc1(PetscMax(PetscMax(maxConeSize, maxSupportSize), PetscMax(maxConeSizeNew, maxSupportSizeNew)), &newpoints));
605   /* Step 5: Set cones and supports */
606   PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
607   for (p = pStart; p < pEnd; ++p) {
608     const PetscInt *points = NULL, *orientations = NULL;
609     PetscInt        size, sizeNew, i, newp = DMPlexShiftPoint_Internal(p, depth, depthShift);
610 
611     PetscCall(DMPlexGetConeSize(dm, p, &size));
612     PetscCall(DMPlexGetCone(dm, p, &points));
613     PetscCall(DMPlexGetConeOrientation(dm, p, &orientations));
614     for (i = 0; i < size; ++i) newpoints[i] = DMPlexShiftPoint_Internal(points[i], depth, depthShift);
615     PetscCall(DMPlexSetCone(dmNew, newp, newpoints));
616     PetscCall(DMPlexSetConeOrientation(dmNew, newp, orientations));
617     PetscCall(DMPlexGetSupportSize(dm, p, &size));
618     PetscCall(DMPlexGetSupportSize(dmNew, newp, &sizeNew));
619     PetscCall(DMPlexGetSupport(dm, p, &points));
620     for (i = 0; i < size; ++i) newpoints[i] = DMPlexShiftPoint_Internal(points[i], depth, depthShift);
621     for (i = size; i < sizeNew; ++i) newpoints[i] = 0;
622     PetscCall(DMPlexSetSupport(dmNew, newp, newpoints));
623   }
624   PetscCall(PetscFree(newpoints));
625   PetscFunctionReturn(PETSC_SUCCESS);
626 }
627 
628 static PetscErrorCode DMPlexShiftCoordinates_Internal(DM dm, PetscInt depthShift[], DM dmNew)
629 {
630   PetscSection coordSection, newCoordSection;
631   Vec          coordinates, newCoordinates;
632   PetscScalar *coords, *newCoords;
633   PetscInt     coordSize, sStart, sEnd;
634   PetscInt     dim, depth = 0, cStart, cEnd, cStartNew, cEndNew, c, vStart, vEnd, vStartNew, vEndNew, v;
635   PetscBool    hasCells;
636 
637   PetscFunctionBegin;
638   PetscCall(DMGetCoordinateDim(dm, &dim));
639   PetscCall(DMSetCoordinateDim(dmNew, dim));
640   PetscCall(DMPlexGetDepth(dm, &depth));
641   /* Step 8: Convert coordinates */
642   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
643   PetscCall(DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd));
644   PetscCall(DMPlexGetDepthStratum(dmNew, 0, &vStartNew, &vEndNew));
645   PetscCall(DMPlexGetHeightStratum(dmNew, 0, &cStartNew, &cEndNew));
646   PetscCall(DMGetCoordinateSection(dm, &coordSection));
647   PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dm), &newCoordSection));
648   PetscCall(PetscSectionSetNumFields(newCoordSection, 1));
649   PetscCall(PetscSectionSetFieldComponents(newCoordSection, 0, dim));
650   PetscCall(PetscSectionGetChart(coordSection, &sStart, &sEnd));
651   hasCells = sStart == cStart ? PETSC_TRUE : PETSC_FALSE;
652   PetscCall(PetscSectionSetChart(newCoordSection, hasCells ? cStartNew : vStartNew, vEndNew));
653   if (hasCells) {
654     for (c = cStart; c < cEnd; ++c) {
655       PetscInt cNew = DMPlexShiftPoint_Internal(c, depth, depthShift), dof;
656 
657       PetscCall(PetscSectionGetDof(coordSection, c, &dof));
658       PetscCall(PetscSectionSetDof(newCoordSection, cNew, dof));
659       PetscCall(PetscSectionSetFieldDof(newCoordSection, cNew, 0, dof));
660     }
661   }
662   for (v = vStartNew; v < vEndNew; ++v) {
663     PetscCall(PetscSectionSetDof(newCoordSection, v, dim));
664     PetscCall(PetscSectionSetFieldDof(newCoordSection, v, 0, dim));
665   }
666   PetscCall(PetscSectionSetUp(newCoordSection));
667   PetscCall(DMSetCoordinateSection(dmNew, PETSC_DETERMINE, newCoordSection));
668   PetscCall(PetscSectionGetStorageSize(newCoordSection, &coordSize));
669   PetscCall(VecCreate(PETSC_COMM_SELF, &newCoordinates));
670   PetscCall(PetscObjectSetName((PetscObject)newCoordinates, "coordinates"));
671   PetscCall(VecSetSizes(newCoordinates, coordSize, PETSC_DETERMINE));
672   PetscCall(VecSetBlockSize(newCoordinates, dim));
673   PetscCall(VecSetType(newCoordinates, VECSTANDARD));
674   PetscCall(DMSetCoordinatesLocal(dmNew, newCoordinates));
675   PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
676   PetscCall(VecGetArray(coordinates, &coords));
677   PetscCall(VecGetArray(newCoordinates, &newCoords));
678   if (hasCells) {
679     for (c = cStart; c < cEnd; ++c) {
680       PetscInt cNew = DMPlexShiftPoint_Internal(c, depth, depthShift), dof, off, noff, d;
681 
682       PetscCall(PetscSectionGetDof(coordSection, c, &dof));
683       PetscCall(PetscSectionGetOffset(coordSection, c, &off));
684       PetscCall(PetscSectionGetOffset(newCoordSection, cNew, &noff));
685       for (d = 0; d < dof; ++d) newCoords[noff + d] = coords[off + d];
686     }
687   }
688   for (v = vStart; v < vEnd; ++v) {
689     PetscInt dof, off, noff, d;
690 
691     PetscCall(PetscSectionGetDof(coordSection, v, &dof));
692     PetscCall(PetscSectionGetOffset(coordSection, v, &off));
693     PetscCall(PetscSectionGetOffset(newCoordSection, DMPlexShiftPoint_Internal(v, depth, depthShift), &noff));
694     for (d = 0; d < dof; ++d) newCoords[noff + d] = coords[off + d];
695   }
696   PetscCall(VecRestoreArray(coordinates, &coords));
697   PetscCall(VecRestoreArray(newCoordinates, &newCoords));
698   PetscCall(VecDestroy(&newCoordinates));
699   PetscCall(PetscSectionDestroy(&newCoordSection));
700   PetscFunctionReturn(PETSC_SUCCESS);
701 }
702 
703 static PetscErrorCode DMPlexShiftSF_Single(DM dm, PetscInt depthShift[], PetscSF sf, PetscSF sfNew)
704 {
705   const PetscSFNode *remotePoints;
706   PetscSFNode       *gremotePoints;
707   const PetscInt    *localPoints;
708   PetscInt          *glocalPoints, *newLocation, *newRemoteLocation;
709   PetscInt           numRoots, numLeaves, l, pStart, pEnd, depth = 0, totShift = 0;
710 
711   PetscFunctionBegin;
712   PetscCall(DMPlexGetDepth(dm, &depth));
713   PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
714   PetscCall(PetscSFGetGraph(sf, &numRoots, &numLeaves, &localPoints, &remotePoints));
715   totShift = DMPlexShiftPoint_Internal(pEnd, depth, depthShift) - pEnd;
716   if (numRoots >= 0) {
717     PetscCall(PetscMalloc2(numRoots, &newLocation, pEnd - pStart, &newRemoteLocation));
718     for (l = 0; l < numRoots; ++l) newLocation[l] = DMPlexShiftPoint_Internal(l, depth, depthShift);
719     PetscCall(PetscSFBcastBegin(sf, MPIU_INT, newLocation, newRemoteLocation, MPI_REPLACE));
720     PetscCall(PetscSFBcastEnd(sf, MPIU_INT, newLocation, newRemoteLocation, MPI_REPLACE));
721     PetscCall(PetscMalloc1(numLeaves, &glocalPoints));
722     PetscCall(PetscMalloc1(numLeaves, &gremotePoints));
723     for (l = 0; l < numLeaves; ++l) {
724       glocalPoints[l]        = DMPlexShiftPoint_Internal(localPoints[l], depth, depthShift);
725       gremotePoints[l].rank  = remotePoints[l].rank;
726       gremotePoints[l].index = newRemoteLocation[localPoints[l]];
727     }
728     PetscCall(PetscFree2(newLocation, newRemoteLocation));
729     PetscCall(PetscSFSetGraph(sfNew, numRoots + totShift, numLeaves, glocalPoints, PETSC_OWN_POINTER, gremotePoints, PETSC_OWN_POINTER));
730   }
731   PetscFunctionReturn(PETSC_SUCCESS);
732 }
733 
734 static PetscErrorCode DMPlexShiftSF_Internal(DM dm, PetscInt depthShift[], DM dmNew)
735 {
736   PetscSF   sfPoint, sfPointNew;
737   PetscBool useNatural;
738 
739   PetscFunctionBegin;
740   /* Step 9: Convert pointSF */
741   PetscCall(DMGetPointSF(dm, &sfPoint));
742   PetscCall(DMGetPointSF(dmNew, &sfPointNew));
743   PetscCall(DMPlexShiftSF_Single(dm, depthShift, sfPoint, sfPointNew));
744   /* Step 9b: Convert naturalSF */
745   PetscCall(DMGetUseNatural(dm, &useNatural));
746   if (useNatural) {
747     PetscSF sfNat, sfNatNew;
748 
749     PetscCall(DMSetUseNatural(dmNew, useNatural));
750     PetscCall(DMGetNaturalSF(dm, &sfNat));
751     PetscCall(DMGetNaturalSF(dmNew, &sfNatNew));
752     PetscCall(DMPlexShiftSF_Single(dm, depthShift, sfNat, sfNatNew));
753   }
754   PetscFunctionReturn(PETSC_SUCCESS);
755 }
756 
757 static PetscErrorCode DMPlexShiftLabels_Internal(DM dm, PetscInt depthShift[], DM dmNew)
758 {
759   PetscInt depth = 0, numLabels, l;
760 
761   PetscFunctionBegin;
762   PetscCall(DMPlexGetDepth(dm, &depth));
763   /* Step 10: Convert labels */
764   PetscCall(DMGetNumLabels(dm, &numLabels));
765   for (l = 0; l < numLabels; ++l) {
766     DMLabel         label, newlabel;
767     const char     *lname;
768     PetscBool       isDepth, isDim;
769     IS              valueIS;
770     const PetscInt *values;
771     PetscInt        numValues, val;
772 
773     PetscCall(DMGetLabelName(dm, l, &lname));
774     PetscCall(PetscStrcmp(lname, "depth", &isDepth));
775     if (isDepth) continue;
776     PetscCall(PetscStrcmp(lname, "dim", &isDim));
777     if (isDim) continue;
778     PetscCall(DMCreateLabel(dmNew, lname));
779     PetscCall(DMGetLabel(dm, lname, &label));
780     PetscCall(DMGetLabel(dmNew, lname, &newlabel));
781     PetscCall(DMLabelGetDefaultValue(label, &val));
782     PetscCall(DMLabelSetDefaultValue(newlabel, val));
783     PetscCall(DMLabelGetValueIS(label, &valueIS));
784     PetscCall(ISGetLocalSize(valueIS, &numValues));
785     PetscCall(ISGetIndices(valueIS, &values));
786     for (val = 0; val < numValues; ++val) {
787       IS              pointIS;
788       const PetscInt *points;
789       PetscInt        numPoints, p;
790 
791       PetscCall(DMLabelGetStratumIS(label, values[val], &pointIS));
792       PetscCall(ISGetLocalSize(pointIS, &numPoints));
793       PetscCall(ISGetIndices(pointIS, &points));
794       for (p = 0; p < numPoints; ++p) {
795         const PetscInt newpoint = DMPlexShiftPoint_Internal(points[p], depth, depthShift);
796 
797         PetscCall(DMLabelSetValue(newlabel, newpoint, values[val]));
798       }
799       PetscCall(ISRestoreIndices(pointIS, &points));
800       PetscCall(ISDestroy(&pointIS));
801     }
802     PetscCall(ISRestoreIndices(valueIS, &values));
803     PetscCall(ISDestroy(&valueIS));
804   }
805   PetscFunctionReturn(PETSC_SUCCESS);
806 }
807 
808 static PetscErrorCode DMPlexCreateVTKLabel_Internal(DM dm, PetscBool createGhostLabel, DM dmNew)
809 {
810   PetscSF            sfPoint;
811   DMLabel            vtkLabel, ghostLabel = NULL;
812   const PetscSFNode *leafRemote;
813   const PetscInt    *leafLocal;
814   PetscInt           cellHeight, cStart, cEnd, c, fStart, fEnd, f, numLeaves, l;
815   PetscMPIInt        rank;
816 
817   PetscFunctionBegin;
818   /* Step 11: Make label for output (vtk) and to mark ghost points (ghost) */
819   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
820   PetscCall(DMGetPointSF(dm, &sfPoint));
821   PetscCall(DMPlexGetVTKCellHeight(dmNew, &cellHeight));
822   PetscCall(DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd));
823   PetscCall(PetscSFGetGraph(sfPoint, NULL, &numLeaves, &leafLocal, &leafRemote));
824   PetscCall(DMCreateLabel(dmNew, "vtk"));
825   PetscCall(DMGetLabel(dmNew, "vtk", &vtkLabel));
826   if (createGhostLabel) {
827     PetscCall(DMCreateLabel(dmNew, "ghost"));
828     PetscCall(DMGetLabel(dmNew, "ghost", &ghostLabel));
829   }
830   for (l = 0, c = cStart; l < numLeaves && c < cEnd; ++l, ++c) {
831     for (; c < leafLocal[l] && c < cEnd; ++c) PetscCall(DMLabelSetValue(vtkLabel, c, 1));
832     if (leafLocal[l] >= cEnd) break;
833     if (leafRemote[l].rank == rank) {
834       PetscCall(DMLabelSetValue(vtkLabel, c, 1));
835     } else if (ghostLabel) PetscCall(DMLabelSetValue(ghostLabel, c, 2));
836   }
837   for (; c < cEnd; ++c) PetscCall(DMLabelSetValue(vtkLabel, c, 1));
838   if (ghostLabel) {
839     PetscCall(DMPlexGetHeightStratum(dmNew, 1, &fStart, &fEnd));
840     for (f = fStart; f < fEnd; ++f) {
841       PetscInt numCells;
842 
843       PetscCall(DMPlexGetSupportSize(dmNew, f, &numCells));
844       if (numCells < 2) {
845         PetscCall(DMLabelSetValue(ghostLabel, f, 1));
846       } else {
847         const PetscInt *cells = NULL;
848         PetscInt        vA, vB;
849 
850         PetscCall(DMPlexGetSupport(dmNew, f, &cells));
851         PetscCall(DMLabelGetValue(vtkLabel, cells[0], &vA));
852         PetscCall(DMLabelGetValue(vtkLabel, cells[1], &vB));
853         if (vA != 1 && vB != 1) PetscCall(DMLabelSetValue(ghostLabel, f, 1));
854       }
855     }
856   }
857   PetscFunctionReturn(PETSC_SUCCESS);
858 }
859 
860 static PetscErrorCode DMPlexShiftTree_Internal(DM dm, PetscInt depthShift[], DM dmNew)
861 {
862   DM           refTree;
863   PetscSection pSec;
864   PetscInt    *parents, *childIDs;
865 
866   PetscFunctionBegin;
867   PetscCall(DMPlexGetReferenceTree(dm, &refTree));
868   PetscCall(DMPlexSetReferenceTree(dmNew, refTree));
869   PetscCall(DMPlexGetTree(dm, &pSec, &parents, &childIDs, NULL, NULL));
870   if (pSec) {
871     PetscInt     p, pStart, pEnd, *parentsShifted, pStartShifted, pEndShifted, depth;
872     PetscInt    *childIDsShifted;
873     PetscSection pSecShifted;
874 
875     PetscCall(PetscSectionGetChart(pSec, &pStart, &pEnd));
876     PetscCall(DMPlexGetDepth(dm, &depth));
877     pStartShifted = DMPlexShiftPoint_Internal(pStart, depth, depthShift);
878     pEndShifted   = DMPlexShiftPoint_Internal(pEnd, depth, depthShift);
879     PetscCall(PetscMalloc2(pEndShifted - pStartShifted, &parentsShifted, pEndShifted - pStartShifted, &childIDsShifted));
880     PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dmNew), &pSecShifted));
881     PetscCall(PetscSectionSetChart(pSecShifted, pStartShifted, pEndShifted));
882     for (p = pStartShifted; p < pEndShifted; p++) {
883       /* start off assuming no children */
884       PetscCall(PetscSectionSetDof(pSecShifted, p, 0));
885     }
886     for (p = pStart; p < pEnd; p++) {
887       PetscInt dof;
888       PetscInt pNew = DMPlexShiftPoint_Internal(p, depth, depthShift);
889 
890       PetscCall(PetscSectionGetDof(pSec, p, &dof));
891       PetscCall(PetscSectionSetDof(pSecShifted, pNew, dof));
892     }
893     PetscCall(PetscSectionSetUp(pSecShifted));
894     for (p = pStart; p < pEnd; p++) {
895       PetscInt dof;
896       PetscInt pNew = DMPlexShiftPoint_Internal(p, depth, depthShift);
897 
898       PetscCall(PetscSectionGetDof(pSec, p, &dof));
899       if (dof) {
900         PetscInt off, offNew;
901 
902         PetscCall(PetscSectionGetOffset(pSec, p, &off));
903         PetscCall(PetscSectionGetOffset(pSecShifted, pNew, &offNew));
904         parentsShifted[offNew]  = DMPlexShiftPoint_Internal(parents[off], depth, depthShift);
905         childIDsShifted[offNew] = childIDs[off];
906       }
907     }
908     PetscCall(DMPlexSetTree(dmNew, pSecShifted, parentsShifted, childIDsShifted));
909     PetscCall(PetscFree2(parentsShifted, childIDsShifted));
910     PetscCall(PetscSectionDestroy(&pSecShifted));
911   }
912   PetscFunctionReturn(PETSC_SUCCESS);
913 }
914 
915 static PetscErrorCode DMPlexConstructGhostCells_Internal(DM dm, DMLabel label, PetscInt *numGhostCells, DM gdm)
916 {
917   PetscSF          sf;
918   IS               valueIS;
919   const PetscInt  *values, *leaves;
920   PetscInt        *depthShift;
921   PetscInt         d, depth = 0, nleaves, loc, Ng, numFS, fs, fStart, fEnd, ghostCell, cEnd, c;
922   const PetscReal *maxCell, *Lstart, *L;
923 
924   PetscFunctionBegin;
925   PetscCall(DMGetPointSF(dm, &sf));
926   PetscCall(PetscSFGetGraph(sf, NULL, &nleaves, &leaves, NULL));
927   nleaves = PetscMax(0, nleaves);
928   PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
929   /* Count ghost cells */
930   PetscCall(DMLabelGetValueIS(label, &valueIS));
931   PetscCall(ISGetLocalSize(valueIS, &numFS));
932   PetscCall(ISGetIndices(valueIS, &values));
933   Ng = 0;
934   for (fs = 0; fs < numFS; ++fs) {
935     IS              faceIS;
936     const PetscInt *faces;
937     PetscInt        numFaces, f, numBdFaces = 0;
938 
939     PetscCall(DMLabelGetStratumIS(label, values[fs], &faceIS));
940     PetscCall(ISGetLocalSize(faceIS, &numFaces));
941     PetscCall(ISGetIndices(faceIS, &faces));
942     for (f = 0; f < numFaces; ++f) {
943       PetscInt numChildren;
944 
945       PetscCall(PetscFindInt(faces[f], nleaves, leaves, &loc));
946       PetscCall(DMPlexGetTreeChildren(dm, faces[f], &numChildren, NULL));
947       /* non-local and ancestors points don't get to register ghosts */
948       if (loc >= 0 || numChildren) continue;
949       if ((faces[f] >= fStart) && (faces[f] < fEnd)) ++numBdFaces;
950     }
951     Ng += numBdFaces;
952     PetscCall(ISRestoreIndices(faceIS, &faces));
953     PetscCall(ISDestroy(&faceIS));
954   }
955   PetscCall(DMPlexGetDepth(dm, &depth));
956   PetscCall(PetscMalloc1(2 * (depth + 1), &depthShift));
957   for (d = 0; d <= depth; d++) {
958     PetscInt dEnd;
959 
960     PetscCall(DMPlexGetDepthStratum(dm, d, NULL, &dEnd));
961     depthShift[2 * d]     = dEnd;
962     depthShift[2 * d + 1] = 0;
963   }
964   if (depth >= 0) depthShift[2 * depth + 1] = Ng;
965   PetscCall(DMPlexShiftPointSetUp_Internal(depth, depthShift));
966   PetscCall(DMPlexShiftSizes_Internal(dm, depthShift, gdm));
967   /* Step 3: Set cone/support sizes for new points */
968   PetscCall(DMPlexGetHeightStratum(dm, 0, NULL, &cEnd));
969   for (c = cEnd; c < cEnd + Ng; ++c) PetscCall(DMPlexSetConeSize(gdm, c, 1));
970   for (fs = 0; fs < numFS; ++fs) {
971     IS              faceIS;
972     const PetscInt *faces;
973     PetscInt        numFaces, f;
974 
975     PetscCall(DMLabelGetStratumIS(label, values[fs], &faceIS));
976     PetscCall(ISGetLocalSize(faceIS, &numFaces));
977     PetscCall(ISGetIndices(faceIS, &faces));
978     for (f = 0; f < numFaces; ++f) {
979       PetscInt size, numChildren;
980 
981       PetscCall(PetscFindInt(faces[f], nleaves, leaves, &loc));
982       PetscCall(DMPlexGetTreeChildren(dm, faces[f], &numChildren, NULL));
983       if (loc >= 0 || numChildren) continue;
984       if ((faces[f] < fStart) || (faces[f] >= fEnd)) continue;
985       PetscCall(DMPlexGetSupportSize(dm, faces[f], &size));
986       PetscCheck(size == 1, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM has boundary face %" PetscInt_FMT " with %" PetscInt_FMT " support cells", faces[f], size);
987       PetscCall(DMPlexSetSupportSize(gdm, faces[f] + Ng, 2));
988     }
989     PetscCall(ISRestoreIndices(faceIS, &faces));
990     PetscCall(ISDestroy(&faceIS));
991   }
992   /* Step 4: Setup ghosted DM */
993   PetscCall(DMSetUp(gdm));
994   PetscCall(DMPlexShiftPoints_Internal(dm, depthShift, gdm));
995   /* Step 6: Set cones and supports for new points */
996   ghostCell = cEnd;
997   for (fs = 0; fs < numFS; ++fs) {
998     IS              faceIS;
999     const PetscInt *faces;
1000     PetscInt        numFaces, f;
1001 
1002     PetscCall(DMLabelGetStratumIS(label, values[fs], &faceIS));
1003     PetscCall(ISGetLocalSize(faceIS, &numFaces));
1004     PetscCall(ISGetIndices(faceIS, &faces));
1005     for (f = 0; f < numFaces; ++f) {
1006       PetscInt newFace = faces[f] + Ng, numChildren;
1007 
1008       PetscCall(PetscFindInt(faces[f], nleaves, leaves, &loc));
1009       PetscCall(DMPlexGetTreeChildren(dm, faces[f], &numChildren, NULL));
1010       if (loc >= 0 || numChildren) continue;
1011       if ((faces[f] < fStart) || (faces[f] >= fEnd)) continue;
1012       PetscCall(DMPlexSetCone(gdm, ghostCell, &newFace));
1013       PetscCall(DMPlexInsertSupport(gdm, newFace, 1, ghostCell));
1014       ++ghostCell;
1015     }
1016     PetscCall(ISRestoreIndices(faceIS, &faces));
1017     PetscCall(ISDestroy(&faceIS));
1018   }
1019   PetscCall(ISRestoreIndices(valueIS, &values));
1020   PetscCall(ISDestroy(&valueIS));
1021   PetscCall(DMPlexShiftCoordinates_Internal(dm, depthShift, gdm));
1022   PetscCall(DMPlexShiftSF_Internal(dm, depthShift, gdm));
1023   PetscCall(DMPlexShiftLabels_Internal(dm, depthShift, gdm));
1024   PetscCall(DMPlexCreateVTKLabel_Internal(dm, PETSC_TRUE, gdm));
1025   PetscCall(DMPlexShiftTree_Internal(dm, depthShift, gdm));
1026   PetscCall(PetscFree(depthShift));
1027   for (c = cEnd; c < cEnd + Ng; ++c) PetscCall(DMPlexSetCellType(gdm, c, DM_POLYTOPE_FV_GHOST));
1028   /* Step 7: Periodicity */
1029   PetscCall(DMGetPeriodicity(dm, &maxCell, &Lstart, &L));
1030   PetscCall(DMSetPeriodicity(gdm, maxCell, Lstart, L));
1031   if (numGhostCells) *numGhostCells = Ng;
1032   PetscFunctionReturn(PETSC_SUCCESS);
1033 }
1034 
1035 /*@C
1036   DMPlexConstructGhostCells - Construct ghost cells which connect to every boundary face
1037 
1038   Collective
1039 
1040   Input Parameters:
1041 + dm        - The original `DM`
1042 - labelName - The label specifying the boundary faces, or "Face Sets" if this is `NULL`
1043 
1044   Output Parameters:
1045 + numGhostCells - The number of ghost cells added to the `DM`
1046 - dmGhosted     - The new `DM`
1047 
1048   Level: developer
1049 
1050   Note:
1051   If no label exists of that name, one will be created marking all boundary faces
1052 
1053 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMCreate()`
1054 @*/
1055 PetscErrorCode DMPlexConstructGhostCells(DM dm, const char labelName[], PetscInt *numGhostCells, DM *dmGhosted)
1056 {
1057   DM          gdm;
1058   DMLabel     label;
1059   const char *name = labelName ? labelName : "Face Sets";
1060   PetscInt    dim, Ng = 0;
1061   PetscBool   useCone, useClosure;
1062 
1063   PetscFunctionBegin;
1064   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1065   if (numGhostCells) PetscAssertPointer(numGhostCells, 3);
1066   PetscAssertPointer(dmGhosted, 4);
1067   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &gdm));
1068   PetscCall(DMSetType(gdm, DMPLEX));
1069   PetscCall(DMGetDimension(dm, &dim));
1070   PetscCall(DMSetDimension(gdm, dim));
1071   PetscCall(DMGetBasicAdjacency(dm, &useCone, &useClosure));
1072   PetscCall(DMSetBasicAdjacency(gdm, useCone, useClosure));
1073   PetscCall(DMGetLabel(dm, name, &label));
1074   if (!label) {
1075     /* Get label for boundary faces */
1076     PetscCall(DMCreateLabel(dm, name));
1077     PetscCall(DMGetLabel(dm, name, &label));
1078     PetscCall(DMPlexMarkBoundaryFaces(dm, 1, label));
1079   }
1080   PetscCall(DMPlexConstructGhostCells_Internal(dm, label, &Ng, gdm));
1081   PetscCall(DMCopyDisc(dm, gdm));
1082   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_TRUE, gdm));
1083   gdm->setfromoptionscalled = dm->setfromoptionscalled;
1084   if (numGhostCells) *numGhostCells = Ng;
1085   *dmGhosted = gdm;
1086   PetscFunctionReturn(PETSC_SUCCESS);
1087 }
1088 
1089 static PetscErrorCode DivideCells_Private(DM dm, DMLabel label, DMPlexPointQueue queue)
1090 {
1091   PetscInt dim, d, shift = 100, *pStart, *pEnd;
1092 
1093   PetscFunctionBegin;
1094   PetscCall(DMGetDimension(dm, &dim));
1095   PetscCall(PetscMalloc2(dim, &pStart, dim, &pEnd));
1096   for (d = 0; d < dim; ++d) PetscCall(DMPlexGetDepthStratum(dm, d, &pStart[d], &pEnd[d]));
1097   while (!DMPlexPointQueueEmpty(queue)) {
1098     PetscInt  cell    = -1;
1099     PetscInt *closure = NULL;
1100     PetscInt  closureSize, cl, cval;
1101 
1102     PetscCall(DMPlexPointQueueDequeue(queue, &cell));
1103     PetscCall(DMLabelGetValue(label, cell, &cval));
1104     PetscCall(DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
1105     /* Mark points in the cell closure that touch the fault */
1106     for (d = 0; d < dim; ++d) {
1107       for (cl = 0; cl < closureSize * 2; cl += 2) {
1108         const PetscInt clp = closure[cl];
1109         PetscInt       clval;
1110 
1111         if ((clp < pStart[d]) || (clp >= pEnd[d])) continue;
1112         PetscCall(DMLabelGetValue(label, clp, &clval));
1113         if (clval == -1) {
1114           const PetscInt *cone;
1115           PetscInt        coneSize, c;
1116 
1117           /* If a cone point touches the fault, then this point touches the fault */
1118           PetscCall(DMPlexGetCone(dm, clp, &cone));
1119           PetscCall(DMPlexGetConeSize(dm, clp, &coneSize));
1120           for (c = 0; c < coneSize; ++c) {
1121             PetscInt cpval;
1122 
1123             PetscCall(DMLabelGetValue(label, cone[c], &cpval));
1124             if (cpval != -1) {
1125               PetscInt dep;
1126 
1127               PetscCall(DMPlexGetPointDepth(dm, clp, &dep));
1128               clval = cval < 0 ? -(shift + dep) : shift + dep;
1129               PetscCall(DMLabelSetValue(label, clp, clval));
1130               break;
1131             }
1132           }
1133         }
1134         /* Mark neighbor cells through marked faces (these cells must also touch the fault) */
1135         if (d == dim - 1 && clval != -1) {
1136           const PetscInt *support;
1137           PetscInt        supportSize, s, nval;
1138 
1139           PetscCall(DMPlexGetSupport(dm, clp, &support));
1140           PetscCall(DMPlexGetSupportSize(dm, clp, &supportSize));
1141           for (s = 0; s < supportSize; ++s) {
1142             PetscCall(DMLabelGetValue(label, support[s], &nval));
1143             if (nval == -1) {
1144               PetscCall(DMLabelSetValue(label, support[s], clval < 0 ? clval - 1 : clval + 1));
1145               PetscCall(DMPlexPointQueueEnqueue(queue, support[s]));
1146             }
1147           }
1148         }
1149       }
1150     }
1151     PetscCall(DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
1152   }
1153   PetscCall(PetscFree2(pStart, pEnd));
1154   PetscFunctionReturn(PETSC_SUCCESS);
1155 }
1156 
1157 typedef struct {
1158   DM               dm;
1159   DMPlexPointQueue queue;
1160 } PointDivision;
1161 
1162 static PetscErrorCode divideCell(DMLabel label, PetscInt p, PetscInt val, void *ctx)
1163 {
1164   PointDivision  *div  = (PointDivision *)ctx;
1165   PetscInt        cval = val < 0 ? val - 1 : val + 1;
1166   const PetscInt *support;
1167   PetscInt        supportSize, s;
1168 
1169   PetscFunctionBegin;
1170   PetscCall(DMPlexGetSupport(div->dm, p, &support));
1171   PetscCall(DMPlexGetSupportSize(div->dm, p, &supportSize));
1172   for (s = 0; s < supportSize; ++s) {
1173     PetscCall(DMLabelSetValue(label, support[s], cval));
1174     PetscCall(DMPlexPointQueueEnqueue(div->queue, support[s]));
1175   }
1176   PetscFunctionReturn(PETSC_SUCCESS);
1177 }
1178 
1179 /* Mark cells by label propagation */
1180 static PetscErrorCode DMPlexLabelFaultHalo(DM dm, DMLabel faultLabel)
1181 {
1182   DMPlexPointQueue queue = NULL;
1183   PointDivision    div;
1184   PetscSF          pointSF;
1185   IS               pointIS;
1186   const PetscInt  *points;
1187   PetscBool        empty;
1188   PetscInt         dim, shift = 100, n, i;
1189 
1190   PetscFunctionBegin;
1191   PetscCall(DMGetDimension(dm, &dim));
1192   PetscCall(DMPlexPointQueueCreate(1024, &queue));
1193   div.dm    = dm;
1194   div.queue = queue;
1195   /* Enqueue cells on fault */
1196   PetscCall(DMLabelGetStratumIS(faultLabel, shift + dim, &pointIS));
1197   if (pointIS) {
1198     PetscCall(ISGetLocalSize(pointIS, &n));
1199     PetscCall(ISGetIndices(pointIS, &points));
1200     for (i = 0; i < n; ++i) PetscCall(DMPlexPointQueueEnqueue(queue, points[i]));
1201     PetscCall(ISRestoreIndices(pointIS, &points));
1202     PetscCall(ISDestroy(&pointIS));
1203   }
1204   PetscCall(DMLabelGetStratumIS(faultLabel, -(shift + dim), &pointIS));
1205   if (pointIS) {
1206     PetscCall(ISGetLocalSize(pointIS, &n));
1207     PetscCall(ISGetIndices(pointIS, &points));
1208     for (i = 0; i < n; ++i) PetscCall(DMPlexPointQueueEnqueue(queue, points[i]));
1209     PetscCall(ISRestoreIndices(pointIS, &points));
1210     PetscCall(ISDestroy(&pointIS));
1211   }
1212 
1213   PetscCall(DMGetPointSF(dm, &pointSF));
1214   PetscCall(DMLabelPropagateBegin(faultLabel, pointSF));
1215   /* While edge queue is not empty: */
1216   PetscCall(DMPlexPointQueueEmptyCollective((PetscObject)dm, queue, &empty));
1217   while (!empty) {
1218     PetscCall(DivideCells_Private(dm, faultLabel, queue));
1219     PetscCall(DMLabelPropagatePush(faultLabel, pointSF, divideCell, &div));
1220     PetscCall(DMPlexPointQueueEmptyCollective((PetscObject)dm, queue, &empty));
1221   }
1222   PetscCall(DMLabelPropagateEnd(faultLabel, pointSF));
1223   PetscCall(DMPlexPointQueueDestroy(&queue));
1224   PetscFunctionReturn(PETSC_SUCCESS);
1225 }
1226 
1227 /*
1228   We are adding three kinds of points here:
1229     Replicated:     Copies of points which exist in the mesh, such as vertices identified across a fault
1230     Non-replicated: Points which exist on the fault, but are not replicated
1231     Ghost:          These are shared fault faces which are not owned by this process. These do not produce hybrid cells and do not replicate
1232     Hybrid:         Entirely new points, such as cohesive cells
1233 
1234   When creating subsequent cohesive cells, we shift the old hybrid cells to the end of the numbering at
1235   each depth so that the new split/hybrid points can be inserted as a block.
1236 */
1237 static PetscErrorCode DMPlexConstructCohesiveCells_Internal(DM dm, DMLabel label, DMLabel splitLabel, DM sdm)
1238 {
1239   MPI_Comm         comm;
1240   IS               valueIS;
1241   PetscInt         numSP = 0; /* The number of depths for which we have replicated points */
1242   const PetscInt  *values;    /* List of depths for which we have replicated points */
1243   IS              *splitIS;
1244   IS              *unsplitIS;
1245   IS               ghostIS;
1246   PetscInt        *numSplitPoints;     /* The number of replicated points at each depth */
1247   PetscInt        *numUnsplitPoints;   /* The number of non-replicated points at each depth which still give rise to hybrid points */
1248   PetscInt        *numHybridPoints;    /* The number of new hybrid points at each depth */
1249   PetscInt        *numHybridPointsOld; /* The number of existing hybrid points at each depth */
1250   PetscInt         numGhostPoints;     /* The number of unowned, shared fault faces */
1251   const PetscInt **splitPoints;        /* Replicated points for each depth */
1252   const PetscInt **unsplitPoints;      /* Non-replicated points for each depth */
1253   const PetscInt  *ghostPoints;        /* Ghost fault faces */
1254   PetscSection     coordSection;
1255   Vec              coordinates;
1256   PetscScalar     *coords;
1257   PetscInt        *depthMax;   /* The first hybrid point at each depth in the original mesh */
1258   PetscInt        *depthEnd;   /* The point limit at each depth in the original mesh */
1259   PetscInt        *depthShift; /* Number of replicated+hybrid points at each depth */
1260   PetscInt        *pMaxNew;    /* The first replicated point at each depth in the new mesh, hybrids come after this */
1261   PetscInt        *coneNew, *coneONew, *supportNew;
1262   PetscInt         shift = 100, shift2 = 200, depth = 0, dep, dim, d, sp, maxConeSize, maxSupportSize, maxConeSizeNew, maxSupportSizeNew, numLabels, vStart, vEnd, pEnd, p, v;
1263 
1264   PetscFunctionBegin;
1265   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
1266   PetscCall(DMGetDimension(dm, &dim));
1267   PetscCall(DMPlexGetDepth(dm, &depth));
1268   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
1269   /* We do not want this label automatically computed, instead we compute it here */
1270   PetscCall(DMCreateLabel(sdm, "celltype"));
1271   /* Count split points and add cohesive cells */
1272   PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize));
1273   PetscCall(PetscMalloc5(depth + 1, &depthMax, depth + 1, &depthEnd, 2 * (depth + 1), &depthShift, depth + 1, &pMaxNew, depth + 1, &numHybridPointsOld));
1274   PetscCall(PetscMalloc7(depth + 1, &splitIS, depth + 1, &unsplitIS, depth + 1, &numSplitPoints, depth + 1, &numUnsplitPoints, depth + 1, &numHybridPoints, depth + 1, &splitPoints, depth + 1, &unsplitPoints));
1275   for (d = 0; d <= depth; ++d) {
1276     PetscCall(DMPlexGetDepthStratum(dm, d, NULL, &pMaxNew[d]));
1277     PetscCall(DMPlexGetTensorPrismBounds_Internal(dm, d, &depthMax[d], NULL));
1278     depthEnd[d]           = pMaxNew[d];
1279     depthMax[d]           = depthMax[d] < 0 ? depthEnd[d] : depthMax[d];
1280     numSplitPoints[d]     = 0;
1281     numUnsplitPoints[d]   = 0;
1282     numHybridPoints[d]    = 0;
1283     numHybridPointsOld[d] = depthMax[d] < 0 ? 0 : depthEnd[d] - depthMax[d];
1284     splitPoints[d]        = NULL;
1285     unsplitPoints[d]      = NULL;
1286     splitIS[d]            = NULL;
1287     unsplitIS[d]          = NULL;
1288     /* we are shifting the existing hybrid points with the stratum behind them, so
1289      * the split comes at the end of the normal points, i.e., at depthMax[d] */
1290     depthShift[2 * d]     = depthMax[d];
1291     depthShift[2 * d + 1] = 0;
1292   }
1293   numGhostPoints = 0;
1294   ghostPoints    = NULL;
1295   if (label) {
1296     PetscCall(DMLabelGetValueIS(label, &valueIS));
1297     PetscCall(ISGetLocalSize(valueIS, &numSP));
1298     PetscCall(ISGetIndices(valueIS, &values));
1299   }
1300   for (sp = 0; sp < numSP; ++sp) {
1301     const PetscInt dep = values[sp];
1302 
1303     if ((dep < 0) || (dep > depth)) continue;
1304     PetscCall(DMLabelGetStratumIS(label, dep, &splitIS[dep]));
1305     if (splitIS[dep]) {
1306       PetscCall(ISGetLocalSize(splitIS[dep], &numSplitPoints[dep]));
1307       PetscCall(ISGetIndices(splitIS[dep], &splitPoints[dep]));
1308     }
1309     PetscCall(DMLabelGetStratumIS(label, shift2 + dep, &unsplitIS[dep]));
1310     if (unsplitIS[dep]) {
1311       PetscCall(ISGetLocalSize(unsplitIS[dep], &numUnsplitPoints[dep]));
1312       PetscCall(ISGetIndices(unsplitIS[dep], &unsplitPoints[dep]));
1313     }
1314   }
1315   PetscCall(DMLabelGetStratumIS(label, shift2 + dim - 1, &ghostIS));
1316   if (ghostIS) {
1317     PetscCall(ISGetLocalSize(ghostIS, &numGhostPoints));
1318     PetscCall(ISGetIndices(ghostIS, &ghostPoints));
1319   }
1320   /* Calculate number of hybrid points */
1321   for (d = 1; d <= depth; ++d) numHybridPoints[d] = numSplitPoints[d - 1] + numUnsplitPoints[d - 1]; /* There is a hybrid cell/face/edge for every split face/edge/vertex   */
1322   for (d = 0; d <= depth; ++d) depthShift[2 * d + 1] = numSplitPoints[d] + numHybridPoints[d];
1323   PetscCall(DMPlexShiftPointSetUp_Internal(depth, depthShift));
1324   /* the end of the points in this stratum that come before the new points:
1325    * shifting pMaxNew[d] gets the new start of the next stratum, then count back the old hybrid points and the newly
1326    * added points */
1327   for (d = 0; d <= depth; ++d) pMaxNew[d] = DMPlexShiftPoint_Internal(pMaxNew[d], depth, depthShift) - (numHybridPointsOld[d] + numSplitPoints[d] + numHybridPoints[d]);
1328   PetscCall(DMPlexShiftSizes_Internal(dm, depthShift, sdm));
1329   /* Step 3: Set cone/support sizes for new points */
1330   for (dep = 0; dep <= depth; ++dep) {
1331     for (p = 0; p < numSplitPoints[dep]; ++p) {
1332       const PetscInt  oldp   = splitPoints[dep][p];
1333       const PetscInt  newp   = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*oldp + depthOffset[dep]*/;
1334       const PetscInt  splitp = p + pMaxNew[dep];
1335       const PetscInt *support;
1336       DMPolytopeType  ct;
1337       PetscInt        coneSize, supportSize, qf, qn, qp, e;
1338 
1339       PetscCall(DMPlexGetConeSize(dm, oldp, &coneSize));
1340       PetscCall(DMPlexSetConeSize(sdm, splitp, coneSize));
1341       PetscCall(DMPlexGetSupportSize(dm, oldp, &supportSize));
1342       PetscCall(DMPlexSetSupportSize(sdm, splitp, supportSize));
1343       PetscCall(DMPlexGetCellType(dm, oldp, &ct));
1344       PetscCall(DMPlexSetCellType(sdm, splitp, ct));
1345       if (dep == depth - 1) {
1346         const PetscInt hybcell = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1];
1347 
1348         /* Add cohesive cells, they are prisms */
1349         PetscCall(DMPlexSetConeSize(sdm, hybcell, 2 + coneSize));
1350         switch (coneSize) {
1351         case 2:
1352           PetscCall(DMPlexSetCellType(sdm, hybcell, DM_POLYTOPE_SEG_PRISM_TENSOR));
1353           break;
1354         case 3:
1355           PetscCall(DMPlexSetCellType(sdm, hybcell, DM_POLYTOPE_TRI_PRISM_TENSOR));
1356           break;
1357         case 4:
1358           PetscCall(DMPlexSetCellType(sdm, hybcell, DM_POLYTOPE_QUAD_PRISM_TENSOR));
1359           break;
1360         }
1361         /* Shared fault faces with only one support cell now have two with the cohesive cell */
1362         /*   TODO Check thaat oldp has rootdegree == 1 */
1363         if (supportSize == 1) {
1364           const PetscInt *support;
1365           PetscInt        val;
1366 
1367           PetscCall(DMPlexGetSupport(dm, oldp, &support));
1368           PetscCall(DMLabelGetValue(label, support[0], &val));
1369           if (val < 0) PetscCall(DMPlexSetSupportSize(sdm, splitp, 2));
1370           else PetscCall(DMPlexSetSupportSize(sdm, newp, 2));
1371         }
1372       } else if (dep == 0) {
1373         const PetscInt hybedge = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1];
1374 
1375         PetscCall(DMPlexGetSupport(dm, oldp, &support));
1376         for (e = 0, qn = 0, qp = 0, qf = 0; e < supportSize; ++e) {
1377           PetscInt val;
1378 
1379           PetscCall(DMLabelGetValue(label, support[e], &val));
1380           if (val == 1) ++qf;
1381           if ((val == 1) || (val == (shift + 1))) ++qn;
1382           if ((val == 1) || (val == -(shift + 1))) ++qp;
1383         }
1384         /* Split old vertex: Edges into original vertex and new cohesive edge */
1385         PetscCall(DMPlexSetSupportSize(sdm, newp, qn + 1));
1386         /* Split new vertex: Edges into split vertex and new cohesive edge */
1387         PetscCall(DMPlexSetSupportSize(sdm, splitp, qp + 1));
1388         /* Add hybrid edge */
1389         PetscCall(DMPlexSetConeSize(sdm, hybedge, 2));
1390         PetscCall(DMPlexSetSupportSize(sdm, hybedge, qf));
1391         PetscCall(DMPlexSetCellType(sdm, hybedge, DM_POLYTOPE_POINT_PRISM_TENSOR));
1392       } else if (dep == dim - 2) {
1393         const PetscInt hybface = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1];
1394 
1395         PetscCall(DMPlexGetSupport(dm, oldp, &support));
1396         for (e = 0, qn = 0, qp = 0, qf = 0; e < supportSize; ++e) {
1397           PetscInt val;
1398 
1399           PetscCall(DMLabelGetValue(label, support[e], &val));
1400           if (val == dim - 1) ++qf;
1401           if ((val == dim - 1) || (val == (shift + dim - 1))) ++qn;
1402           if ((val == dim - 1) || (val == -(shift + dim - 1))) ++qp;
1403         }
1404         /* Split old edge: Faces into original edge and cohesive face (positive side?) */
1405         PetscCall(DMPlexSetSupportSize(sdm, newp, qn + 1));
1406         /* Split new edge: Faces into split edge and cohesive face (negative side?) */
1407         PetscCall(DMPlexSetSupportSize(sdm, splitp, qp + 1));
1408         /* Add hybrid face */
1409         PetscCall(DMPlexSetConeSize(sdm, hybface, 4));
1410         PetscCall(DMPlexSetSupportSize(sdm, hybface, qf));
1411         PetscCall(DMPlexSetCellType(sdm, hybface, DM_POLYTOPE_SEG_PRISM_TENSOR));
1412       }
1413     }
1414   }
1415   for (dep = 0; dep <= depth; ++dep) {
1416     for (p = 0; p < numUnsplitPoints[dep]; ++p) {
1417       const PetscInt  oldp = unsplitPoints[dep][p];
1418       const PetscInt  newp = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*oldp + depthOffset[dep]*/;
1419       const PetscInt *support;
1420       PetscInt        coneSize, supportSize, qf, e, s;
1421 
1422       PetscCall(DMPlexGetConeSize(dm, oldp, &coneSize));
1423       PetscCall(DMPlexGetSupportSize(dm, oldp, &supportSize));
1424       PetscCall(DMPlexGetSupport(dm, oldp, &support));
1425       if (dep == 0) {
1426         const PetscInt hybedge = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1] + numSplitPoints[dep];
1427 
1428         /* Unsplit vertex: Edges into original vertex, split edges, and new cohesive edge twice */
1429         for (s = 0, qf = 0; s < supportSize; ++s, ++qf) {
1430           PetscCall(PetscFindInt(support[s], numSplitPoints[dep + 1], splitPoints[dep + 1], &e));
1431           if (e >= 0) ++qf;
1432         }
1433         PetscCall(DMPlexSetSupportSize(sdm, newp, qf + 2));
1434         /* Add hybrid edge */
1435         PetscCall(DMPlexSetConeSize(sdm, hybedge, 2));
1436         for (e = 0, qf = 0; e < supportSize; ++e) {
1437           PetscInt val;
1438 
1439           PetscCall(DMLabelGetValue(label, support[e], &val));
1440           /* Split and unsplit edges produce hybrid faces */
1441           if (val == 1) ++qf;
1442           if (val == (shift2 + 1)) ++qf;
1443         }
1444         PetscCall(DMPlexSetSupportSize(sdm, hybedge, qf));
1445         PetscCall(DMPlexSetCellType(sdm, hybedge, DM_POLYTOPE_POINT_PRISM_TENSOR));
1446       } else if (dep == dim - 2) {
1447         const PetscInt hybface = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1] + numSplitPoints[dep];
1448         PetscInt       val;
1449 
1450         for (e = 0, qf = 0; e < supportSize; ++e) {
1451           PetscCall(DMLabelGetValue(label, support[e], &val));
1452           if (val == dim - 1) qf += 2;
1453           else ++qf;
1454         }
1455         /* Unsplit edge: Faces into original edge, split face, and cohesive face twice */
1456         PetscCall(DMPlexSetSupportSize(sdm, newp, qf + 2));
1457         /* Add hybrid face */
1458         for (e = 0, qf = 0; e < supportSize; ++e) {
1459           PetscCall(DMLabelGetValue(label, support[e], &val));
1460           if (val == dim - 1) ++qf;
1461         }
1462         PetscCall(DMPlexSetConeSize(sdm, hybface, 4));
1463         PetscCall(DMPlexSetSupportSize(sdm, hybface, qf));
1464         PetscCall(DMPlexSetCellType(sdm, hybface, DM_POLYTOPE_SEG_PRISM_TENSOR));
1465       }
1466     }
1467   }
1468   /* Step 4: Setup split DM */
1469   PetscCall(DMSetUp(sdm));
1470   PetscCall(DMPlexShiftPoints_Internal(dm, depthShift, sdm));
1471   PetscCall(DMPlexGetMaxSizes(sdm, &maxConeSizeNew, &maxSupportSizeNew));
1472   PetscCall(PetscMalloc3(PetscMax(maxConeSize, maxConeSizeNew) * 3, &coneNew, PetscMax(maxConeSize, maxConeSizeNew) * 3, &coneONew, PetscMax(maxSupportSize, maxSupportSizeNew), &supportNew));
1473   /* Step 6: Set cones and supports for new points */
1474   for (dep = 0; dep <= depth; ++dep) {
1475     for (p = 0; p < numSplitPoints[dep]; ++p) {
1476       const PetscInt  oldp   = splitPoints[dep][p];
1477       const PetscInt  newp   = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*oldp + depthOffset[dep]*/;
1478       const PetscInt  splitp = p + pMaxNew[dep];
1479       const PetscInt *cone, *support, *ornt;
1480       DMPolytopeType  ct;
1481       PetscInt        coneSize, supportSize, q, qf, qn, qp, v, e, s;
1482 
1483       PetscCall(DMPlexGetCellType(dm, oldp, &ct));
1484       PetscCall(DMPlexGetConeSize(dm, oldp, &coneSize));
1485       PetscCall(DMPlexGetCone(dm, oldp, &cone));
1486       PetscCall(DMPlexGetConeOrientation(dm, oldp, &ornt));
1487       PetscCall(DMPlexGetSupportSize(dm, oldp, &supportSize));
1488       PetscCall(DMPlexGetSupport(dm, oldp, &support));
1489       if (dep == depth - 1) {
1490         PetscBool       hasUnsplit = PETSC_FALSE;
1491         const PetscInt  hybcell    = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1];
1492         const PetscInt *supportF;
1493 
1494         coneONew[0] = coneONew[1] = -1000;
1495         /* Split face:       copy in old face to new face to start */
1496         PetscCall(DMPlexGetSupport(sdm, newp, &supportF));
1497         PetscCall(DMPlexSetSupport(sdm, splitp, supportF));
1498         /* Split old face:   old vertices/edges in cone so no change */
1499         /* Split new face:   new vertices/edges in cone */
1500         for (q = 0; q < coneSize; ++q) {
1501           PetscCall(PetscFindInt(cone[q], numSplitPoints[dep - 1], splitPoints[dep - 1], &v));
1502           if (v < 0) {
1503             PetscCall(PetscFindInt(cone[q], numUnsplitPoints[dep - 1], unsplitPoints[dep - 1], &v));
1504             PetscCheck(v >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %" PetscInt_FMT " in split or unsplit points of depth %" PetscInt_FMT, cone[q], dep - 1);
1505             coneNew[2 + q] = DMPlexShiftPoint_Internal(cone[q], depth, depthShift) /*cone[q] + depthOffset[dep-1]*/;
1506             hasUnsplit     = PETSC_TRUE;
1507           } else {
1508             coneNew[2 + q] = v + pMaxNew[dep - 1];
1509             if (dep > 1) {
1510               const PetscInt *econe;
1511               PetscInt        econeSize, r, vs, vu;
1512 
1513               PetscCall(DMPlexGetConeSize(dm, cone[q], &econeSize));
1514               PetscCall(DMPlexGetCone(dm, cone[q], &econe));
1515               for (r = 0; r < econeSize; ++r) {
1516                 PetscCall(PetscFindInt(econe[r], numSplitPoints[dep - 2], splitPoints[dep - 2], &vs));
1517                 PetscCall(PetscFindInt(econe[r], numUnsplitPoints[dep - 2], unsplitPoints[dep - 2], &vu));
1518                 if (vs >= 0) continue;
1519                 PetscCheck(vu >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %" PetscInt_FMT " in split or unsplit points of depth %" PetscInt_FMT, econe[r], dep - 2);
1520                 hasUnsplit = PETSC_TRUE;
1521               }
1522             }
1523           }
1524         }
1525         PetscCall(DMPlexSetCone(sdm, splitp, &coneNew[2]));
1526         PetscCall(DMPlexSetConeOrientation(sdm, splitp, ornt));
1527         /* Face support */
1528         PetscInt vals[2];
1529 
1530         PetscCall(DMLabelGetValue(label, support[0], &vals[0]));
1531         if (supportSize > 1) PetscCall(DMLabelGetValue(label, support[1], &vals[1]));
1532         else vals[1] = -vals[0];
1533         PetscCheck(vals[0] * vals[1] < 0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid support labels %" PetscInt_FMT " %" PetscInt_FMT, vals[0], vals[1]);
1534 
1535         for (s = 0; s < 2; ++s) {
1536           if (s >= supportSize) {
1537             if (vals[s] < 0) {
1538               /* Ghost old face:   Replace negative side cell with cohesive cell */
1539               PetscCall(DMPlexInsertSupport(sdm, newp, s, hybcell));
1540             } else {
1541               /* Ghost new face:   Replace positive side cell with cohesive cell */
1542               PetscCall(DMPlexInsertSupport(sdm, splitp, s, hybcell));
1543             }
1544           } else {
1545             if (vals[s] < 0) {
1546               /* Split old face:   Replace negative side cell with cohesive cell */
1547               PetscCall(DMPlexInsertSupport(sdm, newp, s, hybcell));
1548             } else {
1549               /* Split new face:   Replace positive side cell with cohesive cell */
1550               PetscCall(DMPlexInsertSupport(sdm, splitp, s, hybcell));
1551             }
1552           }
1553         }
1554         /* Get orientation for cohesive face using the positive side cell */
1555         {
1556           const PetscInt *ncone, *nconeO;
1557           PetscInt        nconeSize, nc, ocell;
1558           PetscBool       flip = PETSC_FALSE;
1559 
1560           if (supportSize > 1) {
1561             ocell = vals[0] < 0 ? support[1] : support[0];
1562           } else {
1563             ocell = support[0];
1564             flip  = vals[0] < 0 ? PETSC_TRUE : PETSC_FALSE;
1565           }
1566           PetscCall(DMPlexGetConeSize(dm, ocell, &nconeSize));
1567           PetscCall(DMPlexGetCone(dm, ocell, &ncone));
1568           PetscCall(DMPlexGetConeOrientation(dm, ocell, &nconeO));
1569           for (nc = 0; nc < nconeSize; ++nc) {
1570             if (ncone[nc] == oldp) {
1571               coneONew[0] = flip ? -(nconeO[nc] + 1) : nconeO[nc];
1572               break;
1573             }
1574           }
1575           PetscCheck(nc < nconeSize, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate face %" PetscInt_FMT " in neighboring cell %" PetscInt_FMT, oldp, ocell);
1576         }
1577         /* Cohesive cell:    Old and new split face, then new cohesive faces */
1578         {
1579           const PetscInt No = DMPolytopeTypeGetNumArrangements(ct) / 2;
1580           PetscCheck((coneONew[0] >= -No) && (coneONew[0] < No), PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid %s orientation %" PetscInt_FMT, DMPolytopeTypes[ct], coneONew[0]);
1581         }
1582         const PetscInt *arr = DMPolytopeTypeGetArrangement(ct, coneONew[0]);
1583 
1584         coneNew[0]  = newp; /* Extracted negative side orientation above */
1585         coneNew[1]  = splitp;
1586         coneONew[1] = coneONew[0];
1587         for (q = 0; q < coneSize; ++q) {
1588           /* Hybrid faces must follow order from oriented end face */
1589           const PetscInt qa = arr[q * 2 + 0];
1590           const PetscInt qo = arr[q * 2 + 1];
1591           DMPolytopeType ft = dep == 2 ? DM_POLYTOPE_SEGMENT : DM_POLYTOPE_POINT;
1592 
1593           PetscCall(PetscFindInt(cone[qa], numSplitPoints[dep - 1], splitPoints[dep - 1], &v));
1594           if (v < 0) {
1595             PetscCall(PetscFindInt(cone[qa], numUnsplitPoints[dep - 1], unsplitPoints[dep - 1], &v));
1596             coneNew[2 + q] = v + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep - 1];
1597           } else {
1598             coneNew[2 + q] = v + pMaxNew[dep] + numSplitPoints[dep];
1599           }
1600           coneONew[2 + q] = DMPolytopeTypeComposeOrientation(ft, qo, ornt[qa]);
1601         }
1602         PetscCall(DMPlexSetCone(sdm, hybcell, coneNew));
1603         PetscCall(DMPlexSetConeOrientation(sdm, hybcell, coneONew));
1604         /* Label the hybrid cells on the boundary of the split */
1605         if (hasUnsplit) PetscCall(DMLabelSetValue(label, -hybcell, dim));
1606       } else if (dep == 0) {
1607         const PetscInt hybedge = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1];
1608 
1609         /* Split old vertex: Edges in old split faces and new cohesive edge */
1610         for (e = 0, qn = 0; e < supportSize; ++e) {
1611           PetscInt val;
1612 
1613           PetscCall(DMLabelGetValue(label, support[e], &val));
1614           if ((val == 1) || (val == (shift + 1))) supportNew[qn++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1615         }
1616         supportNew[qn] = hybedge;
1617         PetscCall(DMPlexSetSupport(sdm, newp, supportNew));
1618         /* Split new vertex: Edges in new split faces and new cohesive edge */
1619         for (e = 0, qp = 0; e < supportSize; ++e) {
1620           PetscInt val, edge;
1621 
1622           PetscCall(DMLabelGetValue(label, support[e], &val));
1623           if (val == 1) {
1624             PetscCall(PetscFindInt(support[e], numSplitPoints[dep + 1], splitPoints[dep + 1], &edge));
1625             PetscCheck(edge >= 0, comm, PETSC_ERR_ARG_WRONG, "Edge %" PetscInt_FMT " is not a split edge", support[e]);
1626             supportNew[qp++] = edge + pMaxNew[dep + 1];
1627           } else if (val == -(shift + 1)) {
1628             supportNew[qp++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1629           }
1630         }
1631         supportNew[qp] = hybedge;
1632         PetscCall(DMPlexSetSupport(sdm, splitp, supportNew));
1633         /* Hybrid edge:    Old and new split vertex */
1634         coneNew[0] = newp;
1635         coneNew[1] = splitp;
1636         PetscCall(DMPlexSetCone(sdm, hybedge, coneNew));
1637         for (e = 0, qf = 0; e < supportSize; ++e) {
1638           PetscInt val, edge;
1639 
1640           PetscCall(DMLabelGetValue(label, support[e], &val));
1641           if (val == 1) {
1642             PetscCall(PetscFindInt(support[e], numSplitPoints[dep + 1], splitPoints[dep + 1], &edge));
1643             PetscCheck(edge >= 0, comm, PETSC_ERR_ARG_WRONG, "Edge %" PetscInt_FMT " is not a split edge", support[e]);
1644             supportNew[qf++] = edge + pMaxNew[dep + 2] + numSplitPoints[dep + 2];
1645           }
1646         }
1647         PetscCall(DMPlexSetSupport(sdm, hybedge, supportNew));
1648       } else if (dep == dim - 2) {
1649         const PetscInt hybface = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1];
1650 
1651         /* Split old edge:   old vertices in cone so no change */
1652         /* Split new edge:   new vertices in cone */
1653         for (q = 0; q < coneSize; ++q) {
1654           PetscCall(PetscFindInt(cone[q], numSplitPoints[dep - 1], splitPoints[dep - 1], &v));
1655           if (v < 0) {
1656             PetscCall(PetscFindInt(cone[q], numUnsplitPoints[dep - 1], unsplitPoints[dep - 1], &v));
1657             PetscCheck(v >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %" PetscInt_FMT " in split or unsplit points of depth %" PetscInt_FMT, cone[q], dep - 1);
1658             coneNew[q] = DMPlexShiftPoint_Internal(cone[q], depth, depthShift) /*cone[q] + depthOffset[dep-1]*/;
1659           } else {
1660             coneNew[q] = v + pMaxNew[dep - 1];
1661           }
1662         }
1663         PetscCall(DMPlexSetCone(sdm, splitp, coneNew));
1664         /* Split old edge: Faces in positive side cells and old split faces */
1665         for (e = 0, q = 0; e < supportSize; ++e) {
1666           PetscInt val;
1667 
1668           PetscCall(DMLabelGetValue(label, support[e], &val));
1669           if (val == dim - 1) {
1670             supportNew[q++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1671           } else if (val == (shift + dim - 1)) {
1672             supportNew[q++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1673           }
1674         }
1675         supportNew[q++] = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1];
1676         PetscCall(DMPlexSetSupport(sdm, newp, supportNew));
1677         /* Split new edge: Faces in negative side cells and new split faces */
1678         for (e = 0, q = 0; e < supportSize; ++e) {
1679           PetscInt val, face;
1680 
1681           PetscCall(DMLabelGetValue(label, support[e], &val));
1682           if (val == dim - 1) {
1683             PetscCall(PetscFindInt(support[e], numSplitPoints[dep + 1], splitPoints[dep + 1], &face));
1684             PetscCheck(face >= 0, comm, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " is not a split face", support[e]);
1685             supportNew[q++] = face + pMaxNew[dep + 1];
1686           } else if (val == -(shift + dim - 1)) {
1687             supportNew[q++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1688           }
1689         }
1690         supportNew[q++] = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1];
1691         PetscCall(DMPlexSetSupport(sdm, splitp, supportNew));
1692         /* Hybrid face */
1693         coneNew[0] = newp;
1694         coneNew[1] = splitp;
1695         for (v = 0; v < coneSize; ++v) {
1696           PetscInt vertex;
1697           PetscCall(PetscFindInt(cone[v], numSplitPoints[dep - 1], splitPoints[dep - 1], &vertex));
1698           if (vertex < 0) {
1699             PetscCall(PetscFindInt(cone[v], numUnsplitPoints[dep - 1], unsplitPoints[dep - 1], &vertex));
1700             PetscCheck(vertex >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %" PetscInt_FMT " in split or unsplit points of depth %" PetscInt_FMT, cone[v], dep - 1);
1701             coneNew[2 + v] = vertex + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep - 1];
1702           } else {
1703             coneNew[2 + v] = vertex + pMaxNew[dep] + numSplitPoints[dep];
1704           }
1705         }
1706         PetscCall(DMPlexSetCone(sdm, hybface, coneNew));
1707         for (e = 0, qf = 0; e < supportSize; ++e) {
1708           PetscInt val, face;
1709 
1710           PetscCall(DMLabelGetValue(label, support[e], &val));
1711           if (val == dim - 1) {
1712             PetscCall(PetscFindInt(support[e], numSplitPoints[dep + 1], splitPoints[dep + 1], &face));
1713             PetscCheck(face >= 0, comm, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " is not a split face", support[e]);
1714             supportNew[qf++] = face + pMaxNew[dep + 2] + numSplitPoints[dep + 2];
1715           }
1716         }
1717         PetscCall(DMPlexSetSupport(sdm, hybface, supportNew));
1718       }
1719     }
1720   }
1721   for (dep = 0; dep <= depth; ++dep) {
1722     for (p = 0; p < numUnsplitPoints[dep]; ++p) {
1723       const PetscInt  oldp = unsplitPoints[dep][p];
1724       const PetscInt  newp = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*oldp + depthOffset[dep]*/;
1725       const PetscInt *cone, *support;
1726       PetscInt        coneSize, supportSize, supportSizeNew, q, qf, e, f, s;
1727 
1728       PetscCall(DMPlexGetConeSize(dm, oldp, &coneSize));
1729       PetscCall(DMPlexGetCone(dm, oldp, &cone));
1730       PetscCall(DMPlexGetSupportSize(dm, oldp, &supportSize));
1731       PetscCall(DMPlexGetSupport(dm, oldp, &support));
1732       if (dep == 0) {
1733         const PetscInt hybedge = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1] + numSplitPoints[dep];
1734 
1735         /* Unsplit vertex */
1736         PetscCall(DMPlexGetSupportSize(sdm, newp, &supportSizeNew));
1737         for (s = 0, q = 0; s < supportSize; ++s) {
1738           supportNew[q++] = DMPlexShiftPoint_Internal(support[s], depth, depthShift) /*support[s] + depthOffset[dep+1]*/;
1739           PetscCall(PetscFindInt(support[s], numSplitPoints[dep + 1], splitPoints[dep + 1], &e));
1740           if (e >= 0) supportNew[q++] = e + pMaxNew[dep + 1];
1741         }
1742         supportNew[q++] = hybedge;
1743         supportNew[q++] = hybedge;
1744         PetscCheck(q == supportSizeNew, comm, PETSC_ERR_ARG_WRONG, "Support size %" PetscInt_FMT " != %" PetscInt_FMT " for vertex %" PetscInt_FMT, q, supportSizeNew, newp);
1745         PetscCall(DMPlexSetSupport(sdm, newp, supportNew));
1746         /* Hybrid edge */
1747         coneNew[0] = newp;
1748         coneNew[1] = newp;
1749         PetscCall(DMPlexSetCone(sdm, hybedge, coneNew));
1750         for (e = 0, qf = 0; e < supportSize; ++e) {
1751           PetscInt val, edge;
1752 
1753           PetscCall(DMLabelGetValue(label, support[e], &val));
1754           if (val == 1) {
1755             PetscCall(PetscFindInt(support[e], numSplitPoints[dep + 1], splitPoints[dep + 1], &edge));
1756             PetscCheck(edge >= 0, comm, PETSC_ERR_ARG_WRONG, "Edge %" PetscInt_FMT " is not a split edge", support[e]);
1757             supportNew[qf++] = edge + pMaxNew[dep + 2] + numSplitPoints[dep + 2];
1758           } else if (val == (shift2 + 1)) {
1759             PetscCall(PetscFindInt(support[e], numUnsplitPoints[dep + 1], unsplitPoints[dep + 1], &edge));
1760             PetscCheck(edge >= 0, comm, PETSC_ERR_ARG_WRONG, "Edge %" PetscInt_FMT " is not a unsplit edge", support[e]);
1761             supportNew[qf++] = edge + pMaxNew[dep + 2] + numSplitPoints[dep + 2] + numSplitPoints[dep + 1];
1762           }
1763         }
1764         PetscCall(DMPlexSetSupport(sdm, hybedge, supportNew));
1765       } else if (dep == dim - 2) {
1766         const PetscInt hybface = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1] + numSplitPoints[dep];
1767 
1768         /* Unsplit edge: Faces into original edge, split face, and hybrid face twice */
1769         for (f = 0, qf = 0; f < supportSize; ++f) {
1770           PetscInt val, face;
1771 
1772           PetscCall(DMLabelGetValue(label, support[f], &val));
1773           if (val == dim - 1) {
1774             PetscCall(PetscFindInt(support[f], numSplitPoints[dep + 1], splitPoints[dep + 1], &face));
1775             PetscCheck(face >= 0, comm, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " is not a split face", support[f]);
1776             supportNew[qf++] = DMPlexShiftPoint_Internal(support[f], depth, depthShift) /*support[f] + depthOffset[dep+1]*/;
1777             supportNew[qf++] = face + pMaxNew[dep + 1];
1778           } else {
1779             supportNew[qf++] = DMPlexShiftPoint_Internal(support[f], depth, depthShift) /*support[f] + depthOffset[dep+1]*/;
1780           }
1781         }
1782         supportNew[qf++] = hybface;
1783         supportNew[qf++] = hybface;
1784         PetscCall(DMPlexGetSupportSize(sdm, newp, &supportSizeNew));
1785         PetscCheck(qf == supportSizeNew, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Support size for unsplit edge %" PetscInt_FMT " is %" PetscInt_FMT " != %" PetscInt_FMT, newp, qf, supportSizeNew);
1786         PetscCall(DMPlexSetSupport(sdm, newp, supportNew));
1787         /* Add hybrid face */
1788         coneNew[0] = newp;
1789         coneNew[1] = newp;
1790         PetscCall(PetscFindInt(cone[0], numUnsplitPoints[dep - 1], unsplitPoints[dep - 1], &v));
1791         PetscCheck(v >= 0, comm, PETSC_ERR_ARG_WRONG, "Vertex %" PetscInt_FMT " is not an unsplit vertex", cone[0]);
1792         coneNew[2] = v + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep - 1];
1793         PetscCall(PetscFindInt(cone[1], numUnsplitPoints[dep - 1], unsplitPoints[dep - 1], &v));
1794         PetscCheck(v >= 0, comm, PETSC_ERR_ARG_WRONG, "Vertex %" PetscInt_FMT " is not an unsplit vertex", cone[1]);
1795         coneNew[3] = v + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep - 1];
1796         PetscCall(DMPlexSetCone(sdm, hybface, coneNew));
1797         for (f = 0, qf = 0; f < supportSize; ++f) {
1798           PetscInt val, face;
1799 
1800           PetscCall(DMLabelGetValue(label, support[f], &val));
1801           if (val == dim - 1) {
1802             PetscCall(PetscFindInt(support[f], numSplitPoints[dep + 1], splitPoints[dep + 1], &face));
1803             supportNew[qf++] = face + pMaxNew[dep + 2] + numSplitPoints[dep + 2];
1804           }
1805         }
1806         PetscCall(DMPlexGetSupportSize(sdm, hybface, &supportSizeNew));
1807         PetscCheck(qf == supportSizeNew, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Support size for hybrid face %" PetscInt_FMT " is %" PetscInt_FMT " != %" PetscInt_FMT, hybface, qf, supportSizeNew);
1808         PetscCall(DMPlexSetSupport(sdm, hybface, supportNew));
1809       }
1810     }
1811   }
1812   /* Step 6b: Replace split points in negative side cones */
1813   for (sp = 0; sp < numSP; ++sp) {
1814     PetscInt        dep = values[sp];
1815     IS              pIS;
1816     PetscInt        numPoints;
1817     const PetscInt *points;
1818 
1819     if (dep >= 0) continue;
1820     PetscCall(DMLabelGetStratumIS(label, dep, &pIS));
1821     if (!pIS) continue;
1822     dep = -dep - shift;
1823     PetscCall(ISGetLocalSize(pIS, &numPoints));
1824     PetscCall(ISGetIndices(pIS, &points));
1825     for (p = 0; p < numPoints; ++p) {
1826       const PetscInt  oldp = points[p];
1827       const PetscInt  newp = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*depthOffset[dep] + oldp*/;
1828       const PetscInt *cone;
1829       PetscInt        coneSize, c;
1830       /* PetscBool       replaced = PETSC_FALSE; */
1831 
1832       /* Negative edge: replace split vertex */
1833       /* Negative cell: replace split face */
1834       PetscCall(DMPlexGetConeSize(sdm, newp, &coneSize));
1835       PetscCall(DMPlexGetCone(sdm, newp, &cone));
1836       for (c = 0; c < coneSize; ++c) {
1837         const PetscInt coldp = DMPlexShiftPointInverse_Internal(cone[c], depth, depthShift);
1838         PetscInt       csplitp, cp, val;
1839 
1840         PetscCall(DMLabelGetValue(label, coldp, &val));
1841         if (val == dep - 1) {
1842           PetscCall(PetscFindInt(coldp, numSplitPoints[dep - 1], splitPoints[dep - 1], &cp));
1843           PetscCheck(cp >= 0, comm, PETSC_ERR_ARG_WRONG, "Point %" PetscInt_FMT " is not a split point of dimension %" PetscInt_FMT, oldp, dep - 1);
1844           csplitp = pMaxNew[dep - 1] + cp;
1845           PetscCall(DMPlexInsertCone(sdm, newp, c, csplitp));
1846           /* replaced = PETSC_TRUE; */
1847         }
1848       }
1849       /* Cells with only a vertex or edge on the submesh have no replacement */
1850       /* PetscCheck(replaced,comm, PETSC_ERR_ARG_WRONG, "The cone of point %d does not contain split points", oldp); */
1851     }
1852     PetscCall(ISRestoreIndices(pIS, &points));
1853     PetscCall(ISDestroy(&pIS));
1854   }
1855   PetscCall(DMPlexReorderCohesiveSupports(sdm));
1856   /* Step 7: Coordinates */
1857   PetscCall(DMPlexShiftCoordinates_Internal(dm, depthShift, sdm));
1858   PetscCall(DMGetCoordinateSection(sdm, &coordSection));
1859   PetscCall(DMGetCoordinatesLocal(sdm, &coordinates));
1860   PetscCall(VecGetArray(coordinates, &coords));
1861   for (v = 0; v < (numSplitPoints ? numSplitPoints[0] : 0); ++v) {
1862     const PetscInt newp   = DMPlexShiftPoint_Internal(splitPoints[0][v], depth, depthShift) /*depthOffset[0] + splitPoints[0][v]*/;
1863     const PetscInt splitp = pMaxNew[0] + v;
1864     PetscInt       dof, off, soff, d;
1865 
1866     PetscCall(PetscSectionGetDof(coordSection, newp, &dof));
1867     PetscCall(PetscSectionGetOffset(coordSection, newp, &off));
1868     PetscCall(PetscSectionGetOffset(coordSection, splitp, &soff));
1869     for (d = 0; d < dof; ++d) coords[soff + d] = coords[off + d];
1870   }
1871   PetscCall(VecRestoreArray(coordinates, &coords));
1872   /* Step 8: SF, if I can figure this out we can split the mesh in parallel */
1873   PetscCall(DMPlexShiftSF_Internal(dm, depthShift, sdm));
1874   /*   TODO We need to associate the ghost points with the correct replica */
1875   /* Step 9: Labels */
1876   PetscCall(DMPlexShiftLabels_Internal(dm, depthShift, sdm));
1877   PetscCall(DMPlexCreateVTKLabel_Internal(dm, PETSC_FALSE, sdm));
1878   PetscCall(DMGetNumLabels(sdm, &numLabels));
1879   for (dep = 0; dep <= depth; ++dep) {
1880     for (p = 0; p < numSplitPoints[dep]; ++p) {
1881       const PetscInt newp   = DMPlexShiftPoint_Internal(splitPoints[dep][p], depth, depthShift) /*depthOffset[dep] + splitPoints[dep][p]*/;
1882       const PetscInt splitp = pMaxNew[dep] + p;
1883       PetscInt       l;
1884 
1885       if (splitLabel) {
1886         const PetscInt val = 100 + dep;
1887 
1888         PetscCall(DMLabelSetValue(splitLabel, newp, val));
1889         PetscCall(DMLabelSetValue(splitLabel, splitp, -val));
1890       }
1891       for (l = 0; l < numLabels; ++l) {
1892         DMLabel     mlabel;
1893         const char *lname;
1894         PetscInt    val;
1895         PetscBool   isDepth;
1896 
1897         PetscCall(DMGetLabelName(sdm, l, &lname));
1898         PetscCall(PetscStrcmp(lname, "depth", &isDepth));
1899         if (isDepth) continue;
1900         PetscCall(DMGetLabel(sdm, lname, &mlabel));
1901         PetscCall(DMLabelGetValue(mlabel, newp, &val));
1902         if (val >= 0) PetscCall(DMLabelSetValue(mlabel, splitp, val));
1903       }
1904     }
1905   }
1906   for (sp = 0; sp < numSP; ++sp) {
1907     const PetscInt dep = values[sp];
1908 
1909     if ((dep < 0) || (dep > depth)) continue;
1910     if (splitIS[dep]) PetscCall(ISRestoreIndices(splitIS[dep], &splitPoints[dep]));
1911     PetscCall(ISDestroy(&splitIS[dep]));
1912     if (unsplitIS[dep]) PetscCall(ISRestoreIndices(unsplitIS[dep], &unsplitPoints[dep]));
1913     PetscCall(ISDestroy(&unsplitIS[dep]));
1914   }
1915   if (ghostIS) PetscCall(ISRestoreIndices(ghostIS, &ghostPoints));
1916   PetscCall(ISDestroy(&ghostIS));
1917   if (label) {
1918     PetscCall(ISRestoreIndices(valueIS, &values));
1919     PetscCall(ISDestroy(&valueIS));
1920   }
1921   for (d = 0; d <= depth; ++d) {
1922     PetscCall(DMPlexGetDepthStratum(sdm, d, NULL, &pEnd));
1923     pMaxNew[d] = pEnd - numHybridPoints[d] - numHybridPointsOld[d];
1924   }
1925   PetscCall(PetscFree3(coneNew, coneONew, supportNew));
1926   PetscCall(PetscFree5(depthMax, depthEnd, depthShift, pMaxNew, numHybridPointsOld));
1927   PetscCall(PetscFree7(splitIS, unsplitIS, numSplitPoints, numUnsplitPoints, numHybridPoints, splitPoints, unsplitPoints));
1928   PetscFunctionReturn(PETSC_SUCCESS);
1929 }
1930 
1931 /*@C
1932   DMPlexConstructCohesiveCells - Construct cohesive cells which split the face along an internal interface
1933 
1934   Collective
1935 
1936   Input Parameters:
1937 + dm    - The original `DM`
1938 - label - The `DMLabel` specifying the boundary faces (this could be auto-generated)
1939 
1940   Output Parameters:
1941 + splitLabel - The `DMLabel` containing the split points, or `NULL` if no output is desired
1942 - dmSplit    - The new `DM`
1943 
1944   Level: developer
1945 
1946 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMCreate()`, `DMPlexLabelCohesiveComplete()`
1947 @*/
1948 PetscErrorCode DMPlexConstructCohesiveCells(DM dm, DMLabel label, DMLabel splitLabel, DM *dmSplit)
1949 {
1950   DM       sdm;
1951   PetscInt dim;
1952 
1953   PetscFunctionBegin;
1954   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1955   PetscAssertPointer(dmSplit, 4);
1956   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &sdm));
1957   PetscCall(DMSetType(sdm, DMPLEX));
1958   PetscCall(DMGetDimension(dm, &dim));
1959   PetscCall(DMSetDimension(sdm, dim));
1960   switch (dim) {
1961   case 2:
1962   case 3:
1963     PetscCall(DMPlexConstructCohesiveCells_Internal(dm, label, splitLabel, sdm));
1964     break;
1965   default:
1966     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cannot construct cohesive cells for dimension %" PetscInt_FMT, dim);
1967   }
1968   *dmSplit = sdm;
1969   PetscFunctionReturn(PETSC_SUCCESS);
1970 }
1971 
1972 /* Returns the side of the surface for a given cell with a face on the surface */
1973 static PetscErrorCode GetSurfaceSide_Static(DM dm, DM subdm, PetscInt numSubpoints, const PetscInt *subpoints, PetscInt cell, PetscInt face, PetscBool *pos)
1974 {
1975   const PetscInt *cone, *ornt;
1976   PetscInt        dim, coneSize, c;
1977 
1978   PetscFunctionBegin;
1979   *pos = PETSC_TRUE;
1980   PetscCall(DMGetDimension(dm, &dim));
1981   PetscCall(DMPlexGetConeSize(dm, cell, &coneSize));
1982   PetscCall(DMPlexGetCone(dm, cell, &cone));
1983   PetscCall(DMPlexGetConeOrientation(dm, cell, &ornt));
1984   for (c = 0; c < coneSize; ++c) {
1985     if (cone[c] == face) {
1986       PetscInt o = ornt[c];
1987 
1988       if (subdm) {
1989         const PetscInt *subcone, *subornt;
1990         PetscInt        subpoint, subface, subconeSize, sc;
1991 
1992         PetscCall(PetscFindInt(cell, numSubpoints, subpoints, &subpoint));
1993         PetscCall(PetscFindInt(face, numSubpoints, subpoints, &subface));
1994         PetscCall(DMPlexGetConeSize(subdm, subpoint, &subconeSize));
1995         PetscCall(DMPlexGetCone(subdm, subpoint, &subcone));
1996         PetscCall(DMPlexGetConeOrientation(subdm, subpoint, &subornt));
1997         for (sc = 0; sc < subconeSize; ++sc) {
1998           if (subcone[sc] == subface) {
1999             o = subornt[0];
2000             break;
2001           }
2002         }
2003         PetscCheck(sc < subconeSize, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not find subpoint %" PetscInt_FMT " (%" PetscInt_FMT ") in cone for subpoint %" PetscInt_FMT " (%" PetscInt_FMT ")", subface, face, subpoint, cell);
2004       }
2005       if (o >= 0) *pos = PETSC_TRUE;
2006       else *pos = PETSC_FALSE;
2007       break;
2008     }
2009   }
2010   PetscCheck(c != coneSize, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Cell %" PetscInt_FMT " in split face %" PetscInt_FMT " support does not have it in the cone", cell, face);
2011   PetscFunctionReturn(PETSC_SUCCESS);
2012 }
2013 
2014 static PetscErrorCode CheckFaultEdge_Private(DM dm, DMLabel label)
2015 {
2016   IS              facePosIS, faceNegIS, dimIS;
2017   const PetscInt *points;
2018   PetscInt        dim, numPoints, p, shift = 100, shift2 = 200;
2019 
2020   PetscFunctionBegin;
2021   PetscCall(DMGetDimension(dm, &dim));
2022   /* If any faces touching the fault divide cells on either side, split them */
2023   PetscCall(DMLabelGetStratumIS(label, shift + dim - 1, &facePosIS));
2024   PetscCall(DMLabelGetStratumIS(label, -(shift + dim - 1), &faceNegIS));
2025   if (!facePosIS || !faceNegIS) {
2026     PetscCall(ISDestroy(&facePosIS));
2027     PetscCall(ISDestroy(&faceNegIS));
2028     PetscFunctionReturn(PETSC_SUCCESS);
2029   }
2030   PetscCall(ISExpand(facePosIS, faceNegIS, &dimIS));
2031   PetscCall(ISDestroy(&facePosIS));
2032   PetscCall(ISDestroy(&faceNegIS));
2033   PetscCall(ISGetLocalSize(dimIS, &numPoints));
2034   PetscCall(ISGetIndices(dimIS, &points));
2035   for (p = 0; p < numPoints; ++p) {
2036     const PetscInt  point = points[p];
2037     const PetscInt *support;
2038     PetscInt        supportSize, valA, valB;
2039 
2040     PetscCall(DMPlexGetSupportSize(dm, point, &supportSize));
2041     if (supportSize != 2) continue;
2042     PetscCall(DMPlexGetSupport(dm, point, &support));
2043     PetscCall(DMLabelGetValue(label, support[0], &valA));
2044     PetscCall(DMLabelGetValue(label, support[1], &valB));
2045     if ((valA == -1) || (valB == -1)) continue;
2046     if (valA * valB > 0) continue;
2047     /* Check that this face is not incident on only unsplit faces, meaning has at least one split face */
2048     {
2049       PetscInt *closure = NULL;
2050       PetscBool split   = PETSC_FALSE;
2051       PetscInt  closureSize, cl;
2052 
2053       PetscCall(DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
2054       for (cl = 0; cl < closureSize * 2; cl += 2) {
2055         PetscCall(DMLabelGetValue(label, closure[cl], &valA));
2056         if ((valA >= 0) && (valA <= dim)) {
2057           split = PETSC_TRUE;
2058           break;
2059         }
2060       }
2061       PetscCall(DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
2062       if (!split) continue;
2063     }
2064     /* Split the face */
2065     PetscCall(DMLabelGetValue(label, point, &valA));
2066     PetscCall(DMLabelClearValue(label, point, valA));
2067     PetscCall(DMLabelSetValue(label, point, dim - 1));
2068     /* Label its closure:
2069       unmarked: label as unsplit
2070       incident: relabel as split
2071       split:    do nothing
2072     */
2073     {
2074       PetscInt *closure = NULL;
2075       PetscInt  closureSize, cl, dep;
2076 
2077       PetscCall(DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
2078       for (cl = 0; cl < closureSize * 2; cl += 2) {
2079         PetscCall(DMLabelGetValue(label, closure[cl], &valA));
2080         if (valA == -1) { /* Mark as unsplit */
2081           PetscCall(DMPlexGetPointDepth(dm, closure[cl], &dep));
2082           PetscCall(DMLabelSetValue(label, closure[cl], shift2 + dep));
2083         } else if (((valA >= shift) && (valA < shift2)) || ((valA <= -shift) && (valA > -shift2))) {
2084           PetscCall(DMPlexGetPointDepth(dm, closure[cl], &dep));
2085           PetscCall(DMLabelClearValue(label, closure[cl], valA));
2086           PetscCall(DMLabelSetValue(label, closure[cl], dep));
2087         }
2088       }
2089       PetscCall(DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
2090     }
2091   }
2092   PetscCall(ISRestoreIndices(dimIS, &points));
2093   PetscCall(ISDestroy(&dimIS));
2094   PetscFunctionReturn(PETSC_SUCCESS);
2095 }
2096 
2097 /*@
2098   DMPlexLabelCohesiveComplete - Starting with a label marking points on an internal surface, we add all other mesh pieces
2099   to complete the surface
2100 
2101   Input Parameters:
2102 + dm     - The `DM`
2103 . label  - A `DMLabel` marking the surface
2104 . blabel - A `DMLabel` marking the vertices on the boundary which will not be duplicated, or `NULL` to find them automatically
2105 . bvalue - Value of `DMLabel` marking the vertices on the boundary
2106 . flip   - Flag to flip the submesh normal and replace points on the other side
2107 - subdm  - The `DM` associated with the label, or `NULL`
2108 
2109   Output Parameter:
2110 . label - A `DMLabel` marking all surface points
2111 
2112   Level: developer
2113 
2114   Note:
2115   The vertices in blabel are called "unsplit" in the terminology from hybrid cell creation.
2116 
2117 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexConstructCohesiveCells()`, `DMPlexLabelComplete()`
2118 @*/
2119 PetscErrorCode DMPlexLabelCohesiveComplete(DM dm, DMLabel label, DMLabel blabel, PetscInt bvalue, PetscBool flip, DM subdm)
2120 {
2121   DMLabel         depthLabel;
2122   IS              dimIS, subpointIS = NULL;
2123   const PetscInt *points, *subpoints;
2124   const PetscInt  rev   = flip ? -1 : 1;
2125   PetscInt        shift = 100, shift2 = 200, shift3 = 300, dim, depth, numPoints, numSubpoints, p, val;
2126 
2127   PetscFunctionBegin;
2128   PetscCall(DMPlexGetDepth(dm, &depth));
2129   PetscCall(DMGetDimension(dm, &dim));
2130   PetscCall(DMPlexGetDepthLabel(dm, &depthLabel));
2131   if (subdm) {
2132     PetscCall(DMPlexGetSubpointIS(subdm, &subpointIS));
2133     if (subpointIS) {
2134       PetscCall(ISGetLocalSize(subpointIS, &numSubpoints));
2135       PetscCall(ISGetIndices(subpointIS, &subpoints));
2136     }
2137   }
2138   /* Mark cell on the fault, and its faces which touch the fault: cell orientation for face gives the side of the fault */
2139   PetscCall(DMLabelGetStratumIS(label, dim - 1, &dimIS));
2140   if (!dimIS) goto divide;
2141   PetscCall(ISGetLocalSize(dimIS, &numPoints));
2142   PetscCall(ISGetIndices(dimIS, &points));
2143   for (p = 0; p < numPoints; ++p) { /* Loop over fault faces */
2144     const PetscInt *support;
2145     PetscInt        supportSize, s;
2146 
2147     PetscCall(DMPlexGetSupportSize(dm, points[p], &supportSize));
2148 #if 0
2149     if (supportSize != 2) {
2150       const PetscInt *lp;
2151       PetscInt        Nlp, pind;
2152 
2153       /* Check that for a cell with a single support face, that face is in the SF */
2154       /*   THis check only works for the remote side. We would need root side information */
2155       PetscCall(PetscSFGetGraph(dm->sf, NULL, &Nlp, &lp, NULL));
2156       PetscCall(PetscFindInt(points[p], Nlp, lp, &pind));
2157       PetscCheck(pind >= 0,PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Split face %" PetscInt_FMT " has %" PetscInt_FMT " != 2 supports, and the face is not shared with another process", points[p], supportSize);
2158     }
2159 #endif
2160     PetscCall(DMPlexGetSupport(dm, points[p], &support));
2161     for (s = 0; s < supportSize; ++s) {
2162       const PetscInt *cone;
2163       PetscInt        coneSize, c;
2164       PetscBool       pos;
2165 
2166       PetscCall(GetSurfaceSide_Static(dm, subdm, numSubpoints, subpoints, support[s], points[p], &pos));
2167       if (pos) PetscCall(DMLabelSetValue(label, support[s], rev * (shift + dim)));
2168       else PetscCall(DMLabelSetValue(label, support[s], -rev * (shift + dim)));
2169       if (rev < 0) pos = !pos ? PETSC_TRUE : PETSC_FALSE;
2170       /* Put faces touching the fault in the label */
2171       PetscCall(DMPlexGetConeSize(dm, support[s], &coneSize));
2172       PetscCall(DMPlexGetCone(dm, support[s], &cone));
2173       for (c = 0; c < coneSize; ++c) {
2174         const PetscInt point = cone[c];
2175 
2176         PetscCall(DMLabelGetValue(label, point, &val));
2177         if (val == -1) {
2178           PetscInt *closure = NULL;
2179           PetscInt  closureSize, cl;
2180 
2181           PetscCall(DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
2182           for (cl = 0; cl < closureSize * 2; cl += 2) {
2183             const PetscInt clp  = closure[cl];
2184             PetscInt       bval = -1;
2185 
2186             PetscCall(DMLabelGetValue(label, clp, &val));
2187             if (blabel) PetscCall(DMLabelGetValue(blabel, clp, &bval));
2188             if ((val >= 0) && (val < dim - 1) && (bval < 0)) {
2189               PetscCall(DMLabelSetValue(label, point, pos == PETSC_TRUE ? shift + dim - 1 : -(shift + dim - 1)));
2190               break;
2191             }
2192           }
2193           PetscCall(DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
2194         }
2195       }
2196     }
2197   }
2198   PetscCall(ISRestoreIndices(dimIS, &points));
2199   PetscCall(ISDestroy(&dimIS));
2200   /* Mark boundary points as unsplit */
2201   if (blabel) {
2202     IS bdIS;
2203 
2204     PetscCall(DMLabelGetStratumIS(blabel, bvalue, &bdIS));
2205     PetscCall(ISGetLocalSize(bdIS, &numPoints));
2206     PetscCall(ISGetIndices(bdIS, &points));
2207     for (p = 0; p < numPoints; ++p) {
2208       const PetscInt point = points[p];
2209       PetscInt       val, bval;
2210 
2211       PetscCall(DMLabelGetValue(blabel, point, &bval));
2212       if (bval >= 0) {
2213         PetscCall(DMLabelGetValue(label, point, &val));
2214         if ((val < 0) || (val > dim)) {
2215           /* This could be a point added from splitting a vertex on an adjacent fault, otherwise its just wrong */
2216           PetscCall(DMLabelClearValue(blabel, point, bval));
2217         }
2218       }
2219     }
2220     for (p = 0; p < numPoints; ++p) {
2221       const PetscInt point = points[p];
2222       PetscInt       val, bval;
2223 
2224       PetscCall(DMLabelGetValue(blabel, point, &bval));
2225       if (bval >= 0) {
2226         const PetscInt *cone, *support;
2227         PetscInt        coneSize, supportSize, s, valA, valB, valE;
2228 
2229         /* Mark as unsplit */
2230         PetscCall(DMLabelGetValue(label, point, &val));
2231         PetscCheck(val >= 0 && val <= dim, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %" PetscInt_FMT " has label value %" PetscInt_FMT ", should be part of the fault", point, val);
2232         PetscCall(DMLabelClearValue(label, point, val));
2233         PetscCall(DMLabelSetValue(label, point, shift2 + val));
2234         /* Check for cross-edge
2235              A cross-edge has endpoints which are both on the boundary of the surface, but the edge itself is not. */
2236         if (val != 0) continue;
2237         PetscCall(DMPlexGetSupport(dm, point, &support));
2238         PetscCall(DMPlexGetSupportSize(dm, point, &supportSize));
2239         for (s = 0; s < supportSize; ++s) {
2240           PetscCall(DMPlexGetCone(dm, support[s], &cone));
2241           PetscCall(DMPlexGetConeSize(dm, support[s], &coneSize));
2242           PetscCheck(coneSize == 2, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Edge %" PetscInt_FMT " has %" PetscInt_FMT " vertices != 2", support[s], coneSize);
2243           PetscCall(DMLabelGetValue(blabel, cone[0], &valA));
2244           PetscCall(DMLabelGetValue(blabel, cone[1], &valB));
2245           PetscCall(DMLabelGetValue(blabel, support[s], &valE));
2246           if ((valE < 0) && (valA >= 0) && (valB >= 0) && (cone[0] != cone[1])) PetscCall(DMLabelSetValue(blabel, support[s], 2));
2247         }
2248       }
2249     }
2250     PetscCall(ISRestoreIndices(bdIS, &points));
2251     PetscCall(ISDestroy(&bdIS));
2252   }
2253   /* Mark ghost fault cells */
2254   {
2255     PetscSF         sf;
2256     const PetscInt *leaves;
2257     PetscInt        Nl, l;
2258 
2259     PetscCall(DMGetPointSF(dm, &sf));
2260     PetscCall(PetscSFGetGraph(sf, NULL, &Nl, &leaves, NULL));
2261     PetscCall(DMLabelGetStratumIS(label, dim - 1, &dimIS));
2262     if (!dimIS) goto divide;
2263     PetscCall(ISGetLocalSize(dimIS, &numPoints));
2264     PetscCall(ISGetIndices(dimIS, &points));
2265     if (Nl > 0) {
2266       for (p = 0; p < numPoints; ++p) {
2267         const PetscInt point = points[p];
2268         PetscInt       val;
2269 
2270         PetscCall(PetscFindInt(point, Nl, leaves, &l));
2271         if (l >= 0) {
2272           PetscInt *closure = NULL;
2273           PetscInt  closureSize, cl;
2274 
2275           PetscCall(DMLabelGetValue(label, point, &val));
2276           PetscCheck((val == dim - 1) || (val == shift2 + dim - 1), PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %" PetscInt_FMT " has label value %" PetscInt_FMT ", should be a fault face", point, val);
2277           PetscCall(DMLabelClearValue(label, point, val));
2278           PetscCall(DMLabelSetValue(label, point, shift3 + val));
2279           PetscCall(DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
2280           for (cl = 2; cl < closureSize * 2; cl += 2) {
2281             const PetscInt clp = closure[cl];
2282 
2283             PetscCall(DMLabelGetValue(label, clp, &val));
2284             PetscCheck(val != -1, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %" PetscInt_FMT " is missing from label, but is in the closure of a fault face", point);
2285             PetscCall(DMLabelClearValue(label, clp, val));
2286             PetscCall(DMLabelSetValue(label, clp, shift3 + val));
2287           }
2288           PetscCall(DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
2289         }
2290       }
2291     }
2292     PetscCall(ISRestoreIndices(dimIS, &points));
2293     PetscCall(ISDestroy(&dimIS));
2294   }
2295 divide:
2296   if (subpointIS) PetscCall(ISRestoreIndices(subpointIS, &subpoints));
2297   PetscCall(DMPlexLabelFaultHalo(dm, label));
2298   PetscCall(CheckFaultEdge_Private(dm, label));
2299   PetscFunctionReturn(PETSC_SUCCESS);
2300 }
2301 
2302 /* Check that no cell have all vertices on the fault */
2303 static PetscErrorCode DMPlexCheckValidSubmesh_Private(DM dm, DMLabel label, DM subdm)
2304 {
2305   IS              subpointIS;
2306   const PetscInt *dmpoints;
2307   PetscInt        defaultValue, cStart, cEnd, c, vStart, vEnd;
2308 
2309   PetscFunctionBegin;
2310   if (!label) PetscFunctionReturn(PETSC_SUCCESS);
2311   PetscCall(DMLabelGetDefaultValue(label, &defaultValue));
2312   PetscCall(DMPlexGetSubpointIS(subdm, &subpointIS));
2313   if (!subpointIS) PetscFunctionReturn(PETSC_SUCCESS);
2314   PetscCall(DMPlexGetHeightStratum(subdm, 0, &cStart, &cEnd));
2315   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
2316   PetscCall(ISGetIndices(subpointIS, &dmpoints));
2317   for (c = cStart; c < cEnd; ++c) {
2318     PetscBool invalidCell = PETSC_TRUE;
2319     PetscInt *closure     = NULL;
2320     PetscInt  closureSize, cl;
2321 
2322     PetscCall(DMPlexGetTransitiveClosure(dm, dmpoints[c], PETSC_TRUE, &closureSize, &closure));
2323     for (cl = 0; cl < closureSize * 2; cl += 2) {
2324       PetscInt value = 0;
2325 
2326       if ((closure[cl] < vStart) || (closure[cl] >= vEnd)) continue;
2327       PetscCall(DMLabelGetValue(label, closure[cl], &value));
2328       if (value == defaultValue) {
2329         invalidCell = PETSC_FALSE;
2330         break;
2331       }
2332     }
2333     PetscCall(DMPlexRestoreTransitiveClosure(dm, dmpoints[c], PETSC_TRUE, &closureSize, &closure));
2334     if (invalidCell) {
2335       PetscCall(ISRestoreIndices(subpointIS, &dmpoints));
2336       PetscCall(ISDestroy(&subpointIS));
2337       PetscCall(DMDestroy(&subdm));
2338       SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Ambiguous submesh. Cell %" PetscInt_FMT " has all of its vertices on the submesh.", dmpoints[c]);
2339     }
2340   }
2341   PetscCall(ISRestoreIndices(subpointIS, &dmpoints));
2342   PetscFunctionReturn(PETSC_SUCCESS);
2343 }
2344 
2345 /*@
2346   DMPlexCreateHybridMesh - Create a mesh with hybrid cells along an internal interface
2347 
2348   Collective
2349 
2350   Input Parameters:
2351 + dm      - The original `DM`
2352 . label   - The label specifying the interface vertices
2353 . bdlabel - The optional label specifying the interface boundary vertices
2354 - bdvalue - Value of optional label specifying the interface boundary vertices
2355 
2356   Output Parameters:
2357 + hybridLabel - The label fully marking the interface, or `NULL` if no output is desired
2358 . splitLabel  - The label containing the split points, or `NULL` if no output is desired
2359 . dmInterface - The new interface `DM`, or `NULL`
2360 - dmHybrid    - The new `DM` with cohesive cells
2361 
2362   Level: developer
2363 
2364   Note:
2365   The hybridLabel indicates what parts of the original mesh impinged on the division surface. For points
2366   directly on the division surface, they are labeled with their dimension, so an edge 7 on the division surface would be
2367   7 (1) in hybridLabel. For points that impinge from the positive side, they are labeled with 100+dim, so an edge 6 with
2368   one vertex 3 on the surface would be 6 (101) and 3 (0) in hybridLabel. If an edge 9 from the negative side of the
2369   surface also hits vertex 3, it would be 9 (-101) in hybridLabel.
2370 
2371   The splitLabel indicates what points in the new hybrid mesh were the result of splitting points in the original
2372   mesh. The label value is $\pm 100+dim$ for each point. For example, if two edges 10 and 14 in the hybrid resulting from
2373   splitting an edge in the original mesh, you would have 10 (101) and 14 (-101) in the splitLabel.
2374 
2375   The dmInterface is a `DM` built from the original division surface. It has a label which can be retrieved using
2376   `DMPlexGetSubpointMap()` which maps each point back to the point in the surface of the original mesh.
2377 
2378 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexConstructCohesiveCells()`, `DMPlexLabelCohesiveComplete()`, `DMPlexGetSubpointMap()`, `DMCreate()`
2379 @*/
2380 PetscErrorCode DMPlexCreateHybridMesh(DM dm, DMLabel label, DMLabel bdlabel, PetscInt bdvalue, DMLabel *hybridLabel, DMLabel *splitLabel, DM *dmInterface, DM *dmHybrid)
2381 {
2382   DM       idm;
2383   DMLabel  subpointMap, hlabel, slabel = NULL;
2384   PetscInt dim;
2385 
2386   PetscFunctionBegin;
2387   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2388   if (label) PetscAssertPointer(label, 2);
2389   if (bdlabel) PetscAssertPointer(bdlabel, 3);
2390   if (hybridLabel) PetscAssertPointer(hybridLabel, 5);
2391   if (splitLabel) PetscAssertPointer(splitLabel, 6);
2392   if (dmInterface) PetscAssertPointer(dmInterface, 7);
2393   PetscAssertPointer(dmHybrid, 8);
2394   PetscCall(DMGetDimension(dm, &dim));
2395   PetscCall(DMPlexCreateSubmesh(dm, label, 1, PETSC_FALSE, &idm));
2396   PetscCall(DMPlexCheckValidSubmesh_Private(dm, label, idm));
2397   PetscCall(DMPlexOrient(idm));
2398   PetscCall(DMPlexGetSubpointMap(idm, &subpointMap));
2399   PetscCall(DMLabelDuplicate(subpointMap, &hlabel));
2400   PetscCall(DMLabelClearStratum(hlabel, dim));
2401   if (splitLabel) {
2402     const char *name;
2403     char        sname[PETSC_MAX_PATH_LEN];
2404 
2405     PetscCall(PetscObjectGetName((PetscObject)hlabel, &name));
2406     PetscCall(PetscStrncpy(sname, name, sizeof(sname)));
2407     PetscCall(PetscStrlcat(sname, " split", sizeof(sname)));
2408     PetscCall(DMLabelCreate(PETSC_COMM_SELF, sname, &slabel));
2409   }
2410   PetscCall(DMPlexLabelCohesiveComplete(dm, hlabel, bdlabel, bdvalue, PETSC_FALSE, idm));
2411   if (dmInterface) {
2412     *dmInterface = idm;
2413   } else PetscCall(DMDestroy(&idm));
2414   PetscCall(DMPlexConstructCohesiveCells(dm, hlabel, slabel, dmHybrid));
2415   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_TRUE, *dmHybrid));
2416   if (hybridLabel) *hybridLabel = hlabel;
2417   else PetscCall(DMLabelDestroy(&hlabel));
2418   if (splitLabel) *splitLabel = slabel;
2419   {
2420     DM      cdm;
2421     DMLabel ctLabel;
2422 
2423     /* We need to somehow share the celltype label with the coordinate dm */
2424     PetscCall(DMGetCoordinateDM(*dmHybrid, &cdm));
2425     PetscCall(DMPlexGetCellTypeLabel(*dmHybrid, &ctLabel));
2426     PetscCall(DMSetLabel(cdm, ctLabel));
2427   }
2428   PetscFunctionReturn(PETSC_SUCCESS);
2429 }
2430 
2431 /* Here we need the explicit assumption that:
2432 
2433      For any marked cell, the marked vertices constitute a single face
2434 */
2435 static PetscErrorCode DMPlexMarkSubmesh_Uninterpolated(DM dm, DMLabel vertexLabel, PetscInt value, DMLabel subpointMap, PetscInt *numFaces, PetscInt *nFV, DM subdm)
2436 {
2437   IS              subvertexIS = NULL;
2438   const PetscInt *subvertices;
2439   PetscInt       *pStart, *pEnd, pSize;
2440   PetscInt        depth, dim, d, numSubVerticesInitial = 0, v;
2441 
2442   PetscFunctionBegin;
2443   *numFaces = 0;
2444   *nFV      = 0;
2445   PetscCall(DMPlexGetDepth(dm, &depth));
2446   PetscCall(DMGetDimension(dm, &dim));
2447   pSize = PetscMax(depth, dim) + 1;
2448   PetscCall(PetscMalloc2(pSize, &pStart, pSize, &pEnd));
2449   for (d = 0; d <= depth; ++d) PetscCall(DMPlexGetSimplexOrBoxCells(dm, depth - d, &pStart[d], &pEnd[d]));
2450   /* Loop over initial vertices and mark all faces in the collective star() */
2451   if (vertexLabel) PetscCall(DMLabelGetStratumIS(vertexLabel, value, &subvertexIS));
2452   if (subvertexIS) {
2453     PetscCall(ISGetSize(subvertexIS, &numSubVerticesInitial));
2454     PetscCall(ISGetIndices(subvertexIS, &subvertices));
2455   }
2456   for (v = 0; v < numSubVerticesInitial; ++v) {
2457     const PetscInt vertex = subvertices[v];
2458     PetscInt      *star   = NULL;
2459     PetscInt       starSize, s, numCells = 0, c;
2460 
2461     PetscCall(DMPlexGetTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star));
2462     for (s = 0; s < starSize * 2; s += 2) {
2463       const PetscInt point = star[s];
2464       if ((point >= pStart[depth]) && (point < pEnd[depth])) star[numCells++] = point;
2465     }
2466     for (c = 0; c < numCells; ++c) {
2467       const PetscInt cell    = star[c];
2468       PetscInt      *closure = NULL;
2469       PetscInt       closureSize, cl;
2470       PetscInt       cellLoc, numCorners = 0, faceSize = 0;
2471 
2472       PetscCall(DMLabelGetValue(subpointMap, cell, &cellLoc));
2473       if (cellLoc == 2) continue;
2474       PetscCheck(cellLoc < 0, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Cell %" PetscInt_FMT " has dimension %" PetscInt_FMT " in the surface label", cell, cellLoc);
2475       PetscCall(DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
2476       for (cl = 0; cl < closureSize * 2; cl += 2) {
2477         const PetscInt point = closure[cl];
2478         PetscInt       vertexLoc;
2479 
2480         if ((point >= pStart[0]) && (point < pEnd[0])) {
2481           ++numCorners;
2482           PetscCall(DMLabelGetValue(vertexLabel, point, &vertexLoc));
2483           if (vertexLoc == value) closure[faceSize++] = point;
2484         }
2485       }
2486       if (!*nFV) PetscCall(DMPlexGetNumFaceVertices(dm, dim, numCorners, nFV));
2487       PetscCheck(faceSize <= *nFV, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Invalid submesh: Too many vertices %" PetscInt_FMT " of an element on the surface", faceSize);
2488       if (faceSize == *nFV) {
2489         const PetscInt *cells = NULL;
2490         PetscInt        numCells, nc;
2491 
2492         ++(*numFaces);
2493         for (cl = 0; cl < faceSize; ++cl) PetscCall(DMLabelSetValue(subpointMap, closure[cl], 0));
2494         PetscCall(DMPlexGetJoin(dm, faceSize, closure, &numCells, &cells));
2495         for (nc = 0; nc < numCells; ++nc) PetscCall(DMLabelSetValue(subpointMap, cells[nc], 2));
2496         PetscCall(DMPlexRestoreJoin(dm, faceSize, closure, &numCells, &cells));
2497       }
2498       PetscCall(DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
2499     }
2500     PetscCall(DMPlexRestoreTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star));
2501   }
2502   if (subvertexIS) PetscCall(ISRestoreIndices(subvertexIS, &subvertices));
2503   PetscCall(ISDestroy(&subvertexIS));
2504   PetscCall(PetscFree2(pStart, pEnd));
2505   PetscFunctionReturn(PETSC_SUCCESS);
2506 }
2507 
2508 static PetscErrorCode DMPlexMarkSubmesh_Interpolated(DM dm, DMLabel vertexLabel, PetscInt value, PetscBool markedFaces, DMLabel subpointMap, DM subdm)
2509 {
2510   IS              subvertexIS = NULL;
2511   const PetscInt *subvertices;
2512   PetscInt       *pStart, *pEnd;
2513   PetscInt        dim, d, numSubVerticesInitial = 0, v;
2514 
2515   PetscFunctionBegin;
2516   PetscCall(DMGetDimension(dm, &dim));
2517   PetscCall(PetscMalloc2(dim + 1, &pStart, dim + 1, &pEnd));
2518   for (d = 0; d <= dim; ++d) PetscCall(DMPlexGetSimplexOrBoxCells(dm, dim - d, &pStart[d], &pEnd[d]));
2519   /* Loop over initial vertices and mark all faces in the collective star() */
2520   if (vertexLabel) {
2521     PetscCall(DMLabelGetStratumIS(vertexLabel, value, &subvertexIS));
2522     if (subvertexIS) {
2523       PetscCall(ISGetSize(subvertexIS, &numSubVerticesInitial));
2524       PetscCall(ISGetIndices(subvertexIS, &subvertices));
2525     }
2526   }
2527   for (v = 0; v < numSubVerticesInitial; ++v) {
2528     const PetscInt vertex = subvertices[v];
2529     PetscInt      *star   = NULL;
2530     PetscInt       starSize, s, numFaces = 0, f;
2531 
2532     PetscCall(DMPlexGetTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star));
2533     for (s = 0; s < starSize * 2; s += 2) {
2534       const PetscInt point = star[s];
2535       PetscInt       faceLoc;
2536 
2537       if ((point >= pStart[dim - 1]) && (point < pEnd[dim - 1])) {
2538         if (markedFaces) {
2539           PetscCall(DMLabelGetValue(vertexLabel, point, &faceLoc));
2540           if (faceLoc < 0) continue;
2541         }
2542         star[numFaces++] = point;
2543       }
2544     }
2545     for (f = 0; f < numFaces; ++f) {
2546       const PetscInt face    = star[f];
2547       PetscInt      *closure = NULL;
2548       PetscInt       closureSize, c;
2549       PetscInt       faceLoc;
2550 
2551       PetscCall(DMLabelGetValue(subpointMap, face, &faceLoc));
2552       if (faceLoc == dim - 1) continue;
2553       PetscCheck(faceLoc < 0, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Face %" PetscInt_FMT " has dimension %" PetscInt_FMT " in the surface label", face, faceLoc);
2554       PetscCall(DMPlexGetTransitiveClosure(dm, face, PETSC_TRUE, &closureSize, &closure));
2555       for (c = 0; c < closureSize * 2; c += 2) {
2556         const PetscInt point = closure[c];
2557         PetscInt       vertexLoc;
2558 
2559         if ((point >= pStart[0]) && (point < pEnd[0])) {
2560           PetscCall(DMLabelGetValue(vertexLabel, point, &vertexLoc));
2561           if (vertexLoc != value) break;
2562         }
2563       }
2564       if (c == closureSize * 2) {
2565         const PetscInt *support;
2566         PetscInt        supportSize, s;
2567 
2568         for (c = 0; c < closureSize * 2; c += 2) {
2569           const PetscInt point = closure[c];
2570 
2571           for (d = 0; d < dim; ++d) {
2572             if ((point >= pStart[d]) && (point < pEnd[d])) {
2573               PetscCall(DMLabelSetValue(subpointMap, point, d));
2574               break;
2575             }
2576           }
2577         }
2578         PetscCall(DMPlexGetSupportSize(dm, face, &supportSize));
2579         PetscCall(DMPlexGetSupport(dm, face, &support));
2580         for (s = 0; s < supportSize; ++s) PetscCall(DMLabelSetValue(subpointMap, support[s], dim));
2581       }
2582       PetscCall(DMPlexRestoreTransitiveClosure(dm, face, PETSC_TRUE, &closureSize, &closure));
2583     }
2584     PetscCall(DMPlexRestoreTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star));
2585   }
2586   if (subvertexIS) PetscCall(ISRestoreIndices(subvertexIS, &subvertices));
2587   PetscCall(ISDestroy(&subvertexIS));
2588   PetscCall(PetscFree2(pStart, pEnd));
2589   PetscFunctionReturn(PETSC_SUCCESS);
2590 }
2591 
2592 static PetscErrorCode DMPlexMarkCohesiveSubmesh_Uninterpolated(DM dm, PetscBool hasLagrange, const char labelname[], PetscInt value, DMLabel subpointMap, PetscInt *numFaces, PetscInt *nFV, PetscInt *subCells[], DM subdm)
2593 {
2594   DMLabel         label = NULL;
2595   const PetscInt *cone;
2596   PetscInt        dim, cMax, cEnd, c, subc = 0, p, coneSize = -1;
2597 
2598   PetscFunctionBegin;
2599   *numFaces = 0;
2600   *nFV      = 0;
2601   if (labelname) PetscCall(DMGetLabel(dm, labelname, &label));
2602   *subCells = NULL;
2603   PetscCall(DMGetDimension(dm, &dim));
2604   PetscCall(DMPlexGetTensorPrismBounds_Internal(dm, dim, &cMax, &cEnd));
2605   if (cMax < 0) PetscFunctionReturn(PETSC_SUCCESS);
2606   if (label) {
2607     for (c = cMax; c < cEnd; ++c) {
2608       PetscInt val;
2609 
2610       PetscCall(DMLabelGetValue(label, c, &val));
2611       if (val == value) {
2612         ++(*numFaces);
2613         PetscCall(DMPlexGetConeSize(dm, c, &coneSize));
2614       }
2615     }
2616   } else {
2617     *numFaces = cEnd - cMax;
2618     PetscCall(DMPlexGetConeSize(dm, cMax, &coneSize));
2619   }
2620   PetscCall(PetscMalloc1(*numFaces * 2, subCells));
2621   if (!*numFaces) PetscFunctionReturn(PETSC_SUCCESS);
2622   *nFV = hasLagrange ? coneSize / 3 : coneSize / 2;
2623   for (c = cMax; c < cEnd; ++c) {
2624     const PetscInt *cells;
2625     PetscInt        numCells;
2626 
2627     if (label) {
2628       PetscInt val;
2629 
2630       PetscCall(DMLabelGetValue(label, c, &val));
2631       if (val != value) continue;
2632     }
2633     PetscCall(DMPlexGetCone(dm, c, &cone));
2634     for (p = 0; p < *nFV; ++p) PetscCall(DMLabelSetValue(subpointMap, cone[p], 0));
2635     /* Negative face */
2636     PetscCall(DMPlexGetJoin(dm, *nFV, cone, &numCells, &cells));
2637     /* Not true in parallel
2638     PetscCheck(numCells == 2,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive cells should separate two cells"); */
2639     for (p = 0; p < numCells; ++p) {
2640       PetscCall(DMLabelSetValue(subpointMap, cells[p], 2));
2641       (*subCells)[subc++] = cells[p];
2642     }
2643     PetscCall(DMPlexRestoreJoin(dm, *nFV, cone, &numCells, &cells));
2644     /* Positive face is not included */
2645   }
2646   PetscFunctionReturn(PETSC_SUCCESS);
2647 }
2648 
2649 static PetscErrorCode DMPlexMarkCohesiveSubmesh_Interpolated(DM dm, DMLabel label, PetscInt value, DMLabel subpointMap, DM subdm)
2650 {
2651   PetscInt *pStart, *pEnd;
2652   PetscInt  dim, cMax, cEnd, c, d;
2653 
2654   PetscFunctionBegin;
2655   PetscCall(DMGetDimension(dm, &dim));
2656   PetscCall(DMPlexGetTensorPrismBounds_Internal(dm, dim, &cMax, &cEnd));
2657   if (cMax < 0) PetscFunctionReturn(PETSC_SUCCESS);
2658   PetscCall(PetscMalloc2(dim + 1, &pStart, dim + 1, &pEnd));
2659   for (d = 0; d <= dim; ++d) PetscCall(DMPlexGetDepthStratum(dm, d, &pStart[d], &pEnd[d]));
2660   for (c = cMax; c < cEnd; ++c) {
2661     const PetscInt *cone;
2662     PetscInt       *closure = NULL;
2663     PetscInt        fconeSize, coneSize, closureSize, cl, val;
2664 
2665     if (label) {
2666       PetscCall(DMLabelGetValue(label, c, &val));
2667       if (val != value) continue;
2668     }
2669     PetscCall(DMPlexGetConeSize(dm, c, &coneSize));
2670     PetscCall(DMPlexGetCone(dm, c, &cone));
2671     PetscCall(DMPlexGetConeSize(dm, cone[0], &fconeSize));
2672     PetscCheck(coneSize == (fconeSize ? fconeSize : 1) + 2, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive cells should separate two cells");
2673     /* Negative face */
2674     PetscCall(DMPlexGetTransitiveClosure(dm, cone[0], PETSC_TRUE, &closureSize, &closure));
2675     for (cl = 0; cl < closureSize * 2; cl += 2) {
2676       const PetscInt point = closure[cl];
2677 
2678       for (d = 0; d <= dim; ++d) {
2679         if ((point >= pStart[d]) && (point < pEnd[d])) {
2680           PetscCall(DMLabelSetValue(subpointMap, point, d));
2681           break;
2682         }
2683       }
2684     }
2685     PetscCall(DMPlexRestoreTransitiveClosure(dm, cone[0], PETSC_TRUE, &closureSize, &closure));
2686     /* Cells -- positive face is not included */
2687     for (cl = 0; cl < 1; ++cl) {
2688       const PetscInt *support;
2689       PetscInt        supportSize, s;
2690 
2691       PetscCall(DMPlexGetSupportSize(dm, cone[cl], &supportSize));
2692       /* PetscCheck(supportSize == 2,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive faces should separate two cells"); */
2693       PetscCall(DMPlexGetSupport(dm, cone[cl], &support));
2694       for (s = 0; s < supportSize; ++s) PetscCall(DMLabelSetValue(subpointMap, support[s], dim));
2695     }
2696   }
2697   PetscCall(PetscFree2(pStart, pEnd));
2698   PetscFunctionReturn(PETSC_SUCCESS);
2699 }
2700 
2701 static PetscErrorCode DMPlexGetFaceOrientation(DM dm, PetscInt cell, PetscInt numCorners, PetscInt indices[], PetscInt oppositeVertex, PetscInt origVertices[], PetscInt faceVertices[], PetscBool *posOriented)
2702 {
2703   MPI_Comm       comm;
2704   PetscBool      posOrient = PETSC_FALSE;
2705   const PetscInt debug     = 0;
2706   PetscInt       cellDim, faceSize, f;
2707 
2708   PetscFunctionBegin;
2709   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
2710   PetscCall(DMGetDimension(dm, &cellDim));
2711   if (debug) PetscCall(PetscPrintf(comm, "cellDim: %" PetscInt_FMT " numCorners: %" PetscInt_FMT "\n", cellDim, numCorners));
2712 
2713   if (cellDim == 1 && numCorners == 2) {
2714     /* Triangle */
2715     faceSize  = numCorners - 1;
2716     posOrient = !(oppositeVertex % 2) ? PETSC_TRUE : PETSC_FALSE;
2717   } else if (cellDim == 2 && numCorners == 3) {
2718     /* Triangle */
2719     faceSize  = numCorners - 1;
2720     posOrient = !(oppositeVertex % 2) ? PETSC_TRUE : PETSC_FALSE;
2721   } else if (cellDim == 3 && numCorners == 4) {
2722     /* Tetrahedron */
2723     faceSize  = numCorners - 1;
2724     posOrient = (oppositeVertex % 2) ? PETSC_TRUE : PETSC_FALSE;
2725   } else if (cellDim == 1 && numCorners == 3) {
2726     /* Quadratic line */
2727     faceSize  = 1;
2728     posOrient = PETSC_TRUE;
2729   } else if (cellDim == 2 && numCorners == 4) {
2730     /* Quads */
2731     faceSize = 2;
2732     if ((indices[1] > indices[0]) && (indices[1] - indices[0] == 1)) {
2733       posOrient = PETSC_TRUE;
2734     } else if ((indices[0] == 3) && (indices[1] == 0)) {
2735       posOrient = PETSC_TRUE;
2736     } else {
2737       if (((indices[0] > indices[1]) && (indices[0] - indices[1] == 1)) || ((indices[0] == 0) && (indices[1] == 3))) {
2738         posOrient = PETSC_FALSE;
2739       } else SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid quad crossedge");
2740     }
2741   } else if (cellDim == 2 && numCorners == 6) {
2742     /* Quadratic triangle (I hate this) */
2743     /* Edges are determined by the first 2 vertices (corners of edges) */
2744     const PetscInt faceSizeTri = 3;
2745     PetscInt       sortedIndices[3], i, iFace;
2746     PetscBool      found                    = PETSC_FALSE;
2747     PetscInt       faceVerticesTriSorted[9] = {
2748       0, 3, 4, /* bottom */
2749       1, 4, 5, /* right */
2750       2, 3, 5, /* left */
2751     };
2752     PetscInt faceVerticesTri[9] = {
2753       0, 3, 4, /* bottom */
2754       1, 4, 5, /* right */
2755       2, 5, 3, /* left */
2756     };
2757 
2758     for (i = 0; i < faceSizeTri; ++i) sortedIndices[i] = indices[i];
2759     PetscCall(PetscSortInt(faceSizeTri, sortedIndices));
2760     for (iFace = 0; iFace < 3; ++iFace) {
2761       const PetscInt ii = iFace * faceSizeTri;
2762       PetscInt       fVertex, cVertex;
2763 
2764       if ((sortedIndices[0] == faceVerticesTriSorted[ii + 0]) && (sortedIndices[1] == faceVerticesTriSorted[ii + 1])) {
2765         for (fVertex = 0; fVertex < faceSizeTri; ++fVertex) {
2766           for (cVertex = 0; cVertex < faceSizeTri; ++cVertex) {
2767             if (indices[cVertex] == faceVerticesTri[ii + fVertex]) {
2768               faceVertices[fVertex] = origVertices[cVertex];
2769               break;
2770             }
2771           }
2772         }
2773         found = PETSC_TRUE;
2774         break;
2775       }
2776     }
2777     PetscCheck(found, comm, PETSC_ERR_ARG_WRONG, "Invalid tri crossface");
2778     if (posOriented) *posOriented = PETSC_TRUE;
2779     PetscFunctionReturn(PETSC_SUCCESS);
2780   } else if (cellDim == 2 && numCorners == 9) {
2781     /* Quadratic quad (I hate this) */
2782     /* Edges are determined by the first 2 vertices (corners of edges) */
2783     const PetscInt faceSizeQuad = 3;
2784     PetscInt       sortedIndices[3], i, iFace;
2785     PetscBool      found                      = PETSC_FALSE;
2786     PetscInt       faceVerticesQuadSorted[12] = {
2787       0, 1, 4, /* bottom */
2788       1, 2, 5, /* right */
2789       2, 3, 6, /* top */
2790       0, 3, 7, /* left */
2791     };
2792     PetscInt faceVerticesQuad[12] = {
2793       0, 1, 4, /* bottom */
2794       1, 2, 5, /* right */
2795       2, 3, 6, /* top */
2796       3, 0, 7, /* left */
2797     };
2798 
2799     for (i = 0; i < faceSizeQuad; ++i) sortedIndices[i] = indices[i];
2800     PetscCall(PetscSortInt(faceSizeQuad, sortedIndices));
2801     for (iFace = 0; iFace < 4; ++iFace) {
2802       const PetscInt ii = iFace * faceSizeQuad;
2803       PetscInt       fVertex, cVertex;
2804 
2805       if ((sortedIndices[0] == faceVerticesQuadSorted[ii + 0]) && (sortedIndices[1] == faceVerticesQuadSorted[ii + 1])) {
2806         for (fVertex = 0; fVertex < faceSizeQuad; ++fVertex) {
2807           for (cVertex = 0; cVertex < faceSizeQuad; ++cVertex) {
2808             if (indices[cVertex] == faceVerticesQuad[ii + fVertex]) {
2809               faceVertices[fVertex] = origVertices[cVertex];
2810               break;
2811             }
2812           }
2813         }
2814         found = PETSC_TRUE;
2815         break;
2816       }
2817     }
2818     PetscCheck(found, comm, PETSC_ERR_ARG_WRONG, "Invalid quad crossface");
2819     if (posOriented) *posOriented = PETSC_TRUE;
2820     PetscFunctionReturn(PETSC_SUCCESS);
2821   } else if (cellDim == 3 && numCorners == 8) {
2822     /* Hexes
2823        A hex is two oriented quads with the normal of the first
2824        pointing up at the second.
2825 
2826           7---6
2827          /|  /|
2828         4---5 |
2829         | 1-|-2
2830         |/  |/
2831         0---3
2832 
2833         Faces are determined by the first 4 vertices (corners of faces) */
2834     const PetscInt faceSizeHex = 4;
2835     PetscInt       sortedIndices[4], i, iFace;
2836     PetscBool      found                     = PETSC_FALSE;
2837     PetscInt       faceVerticesHexSorted[24] = {
2838       0, 1, 2, 3, /* bottom */
2839       4, 5, 6, 7, /* top */
2840       0, 3, 4, 5, /* front */
2841       2, 3, 5, 6, /* right */
2842       1, 2, 6, 7, /* back */
2843       0, 1, 4, 7, /* left */
2844     };
2845     PetscInt faceVerticesHex[24] = {
2846       1, 2, 3, 0, /* bottom */
2847       4, 5, 6, 7, /* top */
2848       0, 3, 5, 4, /* front */
2849       3, 2, 6, 5, /* right */
2850       2, 1, 7, 6, /* back */
2851       1, 0, 4, 7, /* left */
2852     };
2853 
2854     for (i = 0; i < faceSizeHex; ++i) sortedIndices[i] = indices[i];
2855     PetscCall(PetscSortInt(faceSizeHex, sortedIndices));
2856     for (iFace = 0; iFace < 6; ++iFace) {
2857       const PetscInt ii = iFace * faceSizeHex;
2858       PetscInt       fVertex, cVertex;
2859 
2860       if ((sortedIndices[0] == faceVerticesHexSorted[ii + 0]) && (sortedIndices[1] == faceVerticesHexSorted[ii + 1]) && (sortedIndices[2] == faceVerticesHexSorted[ii + 2]) && (sortedIndices[3] == faceVerticesHexSorted[ii + 3])) {
2861         for (fVertex = 0; fVertex < faceSizeHex; ++fVertex) {
2862           for (cVertex = 0; cVertex < faceSizeHex; ++cVertex) {
2863             if (indices[cVertex] == faceVerticesHex[ii + fVertex]) {
2864               faceVertices[fVertex] = origVertices[cVertex];
2865               break;
2866             }
2867           }
2868         }
2869         found = PETSC_TRUE;
2870         break;
2871       }
2872     }
2873     PetscCheck(found, comm, PETSC_ERR_ARG_WRONG, "Invalid hex crossface");
2874     if (posOriented) *posOriented = PETSC_TRUE;
2875     PetscFunctionReturn(PETSC_SUCCESS);
2876   } else if (cellDim == 3 && numCorners == 10) {
2877     /* Quadratic tet */
2878     /* Faces are determined by the first 3 vertices (corners of faces) */
2879     const PetscInt faceSizeTet = 6;
2880     PetscInt       sortedIndices[6], i, iFace;
2881     PetscBool      found                     = PETSC_FALSE;
2882     PetscInt       faceVerticesTetSorted[24] = {
2883       0, 1, 2, 6, 7, 8, /* bottom */
2884       0, 3, 4, 6, 7, 9, /* front */
2885       1, 4, 5, 7, 8, 9, /* right */
2886       2, 3, 5, 6, 8, 9, /* left */
2887     };
2888     PetscInt faceVerticesTet[24] = {
2889       0, 1, 2, 6, 7, 8, /* bottom */
2890       0, 4, 3, 6, 7, 9, /* front */
2891       1, 5, 4, 7, 8, 9, /* right */
2892       2, 3, 5, 8, 6, 9, /* left */
2893     };
2894 
2895     for (i = 0; i < faceSizeTet; ++i) sortedIndices[i] = indices[i];
2896     PetscCall(PetscSortInt(faceSizeTet, sortedIndices));
2897     for (iFace = 0; iFace < 4; ++iFace) {
2898       const PetscInt ii = iFace * faceSizeTet;
2899       PetscInt       fVertex, cVertex;
2900 
2901       if ((sortedIndices[0] == faceVerticesTetSorted[ii + 0]) && (sortedIndices[1] == faceVerticesTetSorted[ii + 1]) && (sortedIndices[2] == faceVerticesTetSorted[ii + 2]) && (sortedIndices[3] == faceVerticesTetSorted[ii + 3])) {
2902         for (fVertex = 0; fVertex < faceSizeTet; ++fVertex) {
2903           for (cVertex = 0; cVertex < faceSizeTet; ++cVertex) {
2904             if (indices[cVertex] == faceVerticesTet[ii + fVertex]) {
2905               faceVertices[fVertex] = origVertices[cVertex];
2906               break;
2907             }
2908           }
2909         }
2910         found = PETSC_TRUE;
2911         break;
2912       }
2913     }
2914     PetscCheck(found, comm, PETSC_ERR_ARG_WRONG, "Invalid tet crossface");
2915     if (posOriented) *posOriented = PETSC_TRUE;
2916     PetscFunctionReturn(PETSC_SUCCESS);
2917   } else if (cellDim == 3 && numCorners == 27) {
2918     /* Quadratic hexes (I hate this)
2919        A hex is two oriented quads with the normal of the first
2920        pointing up at the second.
2921 
2922          7---6
2923         /|  /|
2924        4---5 |
2925        | 3-|-2
2926        |/  |/
2927        0---1
2928 
2929        Faces are determined by the first 4 vertices (corners of faces) */
2930     const PetscInt faceSizeQuadHex = 9;
2931     PetscInt       sortedIndices[9], i, iFace;
2932     PetscBool      found                         = PETSC_FALSE;
2933     PetscInt       faceVerticesQuadHexSorted[54] = {
2934       0, 1, 2, 3, 8,  9,  10, 11, 24, /* bottom */
2935       4, 5, 6, 7, 12, 13, 14, 15, 25, /* top */
2936       0, 1, 4, 5, 8,  12, 16, 17, 22, /* front */
2937       1, 2, 5, 6, 9,  13, 17, 18, 21, /* right */
2938       2, 3, 6, 7, 10, 14, 18, 19, 23, /* back */
2939       0, 3, 4, 7, 11, 15, 16, 19, 20, /* left */
2940     };
2941     PetscInt faceVerticesQuadHex[54] = {
2942       3, 2, 1, 0, 10, 9,  8,  11, 24, /* bottom */
2943       4, 5, 6, 7, 12, 13, 14, 15, 25, /* top */
2944       0, 1, 5, 4, 8,  17, 12, 16, 22, /* front */
2945       1, 2, 6, 5, 9,  18, 13, 17, 21, /* right */
2946       2, 3, 7, 6, 10, 19, 14, 18, 23, /* back */
2947       3, 0, 4, 7, 11, 16, 15, 19, 20  /* left */
2948     };
2949 
2950     for (i = 0; i < faceSizeQuadHex; ++i) sortedIndices[i] = indices[i];
2951     PetscCall(PetscSortInt(faceSizeQuadHex, sortedIndices));
2952     for (iFace = 0; iFace < 6; ++iFace) {
2953       const PetscInt ii = iFace * faceSizeQuadHex;
2954       PetscInt       fVertex, cVertex;
2955 
2956       if ((sortedIndices[0] == faceVerticesQuadHexSorted[ii + 0]) && (sortedIndices[1] == faceVerticesQuadHexSorted[ii + 1]) && (sortedIndices[2] == faceVerticesQuadHexSorted[ii + 2]) && (sortedIndices[3] == faceVerticesQuadHexSorted[ii + 3])) {
2957         for (fVertex = 0; fVertex < faceSizeQuadHex; ++fVertex) {
2958           for (cVertex = 0; cVertex < faceSizeQuadHex; ++cVertex) {
2959             if (indices[cVertex] == faceVerticesQuadHex[ii + fVertex]) {
2960               faceVertices[fVertex] = origVertices[cVertex];
2961               break;
2962             }
2963           }
2964         }
2965         found = PETSC_TRUE;
2966         break;
2967       }
2968     }
2969     PetscCheck(found, comm, PETSC_ERR_ARG_WRONG, "Invalid hex crossface");
2970     if (posOriented) *posOriented = PETSC_TRUE;
2971     PetscFunctionReturn(PETSC_SUCCESS);
2972   } else SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Unknown cell type for faceOrientation().");
2973   if (!posOrient) {
2974     if (debug) PetscCall(PetscPrintf(comm, "  Reversing initial face orientation\n"));
2975     for (f = 0; f < faceSize; ++f) faceVertices[f] = origVertices[faceSize - 1 - f];
2976   } else {
2977     if (debug) PetscCall(PetscPrintf(comm, "  Keeping initial face orientation\n"));
2978     for (f = 0; f < faceSize; ++f) faceVertices[f] = origVertices[f];
2979   }
2980   if (posOriented) *posOriented = posOrient;
2981   PetscFunctionReturn(PETSC_SUCCESS);
2982 }
2983 
2984 /*@
2985   DMPlexGetOrientedFace - Given a cell and a face, as a set of vertices, return the oriented face, as a set of vertices,
2986   in faceVertices. The orientation is such that the face normal points out of the cell
2987 
2988   Not Collective
2989 
2990   Input Parameters:
2991 + dm           - The original mesh
2992 . cell         - The cell mesh point
2993 . faceSize     - The number of vertices on the face
2994 . face         - The face vertices
2995 . numCorners   - The number of vertices on the cell
2996 . indices      - Local numbering of face vertices in cell cone
2997 - origVertices - Original face vertices
2998 
2999   Output Parameters:
3000 + faceVertices - The face vertices properly oriented
3001 - posOriented  - `PETSC_TRUE` if the face was oriented with outward normal
3002 
3003   Level: developer
3004 
3005 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetCone()`
3006 @*/
3007 PetscErrorCode DMPlexGetOrientedFace(DM dm, PetscInt cell, PetscInt faceSize, const PetscInt face[], PetscInt numCorners, PetscInt indices[], PetscInt origVertices[], PetscInt faceVertices[], PetscBool *posOriented)
3008 {
3009   const PetscInt *cone = NULL;
3010   PetscInt        coneSize, v, f, v2;
3011   PetscInt        oppositeVertex = -1;
3012 
3013   PetscFunctionBegin;
3014   PetscCall(DMPlexGetConeSize(dm, cell, &coneSize));
3015   PetscCall(DMPlexGetCone(dm, cell, &cone));
3016   for (v = 0, v2 = 0; v < coneSize; ++v) {
3017     PetscBool found = PETSC_FALSE;
3018 
3019     for (f = 0; f < faceSize; ++f) {
3020       if (face[f] == cone[v]) {
3021         found = PETSC_TRUE;
3022         break;
3023       }
3024     }
3025     if (found) {
3026       indices[v2]      = v;
3027       origVertices[v2] = cone[v];
3028       ++v2;
3029     } else {
3030       oppositeVertex = v;
3031     }
3032   }
3033   PetscCall(DMPlexGetFaceOrientation(dm, cell, numCorners, indices, oppositeVertex, origVertices, faceVertices, posOriented));
3034   PetscFunctionReturn(PETSC_SUCCESS);
3035 }
3036 
3037 /*
3038   DMPlexInsertFace_Internal - Puts a face into the mesh
3039 
3040   Not Collective
3041 
3042   Input Parameters:
3043   + dm              - The `DMPLEX`
3044   . numFaceVertex   - The number of vertices in the face
3045   . faceVertices    - The vertices in the face for `dm`
3046   . subfaceVertices - The vertices in the face for subdm
3047   . numCorners      - The number of vertices in the `cell`
3048   . cell            - A cell in `dm` containing the face
3049   . subcell         - A cell in subdm containing the face
3050   . firstFace       - First face in the mesh
3051   - newFacePoint    - Next face in the mesh
3052 
3053   Output Parameter:
3054   . newFacePoint - Contains next face point number on input, updated on output
3055 
3056   Level: developer
3057 */
3058 static PetscErrorCode DMPlexInsertFace_Internal(DM dm, DM subdm, PetscInt numFaceVertices, const PetscInt faceVertices[], const PetscInt subfaceVertices[], PetscInt numCorners, PetscInt cell, PetscInt subcell, PetscInt firstFace, PetscInt *newFacePoint)
3059 {
3060   MPI_Comm        comm;
3061   DM_Plex        *submesh = (DM_Plex *)subdm->data;
3062   const PetscInt *faces;
3063   PetscInt        numFaces, coneSize;
3064 
3065   PetscFunctionBegin;
3066   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
3067   PetscCall(DMPlexGetConeSize(subdm, subcell, &coneSize));
3068   PetscCheck(coneSize == 1, comm, PETSC_ERR_ARG_OUTOFRANGE, "Cone size of cell %" PetscInt_FMT " is %" PetscInt_FMT " != 1", cell, coneSize);
3069 #if 0
3070   /* Cannot use this because support() has not been constructed yet */
3071   PetscCall(DMPlexGetJoin(subdm, numFaceVertices, subfaceVertices, &numFaces, &faces));
3072 #else
3073   {
3074     PetscInt f;
3075 
3076     numFaces = 0;
3077     PetscCall(DMGetWorkArray(subdm, 1, MPIU_INT, (void **)&faces));
3078     for (f = firstFace; f < *newFacePoint; ++f) {
3079       PetscInt dof, off, d;
3080 
3081       PetscCall(PetscSectionGetDof(submesh->coneSection, f, &dof));
3082       PetscCall(PetscSectionGetOffset(submesh->coneSection, f, &off));
3083       /* Yes, I know this is quadratic, but I expect the sizes to be <5 */
3084       for (d = 0; d < dof; ++d) {
3085         const PetscInt p = submesh->cones[off + d];
3086         PetscInt       v;
3087 
3088         for (v = 0; v < numFaceVertices; ++v) {
3089           if (subfaceVertices[v] == p) break;
3090         }
3091         if (v == numFaceVertices) break;
3092       }
3093       if (d == dof) {
3094         numFaces               = 1;
3095         ((PetscInt *)faces)[0] = f;
3096       }
3097     }
3098   }
3099 #endif
3100   PetscCheck(numFaces <= 1, comm, PETSC_ERR_ARG_WRONG, "Vertex set had %" PetscInt_FMT " faces, not one", numFaces);
3101   if (numFaces == 1) {
3102     /* Add the other cell neighbor for this face */
3103     PetscCall(DMPlexSetCone(subdm, subcell, faces));
3104   } else {
3105     PetscInt *indices, *origVertices, *orientedVertices, *orientedSubVertices, v, ov;
3106     PetscBool posOriented;
3107 
3108     PetscCall(DMGetWorkArray(subdm, 4 * numFaceVertices * sizeof(PetscInt), MPIU_INT, &orientedVertices));
3109     origVertices        = &orientedVertices[numFaceVertices];
3110     indices             = &orientedVertices[numFaceVertices * 2];
3111     orientedSubVertices = &orientedVertices[numFaceVertices * 3];
3112     PetscCall(DMPlexGetOrientedFace(dm, cell, numFaceVertices, faceVertices, numCorners, indices, origVertices, orientedVertices, &posOriented));
3113     /* TODO: I know that routine should return a permutation, not the indices */
3114     for (v = 0; v < numFaceVertices; ++v) {
3115       const PetscInt vertex = faceVertices[v], subvertex = subfaceVertices[v];
3116       for (ov = 0; ov < numFaceVertices; ++ov) {
3117         if (orientedVertices[ov] == vertex) {
3118           orientedSubVertices[ov] = subvertex;
3119           break;
3120         }
3121       }
3122       PetscCheck(ov != numFaceVertices, comm, PETSC_ERR_PLIB, "Could not find face vertex %" PetscInt_FMT " in orientated set", vertex);
3123     }
3124     PetscCall(DMPlexSetCone(subdm, *newFacePoint, orientedSubVertices));
3125     PetscCall(DMPlexSetCone(subdm, subcell, newFacePoint));
3126     PetscCall(DMRestoreWorkArray(subdm, 4 * numFaceVertices * sizeof(PetscInt), MPIU_INT, &orientedVertices));
3127     ++(*newFacePoint);
3128   }
3129 #if 0
3130   PetscCall(DMPlexRestoreJoin(subdm, numFaceVertices, subfaceVertices, &numFaces, &faces));
3131 #else
3132   PetscCall(DMRestoreWorkArray(subdm, 1, MPIU_INT, (void **)&faces));
3133 #endif
3134   PetscFunctionReturn(PETSC_SUCCESS);
3135 }
3136 
3137 static PetscErrorCode DMPlexCreateSubmesh_Uninterpolated(DM dm, DMLabel vertexLabel, PetscInt value, DM subdm)
3138 {
3139   MPI_Comm        comm;
3140   DMLabel         subpointMap;
3141   IS              subvertexIS, subcellIS;
3142   const PetscInt *subVertices, *subCells;
3143   PetscInt        numSubVertices, firstSubVertex, numSubCells;
3144   PetscInt       *subface, maxConeSize, numSubFaces = 0, firstSubFace, newFacePoint, nFV = 0;
3145   PetscInt        vStart, vEnd, c, f;
3146 
3147   PetscFunctionBegin;
3148   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
3149   /* Create subpointMap which marks the submesh */
3150   PetscCall(DMLabelCreate(PETSC_COMM_SELF, "subpoint_map", &subpointMap));
3151   PetscCall(DMPlexSetSubpointMap(subdm, subpointMap));
3152   if (vertexLabel) PetscCall(DMPlexMarkSubmesh_Uninterpolated(dm, vertexLabel, value, subpointMap, &numSubFaces, &nFV, subdm));
3153   /* Setup chart */
3154   PetscCall(DMLabelGetStratumSize(subpointMap, 0, &numSubVertices));
3155   PetscCall(DMLabelGetStratumSize(subpointMap, 2, &numSubCells));
3156   PetscCall(DMPlexSetChart(subdm, 0, numSubCells + numSubFaces + numSubVertices));
3157   PetscCall(DMPlexSetVTKCellHeight(subdm, 1));
3158   /* Set cone sizes */
3159   firstSubVertex = numSubCells;
3160   firstSubFace   = numSubCells + numSubVertices;
3161   newFacePoint   = firstSubFace;
3162   PetscCall(DMLabelGetStratumIS(subpointMap, 0, &subvertexIS));
3163   if (subvertexIS) PetscCall(ISGetIndices(subvertexIS, &subVertices));
3164   PetscCall(DMLabelGetStratumIS(subpointMap, 2, &subcellIS));
3165   if (subcellIS) PetscCall(ISGetIndices(subcellIS, &subCells));
3166   for (c = 0; c < numSubCells; ++c) PetscCall(DMPlexSetConeSize(subdm, c, 1));
3167   for (f = firstSubFace; f < firstSubFace + numSubFaces; ++f) PetscCall(DMPlexSetConeSize(subdm, f, nFV));
3168   PetscCall(DMSetUp(subdm));
3169   PetscCall(DMLabelDestroy(&subpointMap));
3170   /* Create face cones */
3171   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
3172   PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, NULL));
3173   PetscCall(DMGetWorkArray(subdm, maxConeSize, MPIU_INT, (void **)&subface));
3174   for (c = 0; c < numSubCells; ++c) {
3175     const PetscInt cell    = subCells[c];
3176     const PetscInt subcell = c;
3177     PetscInt      *closure = NULL;
3178     PetscInt       closureSize, cl, numCorners = 0, faceSize = 0;
3179 
3180     PetscCall(DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
3181     for (cl = 0; cl < closureSize * 2; cl += 2) {
3182       const PetscInt point = closure[cl];
3183       PetscInt       subVertex;
3184 
3185       if ((point >= vStart) && (point < vEnd)) {
3186         ++numCorners;
3187         PetscCall(PetscFindInt(point, numSubVertices, subVertices, &subVertex));
3188         if (subVertex >= 0) {
3189           closure[faceSize] = point;
3190           subface[faceSize] = firstSubVertex + subVertex;
3191           ++faceSize;
3192         }
3193       }
3194     }
3195     PetscCheck(faceSize <= nFV, comm, PETSC_ERR_ARG_WRONG, "Invalid submesh: Too many vertices %" PetscInt_FMT " of an element on the surface", faceSize);
3196     if (faceSize == nFV) PetscCall(DMPlexInsertFace_Internal(dm, subdm, faceSize, closure, subface, numCorners, cell, subcell, firstSubFace, &newFacePoint));
3197     PetscCall(DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
3198   }
3199   PetscCall(DMRestoreWorkArray(subdm, maxConeSize, MPIU_INT, (void **)&subface));
3200   PetscCall(DMPlexSymmetrize(subdm));
3201   PetscCall(DMPlexStratify(subdm));
3202   /* Build coordinates */
3203   {
3204     PetscSection coordSection, subCoordSection;
3205     Vec          coordinates, subCoordinates;
3206     PetscScalar *coords, *subCoords;
3207     PetscInt     numComp, coordSize, v;
3208     const char  *name;
3209 
3210     PetscCall(DMGetCoordinateSection(dm, &coordSection));
3211     PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
3212     PetscCall(DMGetCoordinateSection(subdm, &subCoordSection));
3213     PetscCall(PetscSectionSetNumFields(subCoordSection, 1));
3214     PetscCall(PetscSectionGetFieldComponents(coordSection, 0, &numComp));
3215     PetscCall(PetscSectionSetFieldComponents(subCoordSection, 0, numComp));
3216     PetscCall(PetscSectionSetChart(subCoordSection, firstSubVertex, firstSubVertex + numSubVertices));
3217     for (v = 0; v < numSubVertices; ++v) {
3218       const PetscInt vertex    = subVertices[v];
3219       const PetscInt subvertex = firstSubVertex + v;
3220       PetscInt       dof;
3221 
3222       PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3223       PetscCall(PetscSectionSetDof(subCoordSection, subvertex, dof));
3224       PetscCall(PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof));
3225     }
3226     PetscCall(PetscSectionSetUp(subCoordSection));
3227     PetscCall(PetscSectionGetStorageSize(subCoordSection, &coordSize));
3228     PetscCall(VecCreate(PETSC_COMM_SELF, &subCoordinates));
3229     PetscCall(PetscObjectGetName((PetscObject)coordinates, &name));
3230     PetscCall(PetscObjectSetName((PetscObject)subCoordinates, name));
3231     PetscCall(VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE));
3232     PetscCall(VecSetType(subCoordinates, VECSTANDARD));
3233     if (coordSize) {
3234       PetscCall(VecGetArray(coordinates, &coords));
3235       PetscCall(VecGetArray(subCoordinates, &subCoords));
3236       for (v = 0; v < numSubVertices; ++v) {
3237         const PetscInt vertex    = subVertices[v];
3238         const PetscInt subvertex = firstSubVertex + v;
3239         PetscInt       dof, off, sdof, soff, d;
3240 
3241         PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3242         PetscCall(PetscSectionGetOffset(coordSection, vertex, &off));
3243         PetscCall(PetscSectionGetDof(subCoordSection, subvertex, &sdof));
3244         PetscCall(PetscSectionGetOffset(subCoordSection, subvertex, &soff));
3245         PetscCheck(dof == sdof, comm, PETSC_ERR_PLIB, "Coordinate dimension %" PetscInt_FMT " on subvertex %" PetscInt_FMT ", vertex %" PetscInt_FMT " should be %" PetscInt_FMT, sdof, subvertex, vertex, dof);
3246         for (d = 0; d < dof; ++d) subCoords[soff + d] = coords[off + d];
3247       }
3248       PetscCall(VecRestoreArray(coordinates, &coords));
3249       PetscCall(VecRestoreArray(subCoordinates, &subCoords));
3250     }
3251     PetscCall(DMSetCoordinatesLocal(subdm, subCoordinates));
3252     PetscCall(VecDestroy(&subCoordinates));
3253   }
3254   /* Cleanup */
3255   if (subvertexIS) PetscCall(ISRestoreIndices(subvertexIS, &subVertices));
3256   PetscCall(ISDestroy(&subvertexIS));
3257   if (subcellIS) PetscCall(ISRestoreIndices(subcellIS, &subCells));
3258   PetscCall(ISDestroy(&subcellIS));
3259   PetscFunctionReturn(PETSC_SUCCESS);
3260 }
3261 
3262 /* TODO: Fix this to properly propagate up error conditions it may find */
3263 static inline PetscInt DMPlexFilterPoint_Internal(PetscInt point, PetscInt firstSubPoint, PetscInt numSubPoints, const PetscInt subPoints[])
3264 {
3265   PetscInt       subPoint;
3266   PetscErrorCode ierr;
3267 
3268   ierr = PetscFindInt(point, numSubPoints, subPoints, &subPoint);
3269   if (ierr) return -1;
3270   return subPoint < 0 ? subPoint : firstSubPoint + subPoint;
3271 }
3272 
3273 /* TODO: Fix this to properly propagate up error conditions it may find */
3274 static inline PetscInt DMPlexFilterPointPerm_Internal(PetscInt point, PetscInt firstSubPoint, PetscInt numSubPoints, const PetscInt subPoints[], const PetscInt subIndices[])
3275 {
3276   PetscInt       subPoint;
3277   PetscErrorCode ierr;
3278 
3279   ierr = PetscFindInt(point, numSubPoints, subPoints, &subPoint);
3280   if (ierr) return -1;
3281   return subPoint < 0 ? subPoint : firstSubPoint + (subIndices ? subIndices[subPoint] : subPoint);
3282 }
3283 
3284 static PetscErrorCode DMPlexFilterLabels_Internal(DM dm, const PetscInt numSubPoints[], const PetscInt *subpoints[], const PetscInt firstSubPoint[], DM subdm)
3285 {
3286   DMLabel  depthLabel;
3287   PetscInt Nl, l, d;
3288 
3289   PetscFunctionBegin;
3290   // Reset depth label for fast lookup
3291   PetscCall(DMPlexGetDepthLabel(dm, &depthLabel));
3292   PetscCall(DMLabelMakeAllInvalid_Internal(depthLabel));
3293   PetscCall(DMGetNumLabels(dm, &Nl));
3294   for (l = 0; l < Nl; ++l) {
3295     DMLabel         label, newlabel;
3296     const char     *lname;
3297     PetscBool       isDepth, isDim, isCelltype, isVTK;
3298     IS              valueIS;
3299     const PetscInt *values;
3300     PetscInt        Nv, v;
3301 
3302     PetscCall(DMGetLabelName(dm, l, &lname));
3303     PetscCall(PetscStrcmp(lname, "depth", &isDepth));
3304     PetscCall(PetscStrcmp(lname, "dim", &isDim));
3305     PetscCall(PetscStrcmp(lname, "celltype", &isCelltype));
3306     PetscCall(PetscStrcmp(lname, "vtk", &isVTK));
3307     if (isDepth || isDim || isCelltype || isVTK) continue;
3308     PetscCall(DMCreateLabel(subdm, lname));
3309     PetscCall(DMGetLabel(dm, lname, &label));
3310     PetscCall(DMGetLabel(subdm, lname, &newlabel));
3311     PetscCall(DMLabelGetDefaultValue(label, &v));
3312     PetscCall(DMLabelSetDefaultValue(newlabel, v));
3313     PetscCall(DMLabelGetValueIS(label, &valueIS));
3314     PetscCall(ISGetLocalSize(valueIS, &Nv));
3315     PetscCall(ISGetIndices(valueIS, &values));
3316     for (v = 0; v < Nv; ++v) {
3317       IS              pointIS;
3318       const PetscInt *points;
3319       PetscInt        Np, p;
3320 
3321       PetscCall(DMLabelGetStratumIS(label, values[v], &pointIS));
3322       PetscCall(ISGetLocalSize(pointIS, &Np));
3323       PetscCall(ISGetIndices(pointIS, &points));
3324       for (p = 0; p < Np; ++p) {
3325         const PetscInt point = points[p];
3326         PetscInt       subp;
3327 
3328         PetscCall(DMPlexGetPointDepth(dm, point, &d));
3329         subp = DMPlexFilterPoint_Internal(point, firstSubPoint[d], numSubPoints[d], subpoints[d]);
3330         if (subp >= 0) PetscCall(DMLabelSetValue(newlabel, subp, values[v]));
3331       }
3332       PetscCall(ISRestoreIndices(pointIS, &points));
3333       PetscCall(ISDestroy(&pointIS));
3334     }
3335     PetscCall(ISRestoreIndices(valueIS, &values));
3336     PetscCall(ISDestroy(&valueIS));
3337   }
3338   PetscFunctionReturn(PETSC_SUCCESS);
3339 }
3340 
3341 static PetscErrorCode DMPlexCreateSubmeshGeneric_Interpolated(DM dm, DMLabel label, PetscInt value, PetscBool markedFaces, PetscBool isCohesive, PetscInt cellHeight, PetscBool ignoreLabelHalo, PetscBool sanitizeSubmesh, PetscSF *ownershipTransferSF, DM subdm)
3342 {
3343   MPI_Comm         comm;
3344   DMLabel          subpointMap;
3345   IS              *subpointIS;
3346   const PetscInt **subpoints;
3347   PetscInt        *numSubPoints, *firstSubPoint, *coneNew, *orntNew;
3348   PetscInt         totSubPoints = 0, maxConeSize, dim, sdim, cdim, p, d, v;
3349   PetscMPIInt      rank;
3350 
3351   PetscFunctionBegin;
3352   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
3353   PetscCallMPI(MPI_Comm_rank(comm, &rank));
3354   /* Create subpointMap which marks the submesh */
3355   PetscCall(DMLabelCreate(PETSC_COMM_SELF, "subpoint_map", &subpointMap));
3356   PetscCall(DMPlexSetSubpointMap(subdm, subpointMap));
3357   if (cellHeight) {
3358     if (isCohesive) PetscCall(DMPlexMarkCohesiveSubmesh_Interpolated(dm, label, value, subpointMap, subdm));
3359     else PetscCall(DMPlexMarkSubmesh_Interpolated(dm, label, value, markedFaces, subpointMap, subdm));
3360   } else {
3361     DMLabel         depth;
3362     IS              pointIS;
3363     const PetscInt *points;
3364     PetscInt        numPoints = 0, pStart = 0;
3365     PetscBool      *isGhost = NULL;
3366 
3367     PetscCall(DMPlexGetDepthLabel(dm, &depth));
3368     PetscCall(DMLabelGetStratumIS(label, value, &pointIS));
3369     if (pointIS) {
3370       PetscCall(ISGetIndices(pointIS, &points));
3371       PetscCall(ISGetLocalSize(pointIS, &numPoints));
3372     }
3373     if (ignoreLabelHalo) {
3374       PetscSF         pointSF;
3375       PetscInt        nleaves, pEnd;
3376       const PetscInt *ilocal = NULL;
3377 
3378       PetscCall(DMGetPointSF(dm, &pointSF));
3379       PetscCall(PetscSFGetGraph(pointSF, NULL, &nleaves, &ilocal, NULL));
3380       PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
3381       PetscCall(PetscMalloc1(pEnd - pStart, &isGhost));
3382       for (p = 0; p < pEnd - pStart; ++p) isGhost[p] = PETSC_FALSE;
3383       for (p = 0; p < nleaves; ++p) isGhost[(ilocal ? ilocal[p] : p) - pStart] = PETSC_TRUE;
3384     }
3385     for (p = 0; p < numPoints; ++p) {
3386       PetscInt *closure = NULL;
3387       PetscInt  closureSize, c, pdim;
3388 
3389       if (ignoreLabelHalo && isGhost[points[p] - pStart]) continue;
3390       PetscCall(DMPlexGetTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closure));
3391       for (c = 0; c < closureSize * 2; c += 2) {
3392         PetscCall(DMLabelGetValue(depth, closure[c], &pdim));
3393         PetscCall(DMLabelSetValue(subpointMap, closure[c], pdim));
3394       }
3395       PetscCall(DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closure));
3396     }
3397     PetscCall(PetscFree(isGhost));
3398     if (pointIS) PetscCall(ISRestoreIndices(pointIS, &points));
3399     PetscCall(ISDestroy(&pointIS));
3400   }
3401   /* Setup chart */
3402   PetscCall(DMGetDimension(dm, &dim));
3403   PetscCall(DMGetCoordinateDim(dm, &cdim));
3404   PetscCall(PetscMalloc4(dim + 1, &numSubPoints, dim + 1, &firstSubPoint, dim + 1, &subpointIS, dim + 1, &subpoints));
3405   for (d = 0; d <= dim; ++d) {
3406     PetscCall(DMLabelGetStratumSize(subpointMap, d, &numSubPoints[d]));
3407     totSubPoints += numSubPoints[d];
3408   }
3409   // Determine submesh dimension
3410   PetscCall(DMGetDimension(subdm, &sdim));
3411   if (sdim > 0) {
3412     // Calling function knows what dimension to use, and we include neighboring cells as well
3413     sdim = dim;
3414   } else {
3415     // We reset the subdimension based on what is being selected
3416     PetscInt lsdim;
3417     for (lsdim = dim; lsdim >= 0; --lsdim)
3418       if (numSubPoints[lsdim]) break;
3419     PetscCall(MPIU_Allreduce(&lsdim, &sdim, 1, MPIU_INT, MPI_MAX, comm));
3420     PetscCall(DMSetDimension(subdm, sdim));
3421     PetscCall(DMSetCoordinateDim(subdm, cdim));
3422   }
3423   PetscCall(DMPlexSetChart(subdm, 0, totSubPoints));
3424   PetscCall(DMPlexSetVTKCellHeight(subdm, cellHeight));
3425   /* Set cone sizes */
3426   firstSubPoint[sdim] = 0;
3427   firstSubPoint[0]    = firstSubPoint[sdim] + numSubPoints[sdim];
3428   if (sdim > 1) firstSubPoint[sdim - 1] = firstSubPoint[0] + numSubPoints[0];
3429   if (sdim > 2) firstSubPoint[sdim - 2] = firstSubPoint[sdim - 1] + numSubPoints[sdim - 1];
3430   for (d = 0; d <= sdim; ++d) {
3431     PetscCall(DMLabelGetStratumIS(subpointMap, d, &subpointIS[d]));
3432     if (subpointIS[d]) PetscCall(ISGetIndices(subpointIS[d], &subpoints[d]));
3433   }
3434   /* We do not want this label automatically computed, instead we compute it here */
3435   PetscCall(DMCreateLabel(subdm, "celltype"));
3436   for (d = 0; d <= sdim; ++d) {
3437     for (p = 0; p < numSubPoints[d]; ++p) {
3438       const PetscInt  point    = subpoints[d][p];
3439       const PetscInt  subpoint = firstSubPoint[d] + p;
3440       const PetscInt *cone;
3441       PetscInt        coneSize;
3442 
3443       PetscCall(DMPlexGetConeSize(dm, point, &coneSize));
3444       if (cellHeight && (d == sdim)) {
3445         PetscInt coneSizeNew, c, val;
3446 
3447         PetscCall(DMPlexGetCone(dm, point, &cone));
3448         for (c = 0, coneSizeNew = 0; c < coneSize; ++c) {
3449           PetscCall(DMLabelGetValue(subpointMap, cone[c], &val));
3450           if (val >= 0) coneSizeNew++;
3451         }
3452         PetscCall(DMPlexSetConeSize(subdm, subpoint, coneSizeNew));
3453         PetscCall(DMPlexSetCellType(subdm, subpoint, DM_POLYTOPE_FV_GHOST));
3454       } else {
3455         DMPolytopeType ct;
3456 
3457         PetscCall(DMPlexSetConeSize(subdm, subpoint, coneSize));
3458         PetscCall(DMPlexGetCellType(dm, point, &ct));
3459         PetscCall(DMPlexSetCellType(subdm, subpoint, ct));
3460       }
3461     }
3462   }
3463   PetscCall(DMLabelDestroy(&subpointMap));
3464   PetscCall(DMSetUp(subdm));
3465   /* Set cones */
3466   PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, NULL));
3467   PetscCall(PetscMalloc2(maxConeSize, &coneNew, maxConeSize, &orntNew));
3468   for (d = 0; d <= sdim; ++d) {
3469     for (p = 0; p < numSubPoints[d]; ++p) {
3470       const PetscInt  point    = subpoints[d][p];
3471       const PetscInt  subpoint = firstSubPoint[d] + p;
3472       const PetscInt *cone, *ornt;
3473       PetscInt        coneSize, subconeSize, coneSizeNew, c, subc, fornt = 0;
3474 
3475       if (d == sdim - 1) {
3476         const PetscInt *support, *cone, *ornt;
3477         PetscInt        supportSize, coneSize, s, subc;
3478 
3479         PetscCall(DMPlexGetSupport(dm, point, &support));
3480         PetscCall(DMPlexGetSupportSize(dm, point, &supportSize));
3481         for (s = 0; s < supportSize; ++s) {
3482           PetscBool isHybrid = PETSC_FALSE;
3483 
3484           PetscCall(DMPlexCellIsHybrid_Internal(dm, support[s], &isHybrid));
3485           if (!isHybrid) continue;
3486           PetscCall(PetscFindInt(support[s], numSubPoints[d + 1], subpoints[d + 1], &subc));
3487           if (subc >= 0) {
3488             const PetscInt ccell = subpoints[d + 1][subc];
3489 
3490             PetscCall(DMPlexGetCone(dm, ccell, &cone));
3491             PetscCall(DMPlexGetConeSize(dm, ccell, &coneSize));
3492             PetscCall(DMPlexGetConeOrientation(dm, ccell, &ornt));
3493             for (c = 0; c < coneSize; ++c) {
3494               if (cone[c] == point) {
3495                 fornt = ornt[c];
3496                 break;
3497               }
3498             }
3499             break;
3500           }
3501         }
3502       }
3503       PetscCall(DMPlexGetConeSize(dm, point, &coneSize));
3504       PetscCall(DMPlexGetConeSize(subdm, subpoint, &subconeSize));
3505       PetscCall(DMPlexGetCone(dm, point, &cone));
3506       PetscCall(DMPlexGetConeOrientation(dm, point, &ornt));
3507       for (c = 0, coneSizeNew = 0; c < coneSize; ++c) {
3508         PetscCall(PetscFindInt(cone[c], numSubPoints[d - 1], subpoints[d - 1], &subc));
3509         if (subc >= 0) {
3510           coneNew[coneSizeNew] = firstSubPoint[d - 1] + subc;
3511           orntNew[coneSizeNew] = ornt[c];
3512           ++coneSizeNew;
3513         }
3514       }
3515       PetscCheck(coneSizeNew == subconeSize, comm, PETSC_ERR_PLIB, "Number of cone points located %" PetscInt_FMT " does not match subcone size %" PetscInt_FMT, coneSizeNew, subconeSize);
3516       PetscCall(DMPlexSetCone(subdm, subpoint, coneNew));
3517       PetscCall(DMPlexSetConeOrientation(subdm, subpoint, orntNew));
3518       if (fornt < 0) PetscCall(DMPlexOrientPoint(subdm, subpoint, fornt));
3519     }
3520   }
3521   PetscCall(PetscFree2(coneNew, orntNew));
3522   PetscCall(DMPlexSymmetrize(subdm));
3523   PetscCall(DMPlexStratify(subdm));
3524   /* Build coordinates */
3525   {
3526     PetscSection coordSection, subCoordSection;
3527     Vec          coordinates, subCoordinates;
3528     PetscScalar *coords, *subCoords;
3529     PetscInt     cdim, numComp, coordSize;
3530     const char  *name;
3531 
3532     PetscCall(DMGetCoordinateDim(dm, &cdim));
3533     PetscCall(DMGetCoordinateSection(dm, &coordSection));
3534     PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
3535     PetscCall(DMGetCoordinateSection(subdm, &subCoordSection));
3536     PetscCall(PetscSectionSetNumFields(subCoordSection, 1));
3537     PetscCall(PetscSectionGetFieldComponents(coordSection, 0, &numComp));
3538     PetscCall(PetscSectionSetFieldComponents(subCoordSection, 0, numComp));
3539     PetscCall(PetscSectionSetChart(subCoordSection, firstSubPoint[0], firstSubPoint[0] + numSubPoints[0]));
3540     for (v = 0; v < numSubPoints[0]; ++v) {
3541       const PetscInt vertex    = subpoints[0][v];
3542       const PetscInt subvertex = firstSubPoint[0] + v;
3543       PetscInt       dof;
3544 
3545       PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3546       PetscCall(PetscSectionSetDof(subCoordSection, subvertex, dof));
3547       PetscCall(PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof));
3548     }
3549     PetscCall(PetscSectionSetUp(subCoordSection));
3550     PetscCall(PetscSectionGetStorageSize(subCoordSection, &coordSize));
3551     PetscCall(VecCreate(PETSC_COMM_SELF, &subCoordinates));
3552     PetscCall(PetscObjectGetName((PetscObject)coordinates, &name));
3553     PetscCall(PetscObjectSetName((PetscObject)subCoordinates, name));
3554     PetscCall(VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE));
3555     PetscCall(VecSetBlockSize(subCoordinates, cdim));
3556     PetscCall(VecSetType(subCoordinates, VECSTANDARD));
3557     PetscCall(VecGetArray(coordinates, &coords));
3558     PetscCall(VecGetArray(subCoordinates, &subCoords));
3559     for (v = 0; v < numSubPoints[0]; ++v) {
3560       const PetscInt vertex    = subpoints[0][v];
3561       const PetscInt subvertex = firstSubPoint[0] + v;
3562       PetscInt       dof, off, sdof, soff, d;
3563 
3564       PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3565       PetscCall(PetscSectionGetOffset(coordSection, vertex, &off));
3566       PetscCall(PetscSectionGetDof(subCoordSection, subvertex, &sdof));
3567       PetscCall(PetscSectionGetOffset(subCoordSection, subvertex, &soff));
3568       PetscCheck(dof == sdof, comm, PETSC_ERR_PLIB, "Coordinate dimension %" PetscInt_FMT " on subvertex %" PetscInt_FMT ", vertex %" PetscInt_FMT " should be %" PetscInt_FMT, sdof, subvertex, vertex, dof);
3569       for (d = 0; d < dof; ++d) subCoords[soff + d] = coords[off + d];
3570     }
3571     PetscCall(VecRestoreArray(coordinates, &coords));
3572     PetscCall(VecRestoreArray(subCoordinates, &subCoords));
3573     PetscCall(DMSetCoordinatesLocal(subdm, subCoordinates));
3574     PetscCall(VecDestroy(&subCoordinates));
3575   }
3576   /* Build SF: We need this complexity because subpoints might not be selected on the owning process */
3577   {
3578     PetscSF            sfPoint, sfPointSub;
3579     IS                 subpIS;
3580     const PetscSFNode *remotePoints;
3581     PetscSFNode       *sremotePoints = NULL, *newLocalPoints = NULL, *newOwners = NULL;
3582     const PetscInt    *localPoints, *subpoints, *rootdegree;
3583     PetscInt          *slocalPoints = NULL, *sortedPoints = NULL, *sortedIndices = NULL;
3584     PetscInt           numRoots, numLeaves, numSubpoints = 0, numSubroots, numSubleaves = 0, l, sl = 0, ll = 0, pStart, pEnd, p;
3585     PetscMPIInt        rank, size;
3586 
3587     PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
3588     PetscCallMPI(MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size));
3589     PetscCall(DMGetPointSF(dm, &sfPoint));
3590     PetscCall(DMGetPointSF(subdm, &sfPointSub));
3591     PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
3592     PetscCall(DMPlexGetChart(subdm, NULL, &numSubroots));
3593     PetscCall(DMPlexGetSubpointIS(subdm, &subpIS));
3594     if (subpIS) {
3595       PetscBool sorted = PETSC_TRUE;
3596 
3597       PetscCall(ISGetIndices(subpIS, &subpoints));
3598       PetscCall(ISGetLocalSize(subpIS, &numSubpoints));
3599       for (p = 1; p < numSubpoints; ++p) sorted = sorted && (subpoints[p] >= subpoints[p - 1]) ? PETSC_TRUE : PETSC_FALSE;
3600       if (!sorted) {
3601         PetscCall(PetscMalloc2(numSubpoints, &sortedPoints, numSubpoints, &sortedIndices));
3602         for (p = 0; p < numSubpoints; ++p) sortedIndices[p] = p;
3603         PetscCall(PetscArraycpy(sortedPoints, subpoints, numSubpoints));
3604         PetscCall(PetscSortIntWithArray(numSubpoints, sortedPoints, sortedIndices));
3605       }
3606     }
3607     PetscCall(PetscSFGetGraph(sfPoint, &numRoots, &numLeaves, &localPoints, &remotePoints));
3608     if (numRoots >= 0) {
3609       PetscCall(PetscSFComputeDegreeBegin(sfPoint, &rootdegree));
3610       PetscCall(PetscSFComputeDegreeEnd(sfPoint, &rootdegree));
3611       PetscCall(PetscMalloc2(pEnd - pStart, &newLocalPoints, numRoots, &newOwners));
3612       for (p = 0; p < pEnd - pStart; ++p) {
3613         newLocalPoints[p].rank  = -2;
3614         newLocalPoints[p].index = -2;
3615       }
3616       for (p = pStart; p < pEnd; ++p) {
3617         newOwners[p - pStart].rank  = -3;
3618         newOwners[p - pStart].index = -3;
3619       }
3620       if (sanitizeSubmesh) {
3621         /* A subpoint is forced to be owned by a rank that owns */
3622         /* a subcell that contains the subpoint in its closure. */
3623         PetscInt  cStart, cEnd, c, clSize, cl;
3624         PetscInt *ownedCells, *closure = NULL;
3625 
3626         /* claim ownership */
3627         for (p = 0; p < numSubpoints; ++p) {
3628           const PetscInt point = subpoints[p];
3629 
3630           newLocalPoints[point - pStart].rank  = rank;
3631           newLocalPoints[point - pStart].index = p;
3632         }
3633         PetscCall(DMGetDimension(subdm, &sdim));
3634         PetscCall(DMPlexGetDepthStratum(dm, sdim, &cStart, &cEnd));
3635         PetscCall(PetscMalloc1(cEnd - cStart, &ownedCells));
3636         for (c = cStart; c < cEnd; ++c) ownedCells[c - cStart] = 0;
3637         for (p = 0; p < numSubpoints; ++p) {
3638           c = subpoints[p];
3639           if (c >= cStart && c < cEnd) ownedCells[c - cStart] = 1;
3640         }
3641         for (l = 0; l < numLeaves; ++l) {
3642           c = localPoints ? localPoints[l] : l;
3643           if (c >= cStart && c < cEnd) ownedCells[c - cStart] = 0;
3644         }
3645         for (c = cStart; c < cEnd; ++c) {
3646           if (ownedCells[c - cStart] == 0) continue;
3647           PetscCall(DMPlexGetTransitiveClosure(dm, c, PETSC_TRUE, &clSize, &closure));
3648           for (cl = 0; cl < clSize * 2; cl += 2) {
3649             p = closure[cl];
3650             if (newLocalPoints[p - pStart].rank < size) newLocalPoints[p - pStart].rank += size;
3651           }
3652           PetscCall(DMPlexRestoreTransitiveClosure(dm, c, PETSC_TRUE, &clSize, &closure));
3653         }
3654         PetscCall(PetscFree(ownedCells));
3655         for (p = 0; p < numRoots; ++p) {
3656           newOwners[p].rank  = newLocalPoints[p].rank;
3657           newOwners[p].index = newLocalPoints[p].index;
3658         }
3659       } else {
3660         /* Set subleaves */
3661         for (l = 0; l < numLeaves; ++l) {
3662           const PetscInt point    = localPoints[l];
3663           const PetscInt subpoint = DMPlexFilterPointPerm_Internal(point, 0, numSubpoints, sortedPoints ? sortedPoints : subpoints, sortedIndices);
3664 
3665           if (subpoint < 0) continue;
3666           newLocalPoints[point - pStart].rank  = rank;
3667           newLocalPoints[point - pStart].index = subpoint;
3668           ++numSubleaves;
3669         }
3670         /* Must put in owned subpoints */
3671         for (p = 0; p < numSubpoints; ++p) {
3672           /* Hold on to currently owned points */
3673           if (rootdegree[subpoints[p] - pStart]) newOwners[subpoints[p] - pStart].rank = rank + size;
3674           else newOwners[subpoints[p] - pStart].rank = rank;
3675           newOwners[subpoints[p] - pStart].index = p;
3676         }
3677       }
3678       PetscCall(PetscSFReduceBegin(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC));
3679       PetscCall(PetscSFReduceEnd(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC));
3680       for (p = pStart; p < pEnd; ++p)
3681         if (newOwners[p - pStart].rank >= size) newOwners[p - pStart].rank -= size;
3682       if (ownershipTransferSF) {
3683         PetscSFNode *iremote1 = NULL, *newOwners1 = NULL;
3684         PetscInt    *ilocal1 = NULL;
3685         PetscInt     nleaves1, point;
3686 
3687         for (p = 0; p < numSubpoints; ++p) {
3688           point                                = subpoints[p];
3689           newLocalPoints[point - pStart].index = point;
3690         }
3691         PetscCall(PetscMalloc1(numRoots, &newOwners1));
3692         for (p = 0; p < numRoots; ++p) {
3693           newOwners1[p].rank  = -1;
3694           newOwners1[p].index = -1;
3695         }
3696         PetscCall(PetscSFReduceBegin(sfPoint, MPIU_2INT, newLocalPoints, newOwners1, MPI_MAXLOC));
3697         PetscCall(PetscSFReduceEnd(sfPoint, MPIU_2INT, newLocalPoints, newOwners1, MPI_MAXLOC));
3698         for (p = 0, nleaves1 = 0; p < numRoots; ++p) {
3699           if (newOwners[p].rank >= 0 && newOwners[p].rank != rank) { ++nleaves1; }
3700         }
3701         PetscCall(PetscMalloc1(nleaves1, &ilocal1));
3702         PetscCall(PetscMalloc1(nleaves1, &iremote1));
3703         for (p = 0, nleaves1 = 0; p < numRoots; ++p) {
3704           if (newOwners[p].rank >= 0 && newOwners[p].rank != rank) {
3705             ilocal1[nleaves1]        = pStart + p;
3706             iremote1[nleaves1].rank  = newOwners[p].rank;
3707             iremote1[nleaves1].index = newOwners1[p].index;
3708             ++nleaves1;
3709           }
3710         }
3711         PetscCall(PetscFree(newOwners1));
3712         PetscCall(PetscSFCreate(PetscObjectComm((PetscObject)sfPoint), ownershipTransferSF));
3713         PetscCall(PetscSFSetFromOptions(*ownershipTransferSF));
3714         PetscCall(PetscSFSetGraph(*ownershipTransferSF, pEnd - pStart, nleaves1, ilocal1, PETSC_OWN_POINTER, iremote1, PETSC_OWN_POINTER));
3715       }
3716       if (sanitizeSubmesh) {
3717         for (p = pStart; p < pEnd; ++p) {
3718           newLocalPoints[p - pStart].rank  = newOwners[p - pStart].rank;
3719           newLocalPoints[p - pStart].index = newOwners[p - pStart].index;
3720         }
3721       }
3722       PetscCall(PetscSFBcastBegin(sfPoint, MPIU_2INT, newOwners, newLocalPoints, MPI_REPLACE));
3723       PetscCall(PetscSFBcastEnd(sfPoint, MPIU_2INT, newOwners, newLocalPoints, MPI_REPLACE));
3724       if (sanitizeSubmesh) {
3725         for (p = 0; p < numSubpoints; ++p) {
3726           const PetscInt point = subpoints[p];
3727 
3728           if (newLocalPoints[point - pStart].rank >= 0 && newLocalPoints[point - pStart].rank != rank) ++sl;
3729         }
3730         PetscCall(PetscMalloc1(sl, &slocalPoints));
3731         PetscCall(PetscMalloc1(sl, &sremotePoints));
3732         for (p = 0, sl = 0; p < numSubpoints; ++p) {
3733           const PetscInt point = subpoints[p];
3734 
3735           if (newLocalPoints[point - pStart].rank >= 0 && newLocalPoints[point - pStart].rank != rank) {
3736             slocalPoints[sl]        = p;
3737             sremotePoints[sl].rank  = newLocalPoints[point - pStart].rank;
3738             sremotePoints[sl].index = newLocalPoints[point - pStart].index;
3739             PetscCheck(sremotePoints[sl].rank >= 0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote rank");
3740             PetscCheck(sremotePoints[sl].index >= 0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote subpoint");
3741             ++sl;
3742           }
3743         }
3744       } else {
3745         PetscCall(PetscMalloc1(numSubleaves, &slocalPoints));
3746         PetscCall(PetscMalloc1(numSubleaves, &sremotePoints));
3747         for (l = 0; l < numLeaves; ++l) {
3748           const PetscInt point    = localPoints[l];
3749           const PetscInt subpoint = DMPlexFilterPointPerm_Internal(point, 0, numSubpoints, sortedPoints ? sortedPoints : subpoints, sortedIndices);
3750 
3751           if (subpoint < 0) continue;
3752           if (newLocalPoints[point].rank == rank) {
3753             ++ll;
3754             continue;
3755           }
3756           slocalPoints[sl]        = subpoint;
3757           sremotePoints[sl].rank  = newLocalPoints[point].rank;
3758           sremotePoints[sl].index = newLocalPoints[point].index;
3759           PetscCheck(sremotePoints[sl].rank >= 0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote rank for local point %" PetscInt_FMT, point);
3760           PetscCheck(sremotePoints[sl].index >= 0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote subpoint for local point %" PetscInt_FMT, point);
3761           ++sl;
3762         }
3763         PetscCheck(sl + ll == numSubleaves, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatch in number of subleaves %" PetscInt_FMT " + %" PetscInt_FMT " != %" PetscInt_FMT, sl, ll, numSubleaves);
3764       }
3765       PetscCall(PetscFree2(newLocalPoints, newOwners));
3766       PetscCall(PetscSFSetGraph(sfPointSub, numSubroots, sl, slocalPoints, PETSC_OWN_POINTER, sremotePoints, PETSC_OWN_POINTER));
3767     }
3768     if (subpIS) PetscCall(ISRestoreIndices(subpIS, &subpoints));
3769     PetscCall(PetscFree2(sortedPoints, sortedIndices));
3770   }
3771   /* Filter labels */
3772   PetscCall(DMPlexFilterLabels_Internal(dm, numSubPoints, subpoints, firstSubPoint, subdm));
3773   /* Cleanup */
3774   for (d = 0; d <= sdim; ++d) {
3775     if (subpointIS[d]) PetscCall(ISRestoreIndices(subpointIS[d], &subpoints[d]));
3776     PetscCall(ISDestroy(&subpointIS[d]));
3777   }
3778   PetscCall(PetscFree4(numSubPoints, firstSubPoint, subpointIS, subpoints));
3779   PetscFunctionReturn(PETSC_SUCCESS);
3780 }
3781 
3782 static PetscErrorCode DMPlexCreateSubmesh_Interpolated(DM dm, DMLabel vertexLabel, PetscInt value, PetscBool markedFaces, DM subdm)
3783 {
3784   PetscFunctionBegin;
3785   PetscCall(DMPlexCreateSubmeshGeneric_Interpolated(dm, vertexLabel, value, markedFaces, PETSC_FALSE, 1, PETSC_FALSE, PETSC_FALSE, NULL, subdm));
3786   PetscFunctionReturn(PETSC_SUCCESS);
3787 }
3788 
3789 /*@
3790   DMPlexCreateSubmesh - Extract a hypersurface from the mesh using vertices defined by a label
3791 
3792   Input Parameters:
3793 + dm          - The original mesh
3794 . vertexLabel - The `DMLabel` marking points contained in the surface
3795 . value       - The label value to use
3796 - markedFaces - `PETSC_TRUE` if surface faces are marked in addition to vertices, `PETSC_FALSE` if only vertices are marked
3797 
3798   Output Parameter:
3799 . subdm - The surface mesh
3800 
3801   Level: developer
3802 
3803   Note:
3804   This function produces a `DMLabel` mapping original points in the submesh to their depth. This can be obtained using `DMPlexGetSubpointMap()`.
3805 
3806 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetSubpointMap()`, `DMGetLabel()`, `DMLabelSetValue()`
3807 @*/
3808 PetscErrorCode DMPlexCreateSubmesh(DM dm, DMLabel vertexLabel, PetscInt value, PetscBool markedFaces, DM *subdm)
3809 {
3810   DMPlexInterpolatedFlag interpolated;
3811   PetscInt               dim, cdim;
3812 
3813   PetscFunctionBegin;
3814   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3815   PetscAssertPointer(subdm, 5);
3816   PetscCall(DMGetDimension(dm, &dim));
3817   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), subdm));
3818   PetscCall(DMSetType(*subdm, DMPLEX));
3819   PetscCall(DMSetDimension(*subdm, dim - 1));
3820   PetscCall(DMGetCoordinateDim(dm, &cdim));
3821   PetscCall(DMSetCoordinateDim(*subdm, cdim));
3822   PetscCall(DMPlexIsInterpolated(dm, &interpolated));
3823   PetscCheck(interpolated != DMPLEX_INTERPOLATED_PARTIAL, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Not for partially interpolated meshes");
3824   if (interpolated) {
3825     PetscCall(DMPlexCreateSubmesh_Interpolated(dm, vertexLabel, value, markedFaces, *subdm));
3826   } else {
3827     PetscCall(DMPlexCreateSubmesh_Uninterpolated(dm, vertexLabel, value, *subdm));
3828   }
3829   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_TRUE, *subdm));
3830   PetscFunctionReturn(PETSC_SUCCESS);
3831 }
3832 
3833 static PetscErrorCode DMPlexCreateCohesiveSubmesh_Uninterpolated(DM dm, PetscBool hasLagrange, const char label[], PetscInt value, DM subdm)
3834 {
3835   MPI_Comm        comm;
3836   DMLabel         subpointMap;
3837   IS              subvertexIS;
3838   const PetscInt *subVertices;
3839   PetscInt        numSubVertices, firstSubVertex, numSubCells, *subCells = NULL;
3840   PetscInt       *subface, maxConeSize, numSubFaces, firstSubFace, newFacePoint, nFV;
3841   PetscInt        c, f;
3842 
3843   PetscFunctionBegin;
3844   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
3845   /* Create subpointMap which marks the submesh */
3846   PetscCall(DMLabelCreate(PETSC_COMM_SELF, "subpoint_map", &subpointMap));
3847   PetscCall(DMPlexSetSubpointMap(subdm, subpointMap));
3848   PetscCall(DMPlexMarkCohesiveSubmesh_Uninterpolated(dm, hasLagrange, label, value, subpointMap, &numSubFaces, &nFV, &subCells, subdm));
3849   /* Setup chart */
3850   PetscCall(DMLabelGetStratumSize(subpointMap, 0, &numSubVertices));
3851   PetscCall(DMLabelGetStratumSize(subpointMap, 2, &numSubCells));
3852   PetscCall(DMPlexSetChart(subdm, 0, numSubCells + numSubFaces + numSubVertices));
3853   PetscCall(DMPlexSetVTKCellHeight(subdm, 1));
3854   /* Set cone sizes */
3855   firstSubVertex = numSubCells;
3856   firstSubFace   = numSubCells + numSubVertices;
3857   newFacePoint   = firstSubFace;
3858   PetscCall(DMLabelGetStratumIS(subpointMap, 0, &subvertexIS));
3859   if (subvertexIS) PetscCall(ISGetIndices(subvertexIS, &subVertices));
3860   for (c = 0; c < numSubCells; ++c) PetscCall(DMPlexSetConeSize(subdm, c, 1));
3861   for (f = firstSubFace; f < firstSubFace + numSubFaces; ++f) PetscCall(DMPlexSetConeSize(subdm, f, nFV));
3862   PetscCall(DMSetUp(subdm));
3863   PetscCall(DMLabelDestroy(&subpointMap));
3864   /* Create face cones */
3865   PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, NULL));
3866   PetscCall(DMGetWorkArray(subdm, maxConeSize, MPIU_INT, (void **)&subface));
3867   for (c = 0; c < numSubCells; ++c) {
3868     const PetscInt  cell    = subCells[c];
3869     const PetscInt  subcell = c;
3870     const PetscInt *cone, *cells;
3871     PetscBool       isHybrid = PETSC_FALSE;
3872     PetscInt        numCells, subVertex, p, v;
3873 
3874     PetscCall(DMPlexCellIsHybrid_Internal(dm, cell, &isHybrid));
3875     if (!isHybrid) continue;
3876     PetscCall(DMPlexGetCone(dm, cell, &cone));
3877     for (v = 0; v < nFV; ++v) {
3878       PetscCall(PetscFindInt(cone[v], numSubVertices, subVertices, &subVertex));
3879       subface[v] = firstSubVertex + subVertex;
3880     }
3881     PetscCall(DMPlexSetCone(subdm, newFacePoint, subface));
3882     PetscCall(DMPlexSetCone(subdm, subcell, &newFacePoint));
3883     PetscCall(DMPlexGetJoin(dm, nFV, cone, &numCells, &cells));
3884     /* Not true in parallel
3885     PetscCheck(numCells == 2,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive cells should separate two cells"); */
3886     for (p = 0; p < numCells; ++p) {
3887       PetscInt  negsubcell;
3888       PetscBool isHybrid = PETSC_FALSE;
3889 
3890       PetscCall(DMPlexCellIsHybrid_Internal(dm, cells[p], &isHybrid));
3891       if (isHybrid) continue;
3892       /* I know this is a crap search */
3893       for (negsubcell = 0; negsubcell < numSubCells; ++negsubcell) {
3894         if (subCells[negsubcell] == cells[p]) break;
3895       }
3896       PetscCheck(negsubcell != numSubCells, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not find negative face neighbor for cohesive cell %" PetscInt_FMT, cell);
3897       PetscCall(DMPlexSetCone(subdm, negsubcell, &newFacePoint));
3898     }
3899     PetscCall(DMPlexRestoreJoin(dm, nFV, cone, &numCells, &cells));
3900     ++newFacePoint;
3901   }
3902   PetscCall(DMRestoreWorkArray(subdm, maxConeSize, MPIU_INT, (void **)&subface));
3903   PetscCall(DMPlexSymmetrize(subdm));
3904   PetscCall(DMPlexStratify(subdm));
3905   /* Build coordinates */
3906   {
3907     PetscSection coordSection, subCoordSection;
3908     Vec          coordinates, subCoordinates;
3909     PetscScalar *coords, *subCoords;
3910     PetscInt     cdim, numComp, coordSize, v;
3911     const char  *name;
3912 
3913     PetscCall(DMGetCoordinateDim(dm, &cdim));
3914     PetscCall(DMGetCoordinateSection(dm, &coordSection));
3915     PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
3916     PetscCall(DMGetCoordinateSection(subdm, &subCoordSection));
3917     PetscCall(PetscSectionSetNumFields(subCoordSection, 1));
3918     PetscCall(PetscSectionGetFieldComponents(coordSection, 0, &numComp));
3919     PetscCall(PetscSectionSetFieldComponents(subCoordSection, 0, numComp));
3920     PetscCall(PetscSectionSetChart(subCoordSection, firstSubVertex, firstSubVertex + numSubVertices));
3921     for (v = 0; v < numSubVertices; ++v) {
3922       const PetscInt vertex    = subVertices[v];
3923       const PetscInt subvertex = firstSubVertex + v;
3924       PetscInt       dof;
3925 
3926       PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3927       PetscCall(PetscSectionSetDof(subCoordSection, subvertex, dof));
3928       PetscCall(PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof));
3929     }
3930     PetscCall(PetscSectionSetUp(subCoordSection));
3931     PetscCall(PetscSectionGetStorageSize(subCoordSection, &coordSize));
3932     PetscCall(VecCreate(PETSC_COMM_SELF, &subCoordinates));
3933     PetscCall(PetscObjectGetName((PetscObject)coordinates, &name));
3934     PetscCall(PetscObjectSetName((PetscObject)subCoordinates, name));
3935     PetscCall(VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE));
3936     PetscCall(VecSetBlockSize(subCoordinates, cdim));
3937     PetscCall(VecSetType(subCoordinates, VECSTANDARD));
3938     PetscCall(VecGetArray(coordinates, &coords));
3939     PetscCall(VecGetArray(subCoordinates, &subCoords));
3940     for (v = 0; v < numSubVertices; ++v) {
3941       const PetscInt vertex    = subVertices[v];
3942       const PetscInt subvertex = firstSubVertex + v;
3943       PetscInt       dof, off, sdof, soff, d;
3944 
3945       PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3946       PetscCall(PetscSectionGetOffset(coordSection, vertex, &off));
3947       PetscCall(PetscSectionGetDof(subCoordSection, subvertex, &sdof));
3948       PetscCall(PetscSectionGetOffset(subCoordSection, subvertex, &soff));
3949       PetscCheck(dof == sdof, comm, PETSC_ERR_PLIB, "Coordinate dimension %" PetscInt_FMT " on subvertex %" PetscInt_FMT ", vertex %" PetscInt_FMT " should be %" PetscInt_FMT, sdof, subvertex, vertex, dof);
3950       for (d = 0; d < dof; ++d) subCoords[soff + d] = coords[off + d];
3951     }
3952     PetscCall(VecRestoreArray(coordinates, &coords));
3953     PetscCall(VecRestoreArray(subCoordinates, &subCoords));
3954     PetscCall(DMSetCoordinatesLocal(subdm, subCoordinates));
3955     PetscCall(VecDestroy(&subCoordinates));
3956   }
3957   /* Build SF */
3958   CHKMEMQ;
3959   {
3960     PetscSF            sfPoint, sfPointSub;
3961     const PetscSFNode *remotePoints;
3962     PetscSFNode       *sremotePoints, *newLocalPoints, *newOwners;
3963     const PetscInt    *localPoints;
3964     PetscInt          *slocalPoints;
3965     PetscInt           numRoots, numLeaves, numSubRoots = numSubCells + numSubFaces + numSubVertices, numSubLeaves = 0, l, sl, ll, pStart, pEnd, p, vStart, vEnd;
3966     PetscMPIInt        rank;
3967 
3968     PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
3969     PetscCall(DMGetPointSF(dm, &sfPoint));
3970     PetscCall(DMGetPointSF(subdm, &sfPointSub));
3971     PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
3972     PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
3973     PetscCall(PetscSFGetGraph(sfPoint, &numRoots, &numLeaves, &localPoints, &remotePoints));
3974     if (numRoots >= 0) {
3975       /* Only vertices should be shared */
3976       PetscCall(PetscMalloc2(pEnd - pStart, &newLocalPoints, numRoots, &newOwners));
3977       for (p = 0; p < pEnd - pStart; ++p) {
3978         newLocalPoints[p].rank  = -2;
3979         newLocalPoints[p].index = -2;
3980       }
3981       /* Set subleaves */
3982       for (l = 0; l < numLeaves; ++l) {
3983         const PetscInt point    = localPoints[l];
3984         const PetscInt subPoint = DMPlexFilterPoint_Internal(point, firstSubVertex, numSubVertices, subVertices);
3985 
3986         PetscCheck(!(point < vStart) || !(point >= vEnd), PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Should not be mapping anything but vertices, %" PetscInt_FMT, point);
3987         if (subPoint < 0) continue;
3988         newLocalPoints[point - pStart].rank  = rank;
3989         newLocalPoints[point - pStart].index = subPoint;
3990         ++numSubLeaves;
3991       }
3992       /* Must put in owned subpoints */
3993       for (p = pStart; p < pEnd; ++p) {
3994         const PetscInt subPoint = DMPlexFilterPoint_Internal(p, firstSubVertex, numSubVertices, subVertices);
3995 
3996         if (subPoint < 0) {
3997           newOwners[p - pStart].rank  = -3;
3998           newOwners[p - pStart].index = -3;
3999         } else {
4000           newOwners[p - pStart].rank  = rank;
4001           newOwners[p - pStart].index = subPoint;
4002         }
4003       }
4004       PetscCall(PetscSFReduceBegin(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC));
4005       PetscCall(PetscSFReduceEnd(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC));
4006       PetscCall(PetscSFBcastBegin(sfPoint, MPIU_2INT, newOwners, newLocalPoints, MPI_REPLACE));
4007       PetscCall(PetscSFBcastEnd(sfPoint, MPIU_2INT, newOwners, newLocalPoints, MPI_REPLACE));
4008       PetscCall(PetscMalloc1(numSubLeaves, &slocalPoints));
4009       PetscCall(PetscMalloc1(numSubLeaves, &sremotePoints));
4010       for (l = 0, sl = 0, ll = 0; l < numLeaves; ++l) {
4011         const PetscInt point    = localPoints[l];
4012         const PetscInt subPoint = DMPlexFilterPoint_Internal(point, firstSubVertex, numSubVertices, subVertices);
4013 
4014         if (subPoint < 0) continue;
4015         if (newLocalPoints[point].rank == rank) {
4016           ++ll;
4017           continue;
4018         }
4019         slocalPoints[sl]        = subPoint;
4020         sremotePoints[sl].rank  = newLocalPoints[point].rank;
4021         sremotePoints[sl].index = newLocalPoints[point].index;
4022         PetscCheck(sremotePoints[sl].rank >= 0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote rank for local point %" PetscInt_FMT, point);
4023         PetscCheck(sremotePoints[sl].index >= 0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote subpoint for local point %" PetscInt_FMT, point);
4024         ++sl;
4025       }
4026       PetscCall(PetscFree2(newLocalPoints, newOwners));
4027       PetscCheck(sl + ll == numSubLeaves, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatch in number of subleaves %" PetscInt_FMT " + %" PetscInt_FMT " != %" PetscInt_FMT, sl, ll, numSubLeaves);
4028       PetscCall(PetscSFSetGraph(sfPointSub, numSubRoots, sl, slocalPoints, PETSC_OWN_POINTER, sremotePoints, PETSC_OWN_POINTER));
4029     }
4030   }
4031   CHKMEMQ;
4032   /* Cleanup */
4033   if (subvertexIS) PetscCall(ISRestoreIndices(subvertexIS, &subVertices));
4034   PetscCall(ISDestroy(&subvertexIS));
4035   PetscCall(PetscFree(subCells));
4036   PetscFunctionReturn(PETSC_SUCCESS);
4037 }
4038 
4039 static PetscErrorCode DMPlexCreateCohesiveSubmesh_Interpolated(DM dm, const char labelname[], PetscInt value, DM subdm)
4040 {
4041   DMLabel label = NULL;
4042 
4043   PetscFunctionBegin;
4044   if (labelname) PetscCall(DMGetLabel(dm, labelname, &label));
4045   PetscCall(DMPlexCreateSubmeshGeneric_Interpolated(dm, label, value, PETSC_FALSE, PETSC_TRUE, 1, PETSC_FALSE, PETSC_FALSE, NULL, subdm));
4046   PetscFunctionReturn(PETSC_SUCCESS);
4047 }
4048 
4049 /*@C
4050   DMPlexCreateCohesiveSubmesh - Extract from a mesh with cohesive cells the hypersurface defined by one face of the cells. Optionally, a label can be given to restrict the cells.
4051 
4052   Input Parameters:
4053 + dm          - The original mesh
4054 . hasLagrange - The mesh has Lagrange unknowns in the cohesive cells
4055 . label       - A label name, or `NULL`
4056 - value       - A label value
4057 
4058   Output Parameter:
4059 . subdm - The surface mesh
4060 
4061   Level: developer
4062 
4063   Note:
4064   This function produces a `DMLabel` mapping original points in the submesh to their depth. This can be obtained using `DMPlexGetSubpointMap()`.
4065 
4066 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetSubpointMap()`, `DMPlexCreateSubmesh()`
4067 @*/
4068 PetscErrorCode DMPlexCreateCohesiveSubmesh(DM dm, PetscBool hasLagrange, const char label[], PetscInt value, DM *subdm)
4069 {
4070   PetscInt dim, cdim, depth;
4071 
4072   PetscFunctionBegin;
4073   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4074   PetscAssertPointer(subdm, 5);
4075   PetscCall(DMGetDimension(dm, &dim));
4076   PetscCall(DMPlexGetDepth(dm, &depth));
4077   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), subdm));
4078   PetscCall(DMSetType(*subdm, DMPLEX));
4079   PetscCall(DMSetDimension(*subdm, dim - 1));
4080   PetscCall(DMGetCoordinateDim(dm, &cdim));
4081   PetscCall(DMSetCoordinateDim(*subdm, cdim));
4082   if (depth == dim) {
4083     PetscCall(DMPlexCreateCohesiveSubmesh_Interpolated(dm, label, value, *subdm));
4084   } else {
4085     PetscCall(DMPlexCreateCohesiveSubmesh_Uninterpolated(dm, hasLagrange, label, value, *subdm));
4086   }
4087   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_TRUE, *subdm));
4088   PetscFunctionReturn(PETSC_SUCCESS);
4089 }
4090 
4091 /*@
4092   DMPlexReorderCohesiveSupports - Ensure that face supports for cohesive end caps are ordered
4093 
4094   Not Collective
4095 
4096   Input Parameter:
4097 . dm - The `DM` containing cohesive cells
4098 
4099   Level: developer
4100 
4101   Note:
4102   For the negative size (first) face, the cohesive cell should be first in the support, and for
4103   the positive side (second) face, the cohesive cell should be second in the support.
4104 
4105 .seealso: `DMPlexConstructCohesiveCells()`, `DMPlexCreateCohesiveSubmesh()`
4106 @*/
4107 PetscErrorCode DMPlexReorderCohesiveSupports(DM dm)
4108 {
4109   PetscInt dim, cStart, cEnd;
4110 
4111   PetscFunctionBegin;
4112   PetscCall(DMGetDimension(dm, &dim));
4113   PetscCall(DMPlexGetTensorPrismBounds_Internal(dm, dim, &cStart, &cEnd));
4114   for (PetscInt c = cStart; c < cEnd; ++c) {
4115     const PetscInt *cone;
4116     PetscInt        coneSize;
4117 
4118     PetscCall(DMPlexGetConeSize(dm, c, &coneSize));
4119     PetscCall(DMPlexGetCone(dm, c, &cone));
4120     PetscCheck(coneSize >= 2, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Hybrid cell %" PetscInt_FMT " cone size %" PetscInt_FMT " must be >= 2", c, coneSize);
4121     for (PetscInt s = 0; s < 2; ++s) {
4122       const PetscInt *supp;
4123       PetscInt        suppSize, newsupp[2];
4124 
4125       PetscCall(DMPlexGetSupportSize(dm, cone[s], &suppSize));
4126       PetscCall(DMPlexGetSupport(dm, cone[s], &supp));
4127       if (suppSize == 2) {
4128         /* Reorder hybrid end cap support */
4129         if (supp[s] == c) {
4130           newsupp[0] = supp[1];
4131           newsupp[1] = supp[0];
4132         } else {
4133           newsupp[0] = supp[0];
4134           newsupp[1] = supp[1];
4135         }
4136         PetscCheck(newsupp[1 - s] == c, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Hybrid end cap %" PetscInt_FMT " support entry %" PetscInt_FMT " != %" PetscInt_FMT " hybrid cell", cone[s], newsupp[1 - s], c);
4137         PetscCall(DMPlexSetSupport(dm, cone[s], newsupp));
4138       }
4139     }
4140   }
4141   PetscFunctionReturn(PETSC_SUCCESS);
4142 }
4143 
4144 /*@
4145   DMPlexFilter - Extract a subset of mesh cells defined by a label as a separate mesh
4146 
4147   Input Parameters:
4148 + dm              - The original mesh
4149 . cellLabel       - The `DMLabel` marking cells contained in the new mesh
4150 . value           - The label value to use
4151 . ignoreLabelHalo - The flag indicating if labeled points that are in the halo are ignored
4152 - sanitizeSubmesh - The flag indicating if a subpoint is forced to be owned by a rank that owns a subcell that contains that point in its closure
4153 
4154   Output Parameters:
4155 + ownershipTransferSF - The `PetscSF` representing the ownership transfers between parent local meshes due to submeshing.
4156 - subdm               - The new mesh
4157 
4158   Level: developer
4159 
4160   Note:
4161   This function produces a `DMLabel` mapping original points in the submesh to their depth. This can be obtained using `DMPlexGetSubpointMap()`.
4162 
4163   On a given rank, the leaves of the ownershipTransferSF are the points in the local mesh that this rank gives up ownership of (in the submesh), and
4164   the remote locations for these leaves are tuples of rank and point that represent the new owners.
4165 
4166 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexGetSubpointMap()`, `DMGetLabel()`, `DMLabelSetValue()`, `DMPlexCreateSubmesh()`
4167 @*/
4168 PetscErrorCode DMPlexFilter(DM dm, DMLabel cellLabel, PetscInt value, PetscBool ignoreLabelHalo, PetscBool sanitizeSubmesh, PetscSF *ownershipTransferSF, DM *subdm)
4169 {
4170   PetscInt dim, overlap;
4171 
4172   PetscFunctionBegin;
4173   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4174   if (ownershipTransferSF) PetscAssertPointer(ownershipTransferSF, 6);
4175   PetscAssertPointer(subdm, 7);
4176   PetscCall(DMGetDimension(dm, &dim));
4177   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), subdm));
4178   PetscCall(DMSetType(*subdm, DMPLEX));
4179   /* Extract submesh in place, could be empty on some procs, could have inconsistency if procs do not both extract a shared cell */
4180   PetscCall(DMPlexCreateSubmeshGeneric_Interpolated(dm, cellLabel, value, PETSC_FALSE, PETSC_FALSE, 0, ignoreLabelHalo, sanitizeSubmesh, ownershipTransferSF, *subdm));
4181   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_TRUE, *subdm));
4182   // It is possible to obtain a surface mesh where some faces are in SF
4183   //   We should either mark the mesh as having an overlap, or delete these from the SF
4184   PetscCall(DMPlexGetOverlap(dm, &overlap));
4185   if (!overlap) {
4186     PetscSF         sf;
4187     const PetscInt *leaves;
4188     PetscInt        cStart, cEnd, Nl;
4189     PetscBool       hasSubcell = PETSC_FALSE, ghasSubcell;
4190 
4191     PetscCall(DMPlexGetHeightStratum(*subdm, 0, &cStart, &cEnd));
4192     PetscCall(DMGetPointSF(*subdm, &sf));
4193     PetscCall(PetscSFGetGraph(sf, NULL, &Nl, &leaves, NULL));
4194     for (PetscInt l = 0; l < Nl; ++l) {
4195       const PetscInt point = leaves ? leaves[l] : l;
4196 
4197       if (point >= cStart && point < cEnd) {
4198         hasSubcell = PETSC_TRUE;
4199         break;
4200       }
4201     }
4202     PetscCall(MPIU_Allreduce(&hasSubcell, &ghasSubcell, 1, MPIU_BOOL, MPI_LOR, PetscObjectComm((PetscObject)dm)));
4203     if (ghasSubcell) PetscCall(DMPlexSetOverlap(*subdm, NULL, 1));
4204   }
4205   PetscFunctionReturn(PETSC_SUCCESS);
4206 }
4207 
4208 /*@
4209   DMPlexGetSubpointMap - Returns a `DMLabel` with point dimension as values
4210 
4211   Input Parameter:
4212 . dm - The submesh `DM`
4213 
4214   Output Parameter:
4215 . subpointMap - The `DMLabel` of all the points from the original mesh in this submesh, or `NULL` if this is not a submesh
4216 
4217   Level: developer
4218 
4219 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateSubmesh()`, `DMPlexGetSubpointIS()`
4220 @*/
4221 PetscErrorCode DMPlexGetSubpointMap(DM dm, DMLabel *subpointMap)
4222 {
4223   PetscFunctionBegin;
4224   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4225   PetscAssertPointer(subpointMap, 2);
4226   *subpointMap = ((DM_Plex *)dm->data)->subpointMap;
4227   PetscFunctionReturn(PETSC_SUCCESS);
4228 }
4229 
4230 /*@
4231   DMPlexSetSubpointMap - Sets the `DMLabel` with point dimension as values
4232 
4233   Input Parameters:
4234 + dm          - The submesh `DM`
4235 - subpointMap - The `DMLabel` of all the points from the original mesh in this submesh
4236 
4237   Level: developer
4238 
4239   Note:
4240   Should normally not be called by the user, since it is set in `DMPlexCreateSubmesh()`
4241 
4242 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateSubmesh()`, `DMPlexGetSubpointIS()`
4243 @*/
4244 PetscErrorCode DMPlexSetSubpointMap(DM dm, DMLabel subpointMap)
4245 {
4246   DM_Plex *mesh = (DM_Plex *)dm->data;
4247   DMLabel  tmp;
4248 
4249   PetscFunctionBegin;
4250   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4251   tmp               = mesh->subpointMap;
4252   mesh->subpointMap = subpointMap;
4253   PetscCall(PetscObjectReference((PetscObject)mesh->subpointMap));
4254   PetscCall(DMLabelDestroy(&tmp));
4255   PetscFunctionReturn(PETSC_SUCCESS);
4256 }
4257 
4258 static PetscErrorCode DMPlexCreateSubpointIS_Internal(DM dm, IS *subpointIS)
4259 {
4260   DMLabel  spmap;
4261   PetscInt depth, d;
4262 
4263   PetscFunctionBegin;
4264   PetscCall(DMPlexGetSubpointMap(dm, &spmap));
4265   PetscCall(DMPlexGetDepth(dm, &depth));
4266   if (spmap && depth >= 0) {
4267     DM_Plex  *mesh = (DM_Plex *)dm->data;
4268     PetscInt *points, *depths;
4269     PetscInt  pStart, pEnd, p, off;
4270 
4271     PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
4272     PetscCheck(!pStart, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Submeshes must start the point numbering at 0, not %" PetscInt_FMT, pStart);
4273     PetscCall(PetscMalloc1(pEnd, &points));
4274     PetscCall(DMGetWorkArray(dm, depth + 1, MPIU_INT, &depths));
4275     depths[0] = depth;
4276     depths[1] = 0;
4277     for (d = 2; d <= depth; ++d) depths[d] = depth + 1 - d;
4278     for (d = 0, off = 0; d <= depth; ++d) {
4279       const PetscInt dep = depths[d];
4280       PetscInt       depStart, depEnd, n;
4281 
4282       PetscCall(DMPlexGetDepthStratum(dm, dep, &depStart, &depEnd));
4283       PetscCall(DMLabelGetStratumSize(spmap, dep, &n));
4284       if (((d < 2) && (depth > 1)) || (d == 1)) { /* Only check vertices and cells for now since the map is broken for others */
4285         PetscCheck(n == depEnd - depStart, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "The number of mapped submesh points %" PetscInt_FMT " at depth %" PetscInt_FMT " should be %" PetscInt_FMT, n, dep, depEnd - depStart);
4286       } else {
4287         if (!n) {
4288           if (d == 0) {
4289             /* Missing cells */
4290             for (p = 0; p < depEnd - depStart; ++p, ++off) points[off] = -1;
4291           } else {
4292             /* Missing faces */
4293             for (p = 0; p < depEnd - depStart; ++p, ++off) points[off] = PETSC_MAX_INT;
4294           }
4295         }
4296       }
4297       if (n) {
4298         IS              is;
4299         const PetscInt *opoints;
4300 
4301         PetscCall(DMLabelGetStratumIS(spmap, dep, &is));
4302         PetscCall(ISGetIndices(is, &opoints));
4303         for (p = 0; p < n; ++p, ++off) points[off] = opoints[p];
4304         PetscCall(ISRestoreIndices(is, &opoints));
4305         PetscCall(ISDestroy(&is));
4306       }
4307     }
4308     PetscCall(DMRestoreWorkArray(dm, depth + 1, MPIU_INT, &depths));
4309     PetscCheck(off == pEnd, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "The number of mapped submesh points %" PetscInt_FMT " should be %" PetscInt_FMT, off, pEnd);
4310     PetscCall(ISCreateGeneral(PETSC_COMM_SELF, pEnd, points, PETSC_OWN_POINTER, subpointIS));
4311     PetscCall(PetscObjectStateGet((PetscObject)spmap, &mesh->subpointState));
4312   }
4313   PetscFunctionReturn(PETSC_SUCCESS);
4314 }
4315 
4316 /*@
4317   DMPlexGetSubpointIS - Returns an `IS` covering the entire subdm chart with the original points as data
4318 
4319   Input Parameter:
4320 . dm - The submesh `DM`
4321 
4322   Output Parameter:
4323 . subpointIS - The `IS` of all the points from the original mesh in this submesh, or `NULL` if this is not a submesh
4324 
4325   Level: developer
4326 
4327   Note:
4328   This `IS` is guaranteed to be sorted by the construction of the submesh
4329 
4330 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMPlexCreateSubmesh()`, `DMPlexGetSubpointMap()`
4331 @*/
4332 PetscErrorCode DMPlexGetSubpointIS(DM dm, IS *subpointIS)
4333 {
4334   DM_Plex         *mesh = (DM_Plex *)dm->data;
4335   DMLabel          spmap;
4336   PetscObjectState state;
4337 
4338   PetscFunctionBegin;
4339   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4340   PetscAssertPointer(subpointIS, 2);
4341   PetscCall(DMPlexGetSubpointMap(dm, &spmap));
4342   PetscCall(PetscObjectStateGet((PetscObject)spmap, &state));
4343   if (state != mesh->subpointState || !mesh->subpointIS) PetscCall(DMPlexCreateSubpointIS_Internal(dm, &mesh->subpointIS));
4344   *subpointIS = mesh->subpointIS;
4345   PetscFunctionReturn(PETSC_SUCCESS);
4346 }
4347 
4348 /*@
4349   DMGetEnclosureRelation - Get the relationship between `dmA` and `dmB`
4350 
4351   Input Parameters:
4352 + dmA - The first `DM`
4353 - dmB - The second `DM`
4354 
4355   Output Parameter:
4356 . rel - The relation of `dmA` to `dmB`
4357 
4358   Level: intermediate
4359 
4360 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMGetEnclosurePoint()`
4361 @*/
4362 PetscErrorCode DMGetEnclosureRelation(DM dmA, DM dmB, DMEnclosureType *rel)
4363 {
4364   DM       plexA, plexB, sdm;
4365   DMLabel  spmap;
4366   PetscInt pStartA, pEndA, pStartB, pEndB, NpA, NpB;
4367 
4368   PetscFunctionBegin;
4369   PetscAssertPointer(rel, 3);
4370   *rel = DM_ENC_NONE;
4371   if (!dmA || !dmB) PetscFunctionReturn(PETSC_SUCCESS);
4372   PetscValidHeaderSpecific(dmA, DM_CLASSID, 1);
4373   PetscValidHeaderSpecific(dmB, DM_CLASSID, 2);
4374   if (dmA == dmB) {
4375     *rel = DM_ENC_EQUALITY;
4376     PetscFunctionReturn(PETSC_SUCCESS);
4377   }
4378   PetscCall(DMConvert(dmA, DMPLEX, &plexA));
4379   PetscCall(DMConvert(dmB, DMPLEX, &plexB));
4380   PetscCall(DMPlexGetChart(plexA, &pStartA, &pEndA));
4381   PetscCall(DMPlexGetChart(plexB, &pStartB, &pEndB));
4382   /* Assumption 1: subDMs have smaller charts than the DMs that they originate from
4383     - The degenerate case of a subdomain which includes all of the domain on some process can be treated as equality */
4384   if ((pStartA == pStartB) && (pEndA == pEndB)) {
4385     *rel = DM_ENC_EQUALITY;
4386     goto end;
4387   }
4388   NpA = pEndA - pStartA;
4389   NpB = pEndB - pStartB;
4390   if (NpA == NpB) goto end;
4391   sdm = NpA > NpB ? plexB : plexA; /* The other is the original, enclosing dm */
4392   PetscCall(DMPlexGetSubpointMap(sdm, &spmap));
4393   if (!spmap) goto end;
4394   /* TODO Check the space mapped to by subpointMap is same size as dm */
4395   if (NpA > NpB) {
4396     *rel = DM_ENC_SUPERMESH;
4397   } else {
4398     *rel = DM_ENC_SUBMESH;
4399   }
4400 end:
4401   PetscCall(DMDestroy(&plexA));
4402   PetscCall(DMDestroy(&plexB));
4403   PetscFunctionReturn(PETSC_SUCCESS);
4404 }
4405 
4406 /*@
4407   DMGetEnclosurePoint - Get the point `pA` in `dmA` which corresponds to the point `pB` in `dmB`
4408 
4409   Input Parameters:
4410 + dmA   - The first `DM`
4411 . dmB   - The second `DM`
4412 . etype - The type of enclosure relation that `dmA` has to `dmB`
4413 - pB    - A point of `dmB`
4414 
4415   Output Parameter:
4416 . pA - The corresponding point of `dmA`
4417 
4418   Level: intermediate
4419 
4420 .seealso: [](ch_unstructured), `DM`, `DMPLEX`, `DMGetEnclosureRelation()`
4421 @*/
4422 PetscErrorCode DMGetEnclosurePoint(DM dmA, DM dmB, DMEnclosureType etype, PetscInt pB, PetscInt *pA)
4423 {
4424   DM              sdm;
4425   IS              subpointIS;
4426   const PetscInt *subpoints;
4427   PetscInt        numSubpoints;
4428 
4429   PetscFunctionBegin;
4430   /* TODO Cache the IS, making it look like an index */
4431   switch (etype) {
4432   case DM_ENC_SUPERMESH:
4433     sdm = dmB;
4434     PetscCall(DMPlexGetSubpointIS(sdm, &subpointIS));
4435     PetscCall(ISGetIndices(subpointIS, &subpoints));
4436     *pA = subpoints[pB];
4437     PetscCall(ISRestoreIndices(subpointIS, &subpoints));
4438     break;
4439   case DM_ENC_SUBMESH:
4440     sdm = dmA;
4441     PetscCall(DMPlexGetSubpointIS(sdm, &subpointIS));
4442     PetscCall(ISGetLocalSize(subpointIS, &numSubpoints));
4443     PetscCall(ISGetIndices(subpointIS, &subpoints));
4444     PetscCall(PetscFindInt(pB, numSubpoints, subpoints, pA));
4445     if (*pA < 0) {
4446       PetscCall(DMViewFromOptions(dmA, NULL, "-dm_enc_A_view"));
4447       PetscCall(DMViewFromOptions(dmB, NULL, "-dm_enc_B_view"));
4448       SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Point %" PetscInt_FMT " not found in submesh", pB);
4449     }
4450     PetscCall(ISRestoreIndices(subpointIS, &subpoints));
4451     break;
4452   case DM_ENC_EQUALITY:
4453   case DM_ENC_NONE:
4454     *pA = pB;
4455     break;
4456   case DM_ENC_UNKNOWN: {
4457     DMEnclosureType enc;
4458 
4459     PetscCall(DMGetEnclosureRelation(dmA, dmB, &enc));
4460     PetscCall(DMGetEnclosurePoint(dmA, dmB, enc, pB, pA));
4461   } break;
4462   default:
4463     SETERRQ(PetscObjectComm((PetscObject)dmA), PETSC_ERR_ARG_OUTOFRANGE, "Invalid enclosure type %d", (int)etype);
4464   }
4465   PetscFunctionReturn(PETSC_SUCCESS);
4466 }
4467