xref: /petsc/src/dm/impls/plex/plexsubmesh.c (revision 27f49a208b01d2e827ab9db411a2d16003fe9262)
1 #include <petsc/private/dmpleximpl.h>  /*I      "petscdmplex.h"    I*/
2 #include <petsc/private/dmlabelimpl.h> /*I      "petscdmlabel.h"   I*/
3 #include <petscsf.h>
4 
5 static PetscErrorCode DMPlexCellIsHybrid_Internal(DM dm, PetscInt p, PetscBool *isHybrid)
6 {
7   DMPolytopeType ct;
8 
9   PetscFunctionBegin;
10   PetscCall(DMPlexGetCellType(dm, p, &ct));
11   switch (ct) {
12   case DM_POLYTOPE_POINT_PRISM_TENSOR:
13   case DM_POLYTOPE_SEG_PRISM_TENSOR:
14   case DM_POLYTOPE_TRI_PRISM_TENSOR:
15   case DM_POLYTOPE_QUAD_PRISM_TENSOR:
16     *isHybrid = PETSC_TRUE;
17     break;
18   default:
19     *isHybrid = PETSC_FALSE;
20     break;
21   }
22   PetscFunctionReturn(PETSC_SUCCESS);
23 }
24 
25 static PetscErrorCode DMPlexGetTensorPrismBounds_Internal(DM dm, PetscInt dim, PetscInt *cStart, PetscInt *cEnd)
26 {
27   DMLabel ctLabel;
28 
29   PetscFunctionBegin;
30   if (cStart) *cStart = -1;
31   if (cEnd) *cEnd = -1;
32   PetscCall(DMPlexGetCellTypeLabel(dm, &ctLabel));
33   switch (dim) {
34   case 1:
35     PetscCall(DMLabelGetStratumBounds(ctLabel, DM_POLYTOPE_POINT_PRISM_TENSOR, cStart, cEnd));
36     break;
37   case 2:
38     PetscCall(DMLabelGetStratumBounds(ctLabel, DM_POLYTOPE_SEG_PRISM_TENSOR, cStart, cEnd));
39     break;
40   case 3:
41     PetscCall(DMLabelGetStratumBounds(ctLabel, DM_POLYTOPE_TRI_PRISM_TENSOR, cStart, cEnd));
42     if (*cStart < 0) PetscCall(DMLabelGetStratumBounds(ctLabel, DM_POLYTOPE_QUAD_PRISM_TENSOR, cStart, cEnd));
43     break;
44   default:
45     PetscFunctionReturn(PETSC_SUCCESS);
46   }
47   PetscFunctionReturn(PETSC_SUCCESS);
48 }
49 
50 static PetscErrorCode DMPlexMarkBoundaryFaces_Internal(DM dm, PetscInt val, PetscInt cellHeight, DMLabel label)
51 {
52   PetscSF         sf;
53   const PetscInt *rootdegree, *leaves;
54   PetscInt        overlap, Nr = -1, Nl, pStart, fStart, fEnd;
55 
56   PetscFunctionBegin;
57   PetscCall(DMGetPointSF(dm, &sf));
58   PetscCall(DMPlexGetOverlap(dm, &overlap));
59   if (sf && !overlap) PetscCall(PetscSFGetGraph(sf, &Nr, &Nl, &leaves, NULL));
60   if (Nr > 0) {
61     PetscCall(PetscSFComputeDegreeBegin(sf, &rootdegree));
62     PetscCall(PetscSFComputeDegreeEnd(sf, &rootdegree));
63   } else rootdegree = NULL;
64   PetscCall(DMPlexGetChart(dm, &pStart, NULL));
65   PetscCall(DMPlexGetHeightStratum(dm, cellHeight + 1, &fStart, &fEnd));
66   for (PetscInt f = fStart; f < fEnd; ++f) {
67     PetscInt supportSize, loc = -1;
68 
69     PetscCall(DMPlexGetSupportSize(dm, f, &supportSize));
70     if (supportSize == 1) {
71       /* Do not mark faces which are shared, meaning
72            they are  present in the pointSF, or
73            they have rootdegree > 0
74          since they presumably have cells on the other side */
75       if (Nr > 0) {
76         PetscCall(PetscFindInt(f, Nl, leaves, &loc));
77         if (rootdegree[f - pStart] || loc >= 0) continue;
78       }
79       if (val < 0) {
80         PetscInt *closure = NULL;
81         PetscInt  clSize, cl, cval;
82 
83         PetscCall(DMPlexGetTransitiveClosure(dm, f, PETSC_TRUE, &clSize, &closure));
84         for (cl = 0; cl < clSize * 2; cl += 2) {
85           PetscCall(DMLabelGetValue(label, closure[cl], &cval));
86           if (cval < 0) continue;
87           PetscCall(DMLabelSetValue(label, f, cval));
88           break;
89         }
90         if (cl == clSize * 2) PetscCall(DMLabelSetValue(label, f, 1));
91         PetscCall(DMPlexRestoreTransitiveClosure(dm, f, PETSC_TRUE, &clSize, &closure));
92       } else {
93         PetscCall(DMLabelSetValue(label, f, val));
94       }
95     }
96   }
97   PetscFunctionReturn(PETSC_SUCCESS);
98 }
99 
100 /*@
101   DMPlexMarkBoundaryFaces - Mark all faces on the boundary
102 
103   Not Collective
104 
105   Input Parameters:
106 + dm - The original `DM`
107 - val - The marker value, or `PETSC_DETERMINE` to use some value in the closure (or 1 if none are found)
108 
109   Output Parameter:
110 . label - The `DMLabel` marking boundary faces with the given value
111 
112   Level: developer
113 
114   Note:
115   This function will use the point `PetscSF` from the input `DM` to exclude points on the partition boundary from being marked, unless the partition overlap is greater than zero. If you also wish to mark the partition boundary, you can use `DMSetPointSF()` to temporarily set it to `NULL`, and then reset it to the original object after the call.
116 
117 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMLabelCreate()`, `DMCreateLabel()`
118 @*/
119 PetscErrorCode DMPlexMarkBoundaryFaces(DM dm, PetscInt val, DMLabel label)
120 {
121   DMPlexInterpolatedFlag flg;
122 
123   PetscFunctionBegin;
124   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
125   PetscCall(DMPlexIsInterpolated(dm, &flg));
126   PetscCheck(flg == DMPLEX_INTERPOLATED_FULL, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM is not fully interpolated on this rank");
127   PetscCall(DMPlexMarkBoundaryFaces_Internal(dm, val, 0, label));
128   PetscFunctionReturn(PETSC_SUCCESS);
129 }
130 
131 static PetscErrorCode DMPlexLabelComplete_Internal(DM dm, DMLabel label, PetscBool completeCells)
132 {
133   IS              valueIS;
134   PetscSF         sfPoint;
135   const PetscInt *values;
136   PetscInt        numValues, v, cStart, cEnd, nroots;
137 
138   PetscFunctionBegin;
139   PetscCall(DMLabelGetNumValues(label, &numValues));
140   PetscCall(DMLabelGetValueIS(label, &valueIS));
141   PetscCall(DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd));
142   PetscCall(ISGetIndices(valueIS, &values));
143   for (v = 0; v < numValues; ++v) {
144     IS              pointIS;
145     const PetscInt *points;
146     PetscInt        numPoints, p;
147 
148     PetscCall(DMLabelGetStratumSize(label, values[v], &numPoints));
149     PetscCall(DMLabelGetStratumIS(label, values[v], &pointIS));
150     PetscCall(ISGetIndices(pointIS, &points));
151     for (p = 0; p < numPoints; ++p) {
152       PetscInt  q       = points[p];
153       PetscInt *closure = NULL;
154       PetscInt  closureSize, c;
155 
156       if (cStart <= q && q < cEnd && !completeCells) { /* skip cells */
157         continue;
158       }
159       PetscCall(DMPlexGetTransitiveClosure(dm, q, PETSC_TRUE, &closureSize, &closure));
160       for (c = 0; c < closureSize * 2; c += 2) PetscCall(DMLabelSetValue(label, closure[c], values[v]));
161       PetscCall(DMPlexRestoreTransitiveClosure(dm, q, PETSC_TRUE, &closureSize, &closure));
162     }
163     PetscCall(ISRestoreIndices(pointIS, &points));
164     PetscCall(ISDestroy(&pointIS));
165   }
166   PetscCall(ISRestoreIndices(valueIS, &values));
167   PetscCall(ISDestroy(&valueIS));
168   PetscCall(DMGetPointSF(dm, &sfPoint));
169   PetscCall(PetscSFGetGraph(sfPoint, &nroots, NULL, NULL, NULL));
170   if (nroots >= 0) {
171     DMLabel         lblRoots, lblLeaves;
172     IS              valueIS, pointIS;
173     const PetscInt *values;
174     PetscInt        numValues, v;
175 
176     /* Pull point contributions from remote leaves into local roots */
177     PetscCall(DMLabelGather(label, sfPoint, &lblLeaves));
178     PetscCall(DMLabelGetValueIS(lblLeaves, &valueIS));
179     PetscCall(ISGetLocalSize(valueIS, &numValues));
180     PetscCall(ISGetIndices(valueIS, &values));
181     for (v = 0; v < numValues; ++v) {
182       const PetscInt value = values[v];
183 
184       PetscCall(DMLabelGetStratumIS(lblLeaves, value, &pointIS));
185       PetscCall(DMLabelInsertIS(label, pointIS, value));
186       PetscCall(ISDestroy(&pointIS));
187     }
188     PetscCall(ISRestoreIndices(valueIS, &values));
189     PetscCall(ISDestroy(&valueIS));
190     PetscCall(DMLabelDestroy(&lblLeaves));
191     /* Push point contributions from roots into remote leaves */
192     PetscCall(DMLabelDistribute(label, sfPoint, &lblRoots));
193     PetscCall(DMLabelGetValueIS(lblRoots, &valueIS));
194     PetscCall(ISGetLocalSize(valueIS, &numValues));
195     PetscCall(ISGetIndices(valueIS, &values));
196     for (v = 0; v < numValues; ++v) {
197       const PetscInt value = values[v];
198 
199       PetscCall(DMLabelGetStratumIS(lblRoots, value, &pointIS));
200       PetscCall(DMLabelInsertIS(label, pointIS, value));
201       PetscCall(ISDestroy(&pointIS));
202     }
203     PetscCall(ISRestoreIndices(valueIS, &values));
204     PetscCall(ISDestroy(&valueIS));
205     PetscCall(DMLabelDestroy(&lblRoots));
206   }
207   PetscFunctionReturn(PETSC_SUCCESS);
208 }
209 
210 /*@
211   DMPlexLabelComplete - Starting with a label marking points on a surface, we add the transitive closure to the surface
212 
213   Input Parameters:
214 + dm - The `DM`
215 - label - A `DMLabel` marking the surface points
216 
217   Output Parameter:
218 . label - A `DMLabel` marking all surface points in the transitive closure
219 
220   Level: developer
221 
222 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexLabelCohesiveComplete()`
223 @*/
224 PetscErrorCode DMPlexLabelComplete(DM dm, DMLabel label)
225 {
226   PetscFunctionBegin;
227   PetscCall(DMPlexLabelComplete_Internal(dm, label, PETSC_TRUE));
228   PetscFunctionReturn(PETSC_SUCCESS);
229 }
230 
231 /*@
232   DMPlexLabelAddCells - Starting with a label marking points on a surface, we add a cell for each point
233 
234   Input Parameters:
235 + dm - The `DM`
236 - label - A `DMLabel` marking the surface points
237 
238   Output Parameter:
239 . label - A `DMLabel` incorporating cells
240 
241   Level: developer
242 
243   Note:
244   The cells allow FEM boundary conditions to be applied using the cell geometry
245 
246 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexLabelAddFaceCells()`, `DMPlexLabelComplete()`, `DMPlexLabelCohesiveComplete()`
247 @*/
248 PetscErrorCode DMPlexLabelAddCells(DM dm, DMLabel label)
249 {
250   IS              valueIS;
251   const PetscInt *values;
252   PetscInt        numValues, v, csStart, csEnd, chStart, chEnd;
253 
254   PetscFunctionBegin;
255   PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &csStart, &csEnd));
256   PetscCall(DMPlexGetHeightStratum(dm, 0, &chStart, &chEnd));
257   PetscCall(DMLabelGetNumValues(label, &numValues));
258   PetscCall(DMLabelGetValueIS(label, &valueIS));
259   PetscCall(ISGetIndices(valueIS, &values));
260   for (v = 0; v < numValues; ++v) {
261     IS              pointIS;
262     const PetscInt *points;
263     PetscInt        numPoints, p;
264 
265     PetscCall(DMLabelGetStratumSize(label, values[v], &numPoints));
266     PetscCall(DMLabelGetStratumIS(label, values[v], &pointIS));
267     PetscCall(ISGetIndices(pointIS, &points));
268     for (p = 0; p < numPoints; ++p) {
269       const PetscInt point   = points[p];
270       PetscInt      *closure = NULL;
271       PetscInt       closureSize, cl, h, pStart, pEnd, cStart, cEnd;
272 
273       // If the point is a hybrid, allow hybrid cells
274       PetscCall(DMPlexGetPointHeight(dm, point, &h));
275       PetscCall(DMPlexGetSimplexOrBoxCells(dm, h, &pStart, &pEnd));
276       if (point >= pStart && point < pEnd) {
277         cStart = csStart;
278         cEnd   = csEnd;
279       } else {
280         cStart = chStart;
281         cEnd   = chEnd;
282       }
283 
284       PetscCall(DMPlexGetTransitiveClosure(dm, points[p], PETSC_FALSE, &closureSize, &closure));
285       for (cl = closureSize - 1; cl > 0; --cl) {
286         const PetscInt cell = closure[cl * 2];
287         if ((cell >= cStart) && (cell < cEnd)) {
288           PetscCall(DMLabelSetValue(label, cell, values[v]));
289           break;
290         }
291       }
292       PetscCall(DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_FALSE, &closureSize, &closure));
293     }
294     PetscCall(ISRestoreIndices(pointIS, &points));
295     PetscCall(ISDestroy(&pointIS));
296   }
297   PetscCall(ISRestoreIndices(valueIS, &values));
298   PetscCall(ISDestroy(&valueIS));
299   PetscFunctionReturn(PETSC_SUCCESS);
300 }
301 
302 /*@
303   DMPlexLabelAddFaceCells - Starting with a label marking faces on a surface, we add a cell for each face
304 
305   Input Parameters:
306 + dm - The `DM`
307 - label - A `DMLabel` marking the surface points
308 
309   Output Parameter:
310 . label - A `DMLabel` incorporating cells
311 
312   Level: developer
313 
314   Note:
315   The cells allow FEM boundary conditions to be applied using the cell geometry
316 
317 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexLabelAddCells()`, `DMPlexLabelComplete()`, `DMPlexLabelCohesiveComplete()`
318 @*/
319 PetscErrorCode DMPlexLabelAddFaceCells(DM dm, DMLabel label)
320 {
321   IS              valueIS;
322   const PetscInt *values;
323   PetscInt        numValues, v, cStart, cEnd, fStart, fEnd;
324 
325   PetscFunctionBegin;
326   PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd));
327   PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
328   PetscCall(DMLabelGetNumValues(label, &numValues));
329   PetscCall(DMLabelGetValueIS(label, &valueIS));
330   PetscCall(ISGetIndices(valueIS, &values));
331   for (v = 0; v < numValues; ++v) {
332     IS              pointIS;
333     const PetscInt *points;
334     PetscInt        numPoints, p;
335 
336     PetscCall(DMLabelGetStratumSize(label, values[v], &numPoints));
337     PetscCall(DMLabelGetStratumIS(label, values[v], &pointIS));
338     PetscCall(ISGetIndices(pointIS, &points));
339     for (p = 0; p < numPoints; ++p) {
340       const PetscInt face    = points[p];
341       PetscInt      *closure = NULL;
342       PetscInt       closureSize, cl;
343 
344       if ((face < fStart) || (face >= fEnd)) continue;
345       PetscCall(DMPlexGetTransitiveClosure(dm, face, PETSC_FALSE, &closureSize, &closure));
346       for (cl = closureSize - 1; cl > 0; --cl) {
347         const PetscInt cell = closure[cl * 2];
348         if ((cell >= cStart) && (cell < cEnd)) {
349           PetscCall(DMLabelSetValue(label, cell, values[v]));
350           break;
351         }
352       }
353       PetscCall(DMPlexRestoreTransitiveClosure(dm, face, PETSC_FALSE, &closureSize, &closure));
354     }
355     PetscCall(ISRestoreIndices(pointIS, &points));
356     PetscCall(ISDestroy(&pointIS));
357   }
358   PetscCall(ISRestoreIndices(valueIS, &values));
359   PetscCall(ISDestroy(&valueIS));
360   PetscFunctionReturn(PETSC_SUCCESS);
361 }
362 
363 /*@
364   DMPlexLabelClearCells - Remove cells from a label
365 
366   Input Parameters:
367 + dm - The `DM`
368 - label - A `DMLabel` marking surface points and their adjacent cells
369 
370   Output Parameter:
371 . label - A `DMLabel` without cells
372 
373   Level: developer
374 
375   Note:
376   This undoes `DMPlexLabelAddCells()` or `DMPlexLabelAddFaceCells()`
377 
378 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexLabelComplete()`, `DMPlexLabelCohesiveComplete()`, `DMPlexLabelAddCells()`
379 @*/
380 PetscErrorCode DMPlexLabelClearCells(DM dm, DMLabel label)
381 {
382   IS              valueIS;
383   const PetscInt *values;
384   PetscInt        numValues, v, cStart, cEnd;
385 
386   PetscFunctionBegin;
387   PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd));
388   PetscCall(DMLabelGetNumValues(label, &numValues));
389   PetscCall(DMLabelGetValueIS(label, &valueIS));
390   PetscCall(ISGetIndices(valueIS, &values));
391   for (v = 0; v < numValues; ++v) {
392     IS              pointIS;
393     const PetscInt *points;
394     PetscInt        numPoints, p;
395 
396     PetscCall(DMLabelGetStratumSize(label, values[v], &numPoints));
397     PetscCall(DMLabelGetStratumIS(label, values[v], &pointIS));
398     PetscCall(ISGetIndices(pointIS, &points));
399     for (p = 0; p < numPoints; ++p) {
400       PetscInt point = points[p];
401 
402       if (point >= cStart && point < cEnd) PetscCall(DMLabelClearValue(label, point, values[v]));
403     }
404     PetscCall(ISRestoreIndices(pointIS, &points));
405     PetscCall(ISDestroy(&pointIS));
406   }
407   PetscCall(ISRestoreIndices(valueIS, &values));
408   PetscCall(ISDestroy(&valueIS));
409   PetscFunctionReturn(PETSC_SUCCESS);
410 }
411 
412 /* take (oldEnd, added) pairs, ordered by height and convert them to (oldstart, newstart) pairs, ordered by ascending
413  * index (skipping first, which is (0,0)) */
414 static inline PetscErrorCode DMPlexShiftPointSetUp_Internal(PetscInt depth, PetscInt depthShift[])
415 {
416   PetscInt d, off = 0;
417 
418   PetscFunctionBegin;
419   /* sort by (oldend): yes this is an O(n^2) sort, we expect depth <= 3 */
420   for (d = 0; d < depth; d++) {
421     PetscInt firstd     = d;
422     PetscInt firstStart = depthShift[2 * d];
423     PetscInt e;
424 
425     for (e = d + 1; e <= depth; e++) {
426       if (depthShift[2 * e] < firstStart) {
427         firstd     = e;
428         firstStart = depthShift[2 * d];
429       }
430     }
431     if (firstd != d) {
432       PetscInt swap[2];
433 
434       e                     = firstd;
435       swap[0]               = depthShift[2 * d];
436       swap[1]               = depthShift[2 * d + 1];
437       depthShift[2 * d]     = depthShift[2 * e];
438       depthShift[2 * d + 1] = depthShift[2 * e + 1];
439       depthShift[2 * e]     = swap[0];
440       depthShift[2 * e + 1] = swap[1];
441     }
442   }
443   /* convert (oldstart, added) to (oldstart, newstart) */
444   for (d = 0; d <= depth; d++) {
445     off += depthShift[2 * d + 1];
446     depthShift[2 * d + 1] = depthShift[2 * d] + off;
447   }
448   PetscFunctionReturn(PETSC_SUCCESS);
449 }
450 
451 /* depthShift is a list of (old, new) pairs */
452 static inline PetscInt DMPlexShiftPoint_Internal(PetscInt p, PetscInt depth, PetscInt depthShift[])
453 {
454   PetscInt d;
455   PetscInt newOff = 0;
456 
457   for (d = 0; d <= depth; d++) {
458     if (p < depthShift[2 * d]) return p + newOff;
459     else newOff = depthShift[2 * d + 1] - depthShift[2 * d];
460   }
461   return p + newOff;
462 }
463 
464 /* depthShift is a list of (old, new) pairs */
465 static inline PetscInt DMPlexShiftPointInverse_Internal(PetscInt p, PetscInt depth, PetscInt depthShift[])
466 {
467   PetscInt d;
468   PetscInt newOff = 0;
469 
470   for (d = 0; d <= depth; d++) {
471     if (p < depthShift[2 * d + 1]) return p + newOff;
472     else newOff = depthShift[2 * d] - depthShift[2 * d + 1];
473   }
474   return p + newOff;
475 }
476 
477 static PetscErrorCode DMPlexShiftSizes_Internal(DM dm, PetscInt depthShift[], DM dmNew)
478 {
479   PetscInt depth = 0, d, pStart, pEnd, p;
480   DMLabel  depthLabel;
481 
482   PetscFunctionBegin;
483   PetscCall(DMPlexGetDepth(dm, &depth));
484   if (depth < 0) PetscFunctionReturn(PETSC_SUCCESS);
485   /* Step 1: Expand chart */
486   PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
487   pEnd = DMPlexShiftPoint_Internal(pEnd, depth, depthShift);
488   PetscCall(DMPlexSetChart(dmNew, pStart, pEnd));
489   PetscCall(DMCreateLabel(dmNew, "depth"));
490   PetscCall(DMPlexGetDepthLabel(dmNew, &depthLabel));
491   PetscCall(DMCreateLabel(dmNew, "celltype"));
492   /* Step 2: Set cone and support sizes */
493   for (d = 0; d <= depth; ++d) {
494     PetscInt pStartNew, pEndNew;
495     IS       pIS;
496 
497     PetscCall(DMPlexGetDepthStratum(dm, d, &pStart, &pEnd));
498     pStartNew = DMPlexShiftPoint_Internal(pStart, depth, depthShift);
499     pEndNew   = DMPlexShiftPoint_Internal(pEnd, depth, depthShift);
500     PetscCall(ISCreateStride(PETSC_COMM_SELF, pEndNew - pStartNew, pStartNew, 1, &pIS));
501     PetscCall(DMLabelSetStratumIS(depthLabel, d, pIS));
502     PetscCall(ISDestroy(&pIS));
503     for (p = pStart; p < pEnd; ++p) {
504       PetscInt       newp = DMPlexShiftPoint_Internal(p, depth, depthShift);
505       PetscInt       size;
506       DMPolytopeType ct;
507 
508       PetscCall(DMPlexGetConeSize(dm, p, &size));
509       PetscCall(DMPlexSetConeSize(dmNew, newp, size));
510       PetscCall(DMPlexGetSupportSize(dm, p, &size));
511       PetscCall(DMPlexSetSupportSize(dmNew, newp, size));
512       PetscCall(DMPlexGetCellType(dm, p, &ct));
513       PetscCall(DMPlexSetCellType(dmNew, newp, ct));
514     }
515   }
516   PetscFunctionReturn(PETSC_SUCCESS);
517 }
518 
519 static PetscErrorCode DMPlexShiftPoints_Internal(DM dm, PetscInt depthShift[], DM dmNew)
520 {
521   PetscInt *newpoints;
522   PetscInt  depth = 0, maxConeSize, maxSupportSize, maxConeSizeNew, maxSupportSizeNew, pStart, pEnd, p;
523 
524   PetscFunctionBegin;
525   PetscCall(DMPlexGetDepth(dm, &depth));
526   if (depth < 0) PetscFunctionReturn(PETSC_SUCCESS);
527   PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize));
528   PetscCall(DMPlexGetMaxSizes(dmNew, &maxConeSizeNew, &maxSupportSizeNew));
529   PetscCall(PetscMalloc1(PetscMax(PetscMax(maxConeSize, maxSupportSize), PetscMax(maxConeSizeNew, maxSupportSizeNew)), &newpoints));
530   /* Step 5: Set cones and supports */
531   PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
532   for (p = pStart; p < pEnd; ++p) {
533     const PetscInt *points = NULL, *orientations = NULL;
534     PetscInt        size, sizeNew, i, newp = DMPlexShiftPoint_Internal(p, depth, depthShift);
535 
536     PetscCall(DMPlexGetConeSize(dm, p, &size));
537     PetscCall(DMPlexGetCone(dm, p, &points));
538     PetscCall(DMPlexGetConeOrientation(dm, p, &orientations));
539     for (i = 0; i < size; ++i) newpoints[i] = DMPlexShiftPoint_Internal(points[i], depth, depthShift);
540     PetscCall(DMPlexSetCone(dmNew, newp, newpoints));
541     PetscCall(DMPlexSetConeOrientation(dmNew, newp, orientations));
542     PetscCall(DMPlexGetSupportSize(dm, p, &size));
543     PetscCall(DMPlexGetSupportSize(dmNew, newp, &sizeNew));
544     PetscCall(DMPlexGetSupport(dm, p, &points));
545     for (i = 0; i < size; ++i) newpoints[i] = DMPlexShiftPoint_Internal(points[i], depth, depthShift);
546     for (i = size; i < sizeNew; ++i) newpoints[i] = 0;
547     PetscCall(DMPlexSetSupport(dmNew, newp, newpoints));
548   }
549   PetscCall(PetscFree(newpoints));
550   PetscFunctionReturn(PETSC_SUCCESS);
551 }
552 
553 static PetscErrorCode DMPlexShiftCoordinates_Internal(DM dm, PetscInt depthShift[], DM dmNew)
554 {
555   PetscSection coordSection, newCoordSection;
556   Vec          coordinates, newCoordinates;
557   PetscScalar *coords, *newCoords;
558   PetscInt     coordSize, sStart, sEnd;
559   PetscInt     dim, depth = 0, cStart, cEnd, cStartNew, cEndNew, c, vStart, vEnd, vStartNew, vEndNew, v;
560   PetscBool    hasCells;
561 
562   PetscFunctionBegin;
563   PetscCall(DMGetCoordinateDim(dm, &dim));
564   PetscCall(DMSetCoordinateDim(dmNew, dim));
565   PetscCall(DMPlexGetDepth(dm, &depth));
566   /* Step 8: Convert coordinates */
567   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
568   PetscCall(DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd));
569   PetscCall(DMPlexGetDepthStratum(dmNew, 0, &vStartNew, &vEndNew));
570   PetscCall(DMPlexGetHeightStratum(dmNew, 0, &cStartNew, &cEndNew));
571   PetscCall(DMGetCoordinateSection(dm, &coordSection));
572   PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dm), &newCoordSection));
573   PetscCall(PetscSectionSetNumFields(newCoordSection, 1));
574   PetscCall(PetscSectionSetFieldComponents(newCoordSection, 0, dim));
575   PetscCall(PetscSectionGetChart(coordSection, &sStart, &sEnd));
576   hasCells = sStart == cStart ? PETSC_TRUE : PETSC_FALSE;
577   PetscCall(PetscSectionSetChart(newCoordSection, hasCells ? cStartNew : vStartNew, vEndNew));
578   if (hasCells) {
579     for (c = cStart; c < cEnd; ++c) {
580       PetscInt cNew = DMPlexShiftPoint_Internal(c, depth, depthShift), dof;
581 
582       PetscCall(PetscSectionGetDof(coordSection, c, &dof));
583       PetscCall(PetscSectionSetDof(newCoordSection, cNew, dof));
584       PetscCall(PetscSectionSetFieldDof(newCoordSection, cNew, 0, dof));
585     }
586   }
587   for (v = vStartNew; v < vEndNew; ++v) {
588     PetscCall(PetscSectionSetDof(newCoordSection, v, dim));
589     PetscCall(PetscSectionSetFieldDof(newCoordSection, v, 0, dim));
590   }
591   PetscCall(PetscSectionSetUp(newCoordSection));
592   PetscCall(DMSetCoordinateSection(dmNew, PETSC_DETERMINE, newCoordSection));
593   PetscCall(PetscSectionGetStorageSize(newCoordSection, &coordSize));
594   PetscCall(VecCreate(PETSC_COMM_SELF, &newCoordinates));
595   PetscCall(PetscObjectSetName((PetscObject)newCoordinates, "coordinates"));
596   PetscCall(VecSetSizes(newCoordinates, coordSize, PETSC_DETERMINE));
597   PetscCall(VecSetBlockSize(newCoordinates, dim));
598   PetscCall(VecSetType(newCoordinates, VECSTANDARD));
599   PetscCall(DMSetCoordinatesLocal(dmNew, newCoordinates));
600   PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
601   PetscCall(VecGetArray(coordinates, &coords));
602   PetscCall(VecGetArray(newCoordinates, &newCoords));
603   if (hasCells) {
604     for (c = cStart; c < cEnd; ++c) {
605       PetscInt cNew = DMPlexShiftPoint_Internal(c, depth, depthShift), dof, off, noff, d;
606 
607       PetscCall(PetscSectionGetDof(coordSection, c, &dof));
608       PetscCall(PetscSectionGetOffset(coordSection, c, &off));
609       PetscCall(PetscSectionGetOffset(newCoordSection, cNew, &noff));
610       for (d = 0; d < dof; ++d) newCoords[noff + d] = coords[off + d];
611     }
612   }
613   for (v = vStart; v < vEnd; ++v) {
614     PetscInt dof, off, noff, d;
615 
616     PetscCall(PetscSectionGetDof(coordSection, v, &dof));
617     PetscCall(PetscSectionGetOffset(coordSection, v, &off));
618     PetscCall(PetscSectionGetOffset(newCoordSection, DMPlexShiftPoint_Internal(v, depth, depthShift), &noff));
619     for (d = 0; d < dof; ++d) newCoords[noff + d] = coords[off + d];
620   }
621   PetscCall(VecRestoreArray(coordinates, &coords));
622   PetscCall(VecRestoreArray(newCoordinates, &newCoords));
623   PetscCall(VecDestroy(&newCoordinates));
624   PetscCall(PetscSectionDestroy(&newCoordSection));
625   PetscFunctionReturn(PETSC_SUCCESS);
626 }
627 
628 static PetscErrorCode DMPlexShiftSF_Single(DM dm, PetscInt depthShift[], PetscSF sf, PetscSF sfNew)
629 {
630   const PetscSFNode *remotePoints;
631   PetscSFNode       *gremotePoints;
632   const PetscInt    *localPoints;
633   PetscInt          *glocalPoints, *newLocation, *newRemoteLocation;
634   PetscInt           numRoots, numLeaves, l, pStart, pEnd, depth = 0, totShift = 0;
635 
636   PetscFunctionBegin;
637   PetscCall(DMPlexGetDepth(dm, &depth));
638   PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
639   PetscCall(PetscSFGetGraph(sf, &numRoots, &numLeaves, &localPoints, &remotePoints));
640   totShift = DMPlexShiftPoint_Internal(pEnd, depth, depthShift) - pEnd;
641   if (numRoots >= 0) {
642     PetscCall(PetscMalloc2(numRoots, &newLocation, pEnd - pStart, &newRemoteLocation));
643     for (l = 0; l < numRoots; ++l) newLocation[l] = DMPlexShiftPoint_Internal(l, depth, depthShift);
644     PetscCall(PetscSFBcastBegin(sf, MPIU_INT, newLocation, newRemoteLocation, MPI_REPLACE));
645     PetscCall(PetscSFBcastEnd(sf, MPIU_INT, newLocation, newRemoteLocation, MPI_REPLACE));
646     PetscCall(PetscMalloc1(numLeaves, &glocalPoints));
647     PetscCall(PetscMalloc1(numLeaves, &gremotePoints));
648     for (l = 0; l < numLeaves; ++l) {
649       glocalPoints[l]        = DMPlexShiftPoint_Internal(localPoints[l], depth, depthShift);
650       gremotePoints[l].rank  = remotePoints[l].rank;
651       gremotePoints[l].index = newRemoteLocation[localPoints[l]];
652     }
653     PetscCall(PetscFree2(newLocation, newRemoteLocation));
654     PetscCall(PetscSFSetGraph(sfNew, numRoots + totShift, numLeaves, glocalPoints, PETSC_OWN_POINTER, gremotePoints, PETSC_OWN_POINTER));
655   }
656   PetscFunctionReturn(PETSC_SUCCESS);
657 }
658 
659 static PetscErrorCode DMPlexShiftSF_Internal(DM dm, PetscInt depthShift[], DM dmNew)
660 {
661   PetscSF   sfPoint, sfPointNew;
662   PetscBool useNatural;
663 
664   PetscFunctionBegin;
665   /* Step 9: Convert pointSF */
666   PetscCall(DMGetPointSF(dm, &sfPoint));
667   PetscCall(DMGetPointSF(dmNew, &sfPointNew));
668   PetscCall(DMPlexShiftSF_Single(dm, depthShift, sfPoint, sfPointNew));
669   /* Step 9b: Convert naturalSF */
670   PetscCall(DMGetUseNatural(dm, &useNatural));
671   if (useNatural) {
672     PetscSF sfNat, sfNatNew;
673 
674     PetscCall(DMSetUseNatural(dmNew, useNatural));
675     PetscCall(DMGetNaturalSF(dm, &sfNat));
676     PetscCall(DMGetNaturalSF(dmNew, &sfNatNew));
677     PetscCall(DMPlexShiftSF_Single(dm, depthShift, sfNat, sfNatNew));
678   }
679   PetscFunctionReturn(PETSC_SUCCESS);
680 }
681 
682 static PetscErrorCode DMPlexShiftLabels_Internal(DM dm, PetscInt depthShift[], DM dmNew)
683 {
684   PetscInt depth = 0, numLabels, l;
685 
686   PetscFunctionBegin;
687   PetscCall(DMPlexGetDepth(dm, &depth));
688   /* Step 10: Convert labels */
689   PetscCall(DMGetNumLabels(dm, &numLabels));
690   for (l = 0; l < numLabels; ++l) {
691     DMLabel         label, newlabel;
692     const char     *lname;
693     PetscBool       isDepth, isDim;
694     IS              valueIS;
695     const PetscInt *values;
696     PetscInt        numValues, val;
697 
698     PetscCall(DMGetLabelName(dm, l, &lname));
699     PetscCall(PetscStrcmp(lname, "depth", &isDepth));
700     if (isDepth) continue;
701     PetscCall(PetscStrcmp(lname, "dim", &isDim));
702     if (isDim) continue;
703     PetscCall(DMCreateLabel(dmNew, lname));
704     PetscCall(DMGetLabel(dm, lname, &label));
705     PetscCall(DMGetLabel(dmNew, lname, &newlabel));
706     PetscCall(DMLabelGetDefaultValue(label, &val));
707     PetscCall(DMLabelSetDefaultValue(newlabel, val));
708     PetscCall(DMLabelGetValueIS(label, &valueIS));
709     PetscCall(ISGetLocalSize(valueIS, &numValues));
710     PetscCall(ISGetIndices(valueIS, &values));
711     for (val = 0; val < numValues; ++val) {
712       IS              pointIS;
713       const PetscInt *points;
714       PetscInt        numPoints, p;
715 
716       PetscCall(DMLabelGetStratumIS(label, values[val], &pointIS));
717       PetscCall(ISGetLocalSize(pointIS, &numPoints));
718       PetscCall(ISGetIndices(pointIS, &points));
719       for (p = 0; p < numPoints; ++p) {
720         const PetscInt newpoint = DMPlexShiftPoint_Internal(points[p], depth, depthShift);
721 
722         PetscCall(DMLabelSetValue(newlabel, newpoint, values[val]));
723       }
724       PetscCall(ISRestoreIndices(pointIS, &points));
725       PetscCall(ISDestroy(&pointIS));
726     }
727     PetscCall(ISRestoreIndices(valueIS, &values));
728     PetscCall(ISDestroy(&valueIS));
729   }
730   PetscFunctionReturn(PETSC_SUCCESS);
731 }
732 
733 static PetscErrorCode DMPlexCreateVTKLabel_Internal(DM dm, PetscBool createGhostLabel, DM dmNew)
734 {
735   PetscSF            sfPoint;
736   DMLabel            vtkLabel, ghostLabel = NULL;
737   const PetscSFNode *leafRemote;
738   const PetscInt    *leafLocal;
739   PetscInt           cellHeight, cStart, cEnd, c, fStart, fEnd, f, numLeaves, l;
740   PetscMPIInt        rank;
741 
742   PetscFunctionBegin;
743   /* Step 11: Make label for output (vtk) and to mark ghost points (ghost) */
744   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
745   PetscCall(DMGetPointSF(dm, &sfPoint));
746   PetscCall(DMPlexGetVTKCellHeight(dmNew, &cellHeight));
747   PetscCall(DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd));
748   PetscCall(PetscSFGetGraph(sfPoint, NULL, &numLeaves, &leafLocal, &leafRemote));
749   PetscCall(DMCreateLabel(dmNew, "vtk"));
750   PetscCall(DMGetLabel(dmNew, "vtk", &vtkLabel));
751   if (createGhostLabel) {
752     PetscCall(DMCreateLabel(dmNew, "ghost"));
753     PetscCall(DMGetLabel(dmNew, "ghost", &ghostLabel));
754   }
755   for (l = 0, c = cStart; l < numLeaves && c < cEnd; ++l, ++c) {
756     for (; c < leafLocal[l] && c < cEnd; ++c) PetscCall(DMLabelSetValue(vtkLabel, c, 1));
757     if (leafLocal[l] >= cEnd) break;
758     if (leafRemote[l].rank == rank) {
759       PetscCall(DMLabelSetValue(vtkLabel, c, 1));
760     } else if (ghostLabel) PetscCall(DMLabelSetValue(ghostLabel, c, 2));
761   }
762   for (; c < cEnd; ++c) PetscCall(DMLabelSetValue(vtkLabel, c, 1));
763   if (ghostLabel) {
764     PetscCall(DMPlexGetHeightStratum(dmNew, 1, &fStart, &fEnd));
765     for (f = fStart; f < fEnd; ++f) {
766       PetscInt numCells;
767 
768       PetscCall(DMPlexGetSupportSize(dmNew, f, &numCells));
769       if (numCells < 2) {
770         PetscCall(DMLabelSetValue(ghostLabel, f, 1));
771       } else {
772         const PetscInt *cells = NULL;
773         PetscInt        vA, vB;
774 
775         PetscCall(DMPlexGetSupport(dmNew, f, &cells));
776         PetscCall(DMLabelGetValue(vtkLabel, cells[0], &vA));
777         PetscCall(DMLabelGetValue(vtkLabel, cells[1], &vB));
778         if (vA != 1 && vB != 1) PetscCall(DMLabelSetValue(ghostLabel, f, 1));
779       }
780     }
781   }
782   PetscFunctionReturn(PETSC_SUCCESS);
783 }
784 
785 static PetscErrorCode DMPlexShiftTree_Internal(DM dm, PetscInt depthShift[], DM dmNew)
786 {
787   DM           refTree;
788   PetscSection pSec;
789   PetscInt    *parents, *childIDs;
790 
791   PetscFunctionBegin;
792   PetscCall(DMPlexGetReferenceTree(dm, &refTree));
793   PetscCall(DMPlexSetReferenceTree(dmNew, refTree));
794   PetscCall(DMPlexGetTree(dm, &pSec, &parents, &childIDs, NULL, NULL));
795   if (pSec) {
796     PetscInt     p, pStart, pEnd, *parentsShifted, pStartShifted, pEndShifted, depth;
797     PetscInt    *childIDsShifted;
798     PetscSection pSecShifted;
799 
800     PetscCall(PetscSectionGetChart(pSec, &pStart, &pEnd));
801     PetscCall(DMPlexGetDepth(dm, &depth));
802     pStartShifted = DMPlexShiftPoint_Internal(pStart, depth, depthShift);
803     pEndShifted   = DMPlexShiftPoint_Internal(pEnd, depth, depthShift);
804     PetscCall(PetscMalloc2(pEndShifted - pStartShifted, &parentsShifted, pEndShifted - pStartShifted, &childIDsShifted));
805     PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dmNew), &pSecShifted));
806     PetscCall(PetscSectionSetChart(pSecShifted, pStartShifted, pEndShifted));
807     for (p = pStartShifted; p < pEndShifted; p++) {
808       /* start off assuming no children */
809       PetscCall(PetscSectionSetDof(pSecShifted, p, 0));
810     }
811     for (p = pStart; p < pEnd; p++) {
812       PetscInt dof;
813       PetscInt pNew = DMPlexShiftPoint_Internal(p, depth, depthShift);
814 
815       PetscCall(PetscSectionGetDof(pSec, p, &dof));
816       PetscCall(PetscSectionSetDof(pSecShifted, pNew, dof));
817     }
818     PetscCall(PetscSectionSetUp(pSecShifted));
819     for (p = pStart; p < pEnd; p++) {
820       PetscInt dof;
821       PetscInt pNew = DMPlexShiftPoint_Internal(p, depth, depthShift);
822 
823       PetscCall(PetscSectionGetDof(pSec, p, &dof));
824       if (dof) {
825         PetscInt off, offNew;
826 
827         PetscCall(PetscSectionGetOffset(pSec, p, &off));
828         PetscCall(PetscSectionGetOffset(pSecShifted, pNew, &offNew));
829         parentsShifted[offNew]  = DMPlexShiftPoint_Internal(parents[off], depth, depthShift);
830         childIDsShifted[offNew] = childIDs[off];
831       }
832     }
833     PetscCall(DMPlexSetTree(dmNew, pSecShifted, parentsShifted, childIDsShifted));
834     PetscCall(PetscFree2(parentsShifted, childIDsShifted));
835     PetscCall(PetscSectionDestroy(&pSecShifted));
836   }
837   PetscFunctionReturn(PETSC_SUCCESS);
838 }
839 
840 static PetscErrorCode DMPlexConstructGhostCells_Internal(DM dm, DMLabel label, PetscInt *numGhostCells, DM gdm)
841 {
842   PetscSF          sf;
843   IS               valueIS;
844   const PetscInt  *values, *leaves;
845   PetscInt        *depthShift;
846   PetscInt         d, depth = 0, nleaves, loc, Ng, numFS, fs, fStart, fEnd, ghostCell, cEnd, c;
847   const PetscReal *maxCell, *Lstart, *L;
848 
849   PetscFunctionBegin;
850   PetscCall(DMGetPointSF(dm, &sf));
851   PetscCall(PetscSFGetGraph(sf, NULL, &nleaves, &leaves, NULL));
852   nleaves = PetscMax(0, nleaves);
853   PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
854   /* Count ghost cells */
855   PetscCall(DMLabelGetValueIS(label, &valueIS));
856   PetscCall(ISGetLocalSize(valueIS, &numFS));
857   PetscCall(ISGetIndices(valueIS, &values));
858   Ng = 0;
859   for (fs = 0; fs < numFS; ++fs) {
860     IS              faceIS;
861     const PetscInt *faces;
862     PetscInt        numFaces, f, numBdFaces = 0;
863 
864     PetscCall(DMLabelGetStratumIS(label, values[fs], &faceIS));
865     PetscCall(ISGetLocalSize(faceIS, &numFaces));
866     PetscCall(ISGetIndices(faceIS, &faces));
867     for (f = 0; f < numFaces; ++f) {
868       PetscInt numChildren;
869 
870       PetscCall(PetscFindInt(faces[f], nleaves, leaves, &loc));
871       PetscCall(DMPlexGetTreeChildren(dm, faces[f], &numChildren, NULL));
872       /* non-local and ancestors points don't get to register ghosts */
873       if (loc >= 0 || numChildren) continue;
874       if ((faces[f] >= fStart) && (faces[f] < fEnd)) ++numBdFaces;
875     }
876     Ng += numBdFaces;
877     PetscCall(ISRestoreIndices(faceIS, &faces));
878     PetscCall(ISDestroy(&faceIS));
879   }
880   PetscCall(DMPlexGetDepth(dm, &depth));
881   PetscCall(PetscMalloc1(2 * (depth + 1), &depthShift));
882   for (d = 0; d <= depth; d++) {
883     PetscInt dEnd;
884 
885     PetscCall(DMPlexGetDepthStratum(dm, d, NULL, &dEnd));
886     depthShift[2 * d]     = dEnd;
887     depthShift[2 * d + 1] = 0;
888   }
889   if (depth >= 0) depthShift[2 * depth + 1] = Ng;
890   PetscCall(DMPlexShiftPointSetUp_Internal(depth, depthShift));
891   PetscCall(DMPlexShiftSizes_Internal(dm, depthShift, gdm));
892   /* Step 3: Set cone/support sizes for new points */
893   PetscCall(DMPlexGetHeightStratum(dm, 0, NULL, &cEnd));
894   for (c = cEnd; c < cEnd + Ng; ++c) PetscCall(DMPlexSetConeSize(gdm, c, 1));
895   for (fs = 0; fs < numFS; ++fs) {
896     IS              faceIS;
897     const PetscInt *faces;
898     PetscInt        numFaces, f;
899 
900     PetscCall(DMLabelGetStratumIS(label, values[fs], &faceIS));
901     PetscCall(ISGetLocalSize(faceIS, &numFaces));
902     PetscCall(ISGetIndices(faceIS, &faces));
903     for (f = 0; f < numFaces; ++f) {
904       PetscInt size, numChildren;
905 
906       PetscCall(PetscFindInt(faces[f], nleaves, leaves, &loc));
907       PetscCall(DMPlexGetTreeChildren(dm, faces[f], &numChildren, NULL));
908       if (loc >= 0 || numChildren) continue;
909       if ((faces[f] < fStart) || (faces[f] >= fEnd)) continue;
910       PetscCall(DMPlexGetSupportSize(dm, faces[f], &size));
911       PetscCheck(size == 1, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM has boundary face %" PetscInt_FMT " with %" PetscInt_FMT " support cells", faces[f], size);
912       PetscCall(DMPlexSetSupportSize(gdm, faces[f] + Ng, 2));
913     }
914     PetscCall(ISRestoreIndices(faceIS, &faces));
915     PetscCall(ISDestroy(&faceIS));
916   }
917   /* Step 4: Setup ghosted DM */
918   PetscCall(DMSetUp(gdm));
919   PetscCall(DMPlexShiftPoints_Internal(dm, depthShift, gdm));
920   /* Step 6: Set cones and supports for new points */
921   ghostCell = cEnd;
922   for (fs = 0; fs < numFS; ++fs) {
923     IS              faceIS;
924     const PetscInt *faces;
925     PetscInt        numFaces, f;
926 
927     PetscCall(DMLabelGetStratumIS(label, values[fs], &faceIS));
928     PetscCall(ISGetLocalSize(faceIS, &numFaces));
929     PetscCall(ISGetIndices(faceIS, &faces));
930     for (f = 0; f < numFaces; ++f) {
931       PetscInt newFace = faces[f] + Ng, numChildren;
932 
933       PetscCall(PetscFindInt(faces[f], nleaves, leaves, &loc));
934       PetscCall(DMPlexGetTreeChildren(dm, faces[f], &numChildren, NULL));
935       if (loc >= 0 || numChildren) continue;
936       if ((faces[f] < fStart) || (faces[f] >= fEnd)) continue;
937       PetscCall(DMPlexSetCone(gdm, ghostCell, &newFace));
938       PetscCall(DMPlexInsertSupport(gdm, newFace, 1, ghostCell));
939       ++ghostCell;
940     }
941     PetscCall(ISRestoreIndices(faceIS, &faces));
942     PetscCall(ISDestroy(&faceIS));
943   }
944   PetscCall(ISRestoreIndices(valueIS, &values));
945   PetscCall(ISDestroy(&valueIS));
946   PetscCall(DMPlexShiftCoordinates_Internal(dm, depthShift, gdm));
947   PetscCall(DMPlexShiftSF_Internal(dm, depthShift, gdm));
948   PetscCall(DMPlexShiftLabels_Internal(dm, depthShift, gdm));
949   PetscCall(DMPlexCreateVTKLabel_Internal(dm, PETSC_TRUE, gdm));
950   PetscCall(DMPlexShiftTree_Internal(dm, depthShift, gdm));
951   PetscCall(PetscFree(depthShift));
952   for (c = cEnd; c < cEnd + Ng; ++c) PetscCall(DMPlexSetCellType(gdm, c, DM_POLYTOPE_FV_GHOST));
953   /* Step 7: Periodicity */
954   PetscCall(DMGetPeriodicity(dm, &maxCell, &Lstart, &L));
955   PetscCall(DMSetPeriodicity(gdm, maxCell, Lstart, L));
956   if (numGhostCells) *numGhostCells = Ng;
957   PetscFunctionReturn(PETSC_SUCCESS);
958 }
959 
960 /*@C
961   DMPlexConstructGhostCells - Construct ghost cells which connect to every boundary face
962 
963   Collective
964 
965   Input Parameters:
966 + dm - The original `DM`
967 - labelName - The label specifying the boundary faces, or "Face Sets" if this is `NULL`
968 
969   Output Parameters:
970 + numGhostCells - The number of ghost cells added to the `DM`
971 - dmGhosted - The new `DM`
972 
973   Level: developer
974 
975   Note:
976   If no label exists of that name, one will be created marking all boundary faces
977 
978 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMCreate()`
979 @*/
980 PetscErrorCode DMPlexConstructGhostCells(DM dm, const char labelName[], PetscInt *numGhostCells, DM *dmGhosted)
981 {
982   DM          gdm;
983   DMLabel     label;
984   const char *name = labelName ? labelName : "Face Sets";
985   PetscInt    dim, Ng = 0;
986   PetscBool   useCone, useClosure;
987 
988   PetscFunctionBegin;
989   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
990   if (numGhostCells) PetscValidIntPointer(numGhostCells, 3);
991   PetscValidPointer(dmGhosted, 4);
992   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &gdm));
993   PetscCall(DMSetType(gdm, DMPLEX));
994   PetscCall(DMGetDimension(dm, &dim));
995   PetscCall(DMSetDimension(gdm, dim));
996   PetscCall(DMGetBasicAdjacency(dm, &useCone, &useClosure));
997   PetscCall(DMSetBasicAdjacency(gdm, useCone, useClosure));
998   PetscCall(DMGetLabel(dm, name, &label));
999   if (!label) {
1000     /* Get label for boundary faces */
1001     PetscCall(DMCreateLabel(dm, name));
1002     PetscCall(DMGetLabel(dm, name, &label));
1003     PetscCall(DMPlexMarkBoundaryFaces(dm, 1, label));
1004   }
1005   PetscCall(DMPlexConstructGhostCells_Internal(dm, label, &Ng, gdm));
1006   PetscCall(DMCopyDisc(dm, gdm));
1007   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_TRUE, gdm));
1008   gdm->setfromoptionscalled = dm->setfromoptionscalled;
1009   if (numGhostCells) *numGhostCells = Ng;
1010   *dmGhosted = gdm;
1011   PetscFunctionReturn(PETSC_SUCCESS);
1012 }
1013 
1014 static PetscErrorCode DivideCells_Private(DM dm, DMLabel label, DMPlexPointQueue queue)
1015 {
1016   PetscInt dim, d, shift = 100, *pStart, *pEnd;
1017 
1018   PetscFunctionBegin;
1019   PetscCall(DMGetDimension(dm, &dim));
1020   PetscCall(PetscMalloc2(dim, &pStart, dim, &pEnd));
1021   for (d = 0; d < dim; ++d) PetscCall(DMPlexGetDepthStratum(dm, d, &pStart[d], &pEnd[d]));
1022   while (!DMPlexPointQueueEmpty(queue)) {
1023     PetscInt  cell    = -1;
1024     PetscInt *closure = NULL;
1025     PetscInt  closureSize, cl, cval;
1026 
1027     PetscCall(DMPlexPointQueueDequeue(queue, &cell));
1028     PetscCall(DMLabelGetValue(label, cell, &cval));
1029     PetscCall(DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
1030     /* Mark points in the cell closure that touch the fault */
1031     for (d = 0; d < dim; ++d) {
1032       for (cl = 0; cl < closureSize * 2; cl += 2) {
1033         const PetscInt clp = closure[cl];
1034         PetscInt       clval;
1035 
1036         if ((clp < pStart[d]) || (clp >= pEnd[d])) continue;
1037         PetscCall(DMLabelGetValue(label, clp, &clval));
1038         if (clval == -1) {
1039           const PetscInt *cone;
1040           PetscInt        coneSize, c;
1041 
1042           /* If a cone point touches the fault, then this point touches the fault */
1043           PetscCall(DMPlexGetCone(dm, clp, &cone));
1044           PetscCall(DMPlexGetConeSize(dm, clp, &coneSize));
1045           for (c = 0; c < coneSize; ++c) {
1046             PetscInt cpval;
1047 
1048             PetscCall(DMLabelGetValue(label, cone[c], &cpval));
1049             if (cpval != -1) {
1050               PetscInt dep;
1051 
1052               PetscCall(DMPlexGetPointDepth(dm, clp, &dep));
1053               clval = cval < 0 ? -(shift + dep) : shift + dep;
1054               PetscCall(DMLabelSetValue(label, clp, clval));
1055               break;
1056             }
1057           }
1058         }
1059         /* Mark neighbor cells through marked faces (these cells must also touch the fault) */
1060         if (d == dim - 1 && clval != -1) {
1061           const PetscInt *support;
1062           PetscInt        supportSize, s, nval;
1063 
1064           PetscCall(DMPlexGetSupport(dm, clp, &support));
1065           PetscCall(DMPlexGetSupportSize(dm, clp, &supportSize));
1066           for (s = 0; s < supportSize; ++s) {
1067             PetscCall(DMLabelGetValue(label, support[s], &nval));
1068             if (nval == -1) {
1069               PetscCall(DMLabelSetValue(label, support[s], clval < 0 ? clval - 1 : clval + 1));
1070               PetscCall(DMPlexPointQueueEnqueue(queue, support[s]));
1071             }
1072           }
1073         }
1074       }
1075     }
1076     PetscCall(DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
1077   }
1078   PetscCall(PetscFree2(pStart, pEnd));
1079   PetscFunctionReturn(PETSC_SUCCESS);
1080 }
1081 
1082 typedef struct {
1083   DM               dm;
1084   DMPlexPointQueue queue;
1085 } PointDivision;
1086 
1087 static PetscErrorCode divideCell(DMLabel label, PetscInt p, PetscInt val, void *ctx)
1088 {
1089   PointDivision  *div  = (PointDivision *)ctx;
1090   PetscInt        cval = val < 0 ? val - 1 : val + 1;
1091   const PetscInt *support;
1092   PetscInt        supportSize, s;
1093 
1094   PetscFunctionBegin;
1095   PetscCall(DMPlexGetSupport(div->dm, p, &support));
1096   PetscCall(DMPlexGetSupportSize(div->dm, p, &supportSize));
1097   for (s = 0; s < supportSize; ++s) {
1098     PetscCall(DMLabelSetValue(label, support[s], cval));
1099     PetscCall(DMPlexPointQueueEnqueue(div->queue, support[s]));
1100   }
1101   PetscFunctionReturn(PETSC_SUCCESS);
1102 }
1103 
1104 /* Mark cells by label propagation */
1105 static PetscErrorCode DMPlexLabelFaultHalo(DM dm, DMLabel faultLabel)
1106 {
1107   DMPlexPointQueue queue = NULL;
1108   PointDivision    div;
1109   PetscSF          pointSF;
1110   IS               pointIS;
1111   const PetscInt  *points;
1112   PetscBool        empty;
1113   PetscInt         dim, shift = 100, n, i;
1114 
1115   PetscFunctionBegin;
1116   PetscCall(DMGetDimension(dm, &dim));
1117   PetscCall(DMPlexPointQueueCreate(1024, &queue));
1118   div.dm    = dm;
1119   div.queue = queue;
1120   /* Enqueue cells on fault */
1121   PetscCall(DMLabelGetStratumIS(faultLabel, shift + dim, &pointIS));
1122   if (pointIS) {
1123     PetscCall(ISGetLocalSize(pointIS, &n));
1124     PetscCall(ISGetIndices(pointIS, &points));
1125     for (i = 0; i < n; ++i) PetscCall(DMPlexPointQueueEnqueue(queue, points[i]));
1126     PetscCall(ISRestoreIndices(pointIS, &points));
1127     PetscCall(ISDestroy(&pointIS));
1128   }
1129   PetscCall(DMLabelGetStratumIS(faultLabel, -(shift + dim), &pointIS));
1130   if (pointIS) {
1131     PetscCall(ISGetLocalSize(pointIS, &n));
1132     PetscCall(ISGetIndices(pointIS, &points));
1133     for (i = 0; i < n; ++i) PetscCall(DMPlexPointQueueEnqueue(queue, points[i]));
1134     PetscCall(ISRestoreIndices(pointIS, &points));
1135     PetscCall(ISDestroy(&pointIS));
1136   }
1137 
1138   PetscCall(DMGetPointSF(dm, &pointSF));
1139   PetscCall(DMLabelPropagateBegin(faultLabel, pointSF));
1140   /* While edge queue is not empty: */
1141   PetscCall(DMPlexPointQueueEmptyCollective((PetscObject)dm, queue, &empty));
1142   while (!empty) {
1143     PetscCall(DivideCells_Private(dm, faultLabel, queue));
1144     PetscCall(DMLabelPropagatePush(faultLabel, pointSF, divideCell, &div));
1145     PetscCall(DMPlexPointQueueEmptyCollective((PetscObject)dm, queue, &empty));
1146   }
1147   PetscCall(DMLabelPropagateEnd(faultLabel, pointSF));
1148   PetscCall(DMPlexPointQueueDestroy(&queue));
1149   PetscFunctionReturn(PETSC_SUCCESS);
1150 }
1151 
1152 /*
1153   We are adding three kinds of points here:
1154     Replicated:     Copies of points which exist in the mesh, such as vertices identified across a fault
1155     Non-replicated: Points which exist on the fault, but are not replicated
1156     Ghost:          These are shared fault faces which are not owned by this process. These do not produce hybrid cells and do not replicate
1157     Hybrid:         Entirely new points, such as cohesive cells
1158 
1159   When creating subsequent cohesive cells, we shift the old hybrid cells to the end of the numbering at
1160   each depth so that the new split/hybrid points can be inserted as a block.
1161 */
1162 static PetscErrorCode DMPlexConstructCohesiveCells_Internal(DM dm, DMLabel label, DMLabel splitLabel, DM sdm)
1163 {
1164   MPI_Comm         comm;
1165   IS               valueIS;
1166   PetscInt         numSP = 0; /* The number of depths for which we have replicated points */
1167   const PetscInt  *values;    /* List of depths for which we have replicated points */
1168   IS              *splitIS;
1169   IS              *unsplitIS;
1170   IS               ghostIS;
1171   PetscInt        *numSplitPoints;     /* The number of replicated points at each depth */
1172   PetscInt        *numUnsplitPoints;   /* The number of non-replicated points at each depth which still give rise to hybrid points */
1173   PetscInt        *numHybridPoints;    /* The number of new hybrid points at each depth */
1174   PetscInt        *numHybridPointsOld; /* The number of existing hybrid points at each depth */
1175   PetscInt         numGhostPoints;     /* The number of unowned, shared fault faces */
1176   const PetscInt **splitPoints;        /* Replicated points for each depth */
1177   const PetscInt **unsplitPoints;      /* Non-replicated points for each depth */
1178   const PetscInt  *ghostPoints;        /* Ghost fault faces */
1179   PetscSection     coordSection;
1180   Vec              coordinates;
1181   PetscScalar     *coords;
1182   PetscInt        *depthMax;   /* The first hybrid point at each depth in the original mesh */
1183   PetscInt        *depthEnd;   /* The point limit at each depth in the original mesh */
1184   PetscInt        *depthShift; /* Number of replicated+hybrid points at each depth */
1185   PetscInt        *pMaxNew;    /* The first replicated point at each depth in the new mesh, hybrids come after this */
1186   PetscInt        *coneNew, *coneONew, *supportNew;
1187   PetscInt         shift = 100, shift2 = 200, depth = 0, dep, dim, d, sp, maxConeSize, maxSupportSize, maxConeSizeNew, maxSupportSizeNew, numLabels, vStart, vEnd, pEnd, p, v;
1188 
1189   PetscFunctionBegin;
1190   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
1191   PetscCall(DMGetDimension(dm, &dim));
1192   PetscCall(DMPlexGetDepth(dm, &depth));
1193   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
1194   /* We do not want this label automatically computed, instead we compute it here */
1195   PetscCall(DMCreateLabel(sdm, "celltype"));
1196   /* Count split points and add cohesive cells */
1197   PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize));
1198   PetscCall(PetscMalloc5(depth + 1, &depthMax, depth + 1, &depthEnd, 2 * (depth + 1), &depthShift, depth + 1, &pMaxNew, depth + 1, &numHybridPointsOld));
1199   PetscCall(PetscMalloc7(depth + 1, &splitIS, depth + 1, &unsplitIS, depth + 1, &numSplitPoints, depth + 1, &numUnsplitPoints, depth + 1, &numHybridPoints, depth + 1, &splitPoints, depth + 1, &unsplitPoints));
1200   for (d = 0; d <= depth; ++d) {
1201     PetscCall(DMPlexGetDepthStratum(dm, d, NULL, &pMaxNew[d]));
1202     PetscCall(DMPlexGetTensorPrismBounds_Internal(dm, d, &depthMax[d], NULL));
1203     depthEnd[d]           = pMaxNew[d];
1204     depthMax[d]           = depthMax[d] < 0 ? depthEnd[d] : depthMax[d];
1205     numSplitPoints[d]     = 0;
1206     numUnsplitPoints[d]   = 0;
1207     numHybridPoints[d]    = 0;
1208     numHybridPointsOld[d] = depthMax[d] < 0 ? 0 : depthEnd[d] - depthMax[d];
1209     splitPoints[d]        = NULL;
1210     unsplitPoints[d]      = NULL;
1211     splitIS[d]            = NULL;
1212     unsplitIS[d]          = NULL;
1213     /* we are shifting the existing hybrid points with the stratum behind them, so
1214      * the split comes at the end of the normal points, i.e., at depthMax[d] */
1215     depthShift[2 * d]     = depthMax[d];
1216     depthShift[2 * d + 1] = 0;
1217   }
1218   numGhostPoints = 0;
1219   ghostPoints    = NULL;
1220   if (label) {
1221     PetscCall(DMLabelGetValueIS(label, &valueIS));
1222     PetscCall(ISGetLocalSize(valueIS, &numSP));
1223     PetscCall(ISGetIndices(valueIS, &values));
1224   }
1225   for (sp = 0; sp < numSP; ++sp) {
1226     const PetscInt dep = values[sp];
1227 
1228     if ((dep < 0) || (dep > depth)) continue;
1229     PetscCall(DMLabelGetStratumIS(label, dep, &splitIS[dep]));
1230     if (splitIS[dep]) {
1231       PetscCall(ISGetLocalSize(splitIS[dep], &numSplitPoints[dep]));
1232       PetscCall(ISGetIndices(splitIS[dep], &splitPoints[dep]));
1233     }
1234     PetscCall(DMLabelGetStratumIS(label, shift2 + dep, &unsplitIS[dep]));
1235     if (unsplitIS[dep]) {
1236       PetscCall(ISGetLocalSize(unsplitIS[dep], &numUnsplitPoints[dep]));
1237       PetscCall(ISGetIndices(unsplitIS[dep], &unsplitPoints[dep]));
1238     }
1239   }
1240   PetscCall(DMLabelGetStratumIS(label, shift2 + dim - 1, &ghostIS));
1241   if (ghostIS) {
1242     PetscCall(ISGetLocalSize(ghostIS, &numGhostPoints));
1243     PetscCall(ISGetIndices(ghostIS, &ghostPoints));
1244   }
1245   /* Calculate number of hybrid points */
1246   for (d = 1; d <= depth; ++d) numHybridPoints[d] = numSplitPoints[d - 1] + numUnsplitPoints[d - 1]; /* There is a hybrid cell/face/edge for every split face/edge/vertex   */
1247   for (d = 0; d <= depth; ++d) depthShift[2 * d + 1] = numSplitPoints[d] + numHybridPoints[d];
1248   PetscCall(DMPlexShiftPointSetUp_Internal(depth, depthShift));
1249   /* the end of the points in this stratum that come before the new points:
1250    * shifting pMaxNew[d] gets the new start of the next stratum, then count back the old hybrid points and the newly
1251    * added points */
1252   for (d = 0; d <= depth; ++d) pMaxNew[d] = DMPlexShiftPoint_Internal(pMaxNew[d], depth, depthShift) - (numHybridPointsOld[d] + numSplitPoints[d] + numHybridPoints[d]);
1253   PetscCall(DMPlexShiftSizes_Internal(dm, depthShift, sdm));
1254   /* Step 3: Set cone/support sizes for new points */
1255   for (dep = 0; dep <= depth; ++dep) {
1256     for (p = 0; p < numSplitPoints[dep]; ++p) {
1257       const PetscInt  oldp   = splitPoints[dep][p];
1258       const PetscInt  newp   = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*oldp + depthOffset[dep]*/;
1259       const PetscInt  splitp = p + pMaxNew[dep];
1260       const PetscInt *support;
1261       DMPolytopeType  ct;
1262       PetscInt        coneSize, supportSize, qf, qn, qp, e;
1263 
1264       PetscCall(DMPlexGetConeSize(dm, oldp, &coneSize));
1265       PetscCall(DMPlexSetConeSize(sdm, splitp, coneSize));
1266       PetscCall(DMPlexGetSupportSize(dm, oldp, &supportSize));
1267       PetscCall(DMPlexSetSupportSize(sdm, splitp, supportSize));
1268       PetscCall(DMPlexGetCellType(dm, oldp, &ct));
1269       PetscCall(DMPlexSetCellType(sdm, splitp, ct));
1270       if (dep == depth - 1) {
1271         const PetscInt hybcell = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1];
1272 
1273         /* Add cohesive cells, they are prisms */
1274         PetscCall(DMPlexSetConeSize(sdm, hybcell, 2 + coneSize));
1275         switch (coneSize) {
1276         case 2:
1277           PetscCall(DMPlexSetCellType(sdm, hybcell, DM_POLYTOPE_SEG_PRISM_TENSOR));
1278           break;
1279         case 3:
1280           PetscCall(DMPlexSetCellType(sdm, hybcell, DM_POLYTOPE_TRI_PRISM_TENSOR));
1281           break;
1282         case 4:
1283           PetscCall(DMPlexSetCellType(sdm, hybcell, DM_POLYTOPE_QUAD_PRISM_TENSOR));
1284           break;
1285         }
1286         /* Shared fault faces with only one support cell now have two with the cohesive cell */
1287         /*   TODO Check thaat oldp has rootdegree == 1 */
1288         if (supportSize == 1) {
1289           const PetscInt *support;
1290           PetscInt        val;
1291 
1292           PetscCall(DMPlexGetSupport(dm, oldp, &support));
1293           PetscCall(DMLabelGetValue(label, support[0], &val));
1294           if (val < 0) PetscCall(DMPlexSetSupportSize(sdm, splitp, 2));
1295           else PetscCall(DMPlexSetSupportSize(sdm, newp, 2));
1296         }
1297       } else if (dep == 0) {
1298         const PetscInt hybedge = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1];
1299 
1300         PetscCall(DMPlexGetSupport(dm, oldp, &support));
1301         for (e = 0, qn = 0, qp = 0, qf = 0; e < supportSize; ++e) {
1302           PetscInt val;
1303 
1304           PetscCall(DMLabelGetValue(label, support[e], &val));
1305           if (val == 1) ++qf;
1306           if ((val == 1) || (val == (shift + 1))) ++qn;
1307           if ((val == 1) || (val == -(shift + 1))) ++qp;
1308         }
1309         /* Split old vertex: Edges into original vertex and new cohesive edge */
1310         PetscCall(DMPlexSetSupportSize(sdm, newp, qn + 1));
1311         /* Split new vertex: Edges into split vertex and new cohesive edge */
1312         PetscCall(DMPlexSetSupportSize(sdm, splitp, qp + 1));
1313         /* Add hybrid edge */
1314         PetscCall(DMPlexSetConeSize(sdm, hybedge, 2));
1315         PetscCall(DMPlexSetSupportSize(sdm, hybedge, qf));
1316         PetscCall(DMPlexSetCellType(sdm, hybedge, DM_POLYTOPE_POINT_PRISM_TENSOR));
1317       } else if (dep == dim - 2) {
1318         const PetscInt hybface = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1];
1319 
1320         PetscCall(DMPlexGetSupport(dm, oldp, &support));
1321         for (e = 0, qn = 0, qp = 0, qf = 0; e < supportSize; ++e) {
1322           PetscInt val;
1323 
1324           PetscCall(DMLabelGetValue(label, support[e], &val));
1325           if (val == dim - 1) ++qf;
1326           if ((val == dim - 1) || (val == (shift + dim - 1))) ++qn;
1327           if ((val == dim - 1) || (val == -(shift + dim - 1))) ++qp;
1328         }
1329         /* Split old edge: Faces into original edge and cohesive face (positive side?) */
1330         PetscCall(DMPlexSetSupportSize(sdm, newp, qn + 1));
1331         /* Split new edge: Faces into split edge and cohesive face (negative side?) */
1332         PetscCall(DMPlexSetSupportSize(sdm, splitp, qp + 1));
1333         /* Add hybrid face */
1334         PetscCall(DMPlexSetConeSize(sdm, hybface, 4));
1335         PetscCall(DMPlexSetSupportSize(sdm, hybface, qf));
1336         PetscCall(DMPlexSetCellType(sdm, hybface, DM_POLYTOPE_SEG_PRISM_TENSOR));
1337       }
1338     }
1339   }
1340   for (dep = 0; dep <= depth; ++dep) {
1341     for (p = 0; p < numUnsplitPoints[dep]; ++p) {
1342       const PetscInt  oldp = unsplitPoints[dep][p];
1343       const PetscInt  newp = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*oldp + depthOffset[dep]*/;
1344       const PetscInt *support;
1345       PetscInt        coneSize, supportSize, qf, e, s;
1346 
1347       PetscCall(DMPlexGetConeSize(dm, oldp, &coneSize));
1348       PetscCall(DMPlexGetSupportSize(dm, oldp, &supportSize));
1349       PetscCall(DMPlexGetSupport(dm, oldp, &support));
1350       if (dep == 0) {
1351         const PetscInt hybedge = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1] + numSplitPoints[dep];
1352 
1353         /* Unsplit vertex: Edges into original vertex, split edges, and new cohesive edge twice */
1354         for (s = 0, qf = 0; s < supportSize; ++s, ++qf) {
1355           PetscCall(PetscFindInt(support[s], numSplitPoints[dep + 1], splitPoints[dep + 1], &e));
1356           if (e >= 0) ++qf;
1357         }
1358         PetscCall(DMPlexSetSupportSize(sdm, newp, qf + 2));
1359         /* Add hybrid edge */
1360         PetscCall(DMPlexSetConeSize(sdm, hybedge, 2));
1361         for (e = 0, qf = 0; e < supportSize; ++e) {
1362           PetscInt val;
1363 
1364           PetscCall(DMLabelGetValue(label, support[e], &val));
1365           /* Split and unsplit edges produce hybrid faces */
1366           if (val == 1) ++qf;
1367           if (val == (shift2 + 1)) ++qf;
1368         }
1369         PetscCall(DMPlexSetSupportSize(sdm, hybedge, qf));
1370         PetscCall(DMPlexSetCellType(sdm, hybedge, DM_POLYTOPE_POINT_PRISM_TENSOR));
1371       } else if (dep == dim - 2) {
1372         const PetscInt hybface = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1] + numSplitPoints[dep];
1373         PetscInt       val;
1374 
1375         for (e = 0, qf = 0; e < supportSize; ++e) {
1376           PetscCall(DMLabelGetValue(label, support[e], &val));
1377           if (val == dim - 1) qf += 2;
1378           else ++qf;
1379         }
1380         /* Unsplit edge: Faces into original edge, split face, and cohesive face twice */
1381         PetscCall(DMPlexSetSupportSize(sdm, newp, qf + 2));
1382         /* Add hybrid face */
1383         for (e = 0, qf = 0; e < supportSize; ++e) {
1384           PetscCall(DMLabelGetValue(label, support[e], &val));
1385           if (val == dim - 1) ++qf;
1386         }
1387         PetscCall(DMPlexSetConeSize(sdm, hybface, 4));
1388         PetscCall(DMPlexSetSupportSize(sdm, hybface, qf));
1389         PetscCall(DMPlexSetCellType(sdm, hybface, DM_POLYTOPE_SEG_PRISM_TENSOR));
1390       }
1391     }
1392   }
1393   /* Step 4: Setup split DM */
1394   PetscCall(DMSetUp(sdm));
1395   PetscCall(DMPlexShiftPoints_Internal(dm, depthShift, sdm));
1396   PetscCall(DMPlexGetMaxSizes(sdm, &maxConeSizeNew, &maxSupportSizeNew));
1397   PetscCall(PetscMalloc3(PetscMax(maxConeSize, maxConeSizeNew) * 3, &coneNew, PetscMax(maxConeSize, maxConeSizeNew) * 3, &coneONew, PetscMax(maxSupportSize, maxSupportSizeNew), &supportNew));
1398   /* Step 6: Set cones and supports for new points */
1399   for (dep = 0; dep <= depth; ++dep) {
1400     for (p = 0; p < numSplitPoints[dep]; ++p) {
1401       const PetscInt  oldp   = splitPoints[dep][p];
1402       const PetscInt  newp   = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*oldp + depthOffset[dep]*/;
1403       const PetscInt  splitp = p + pMaxNew[dep];
1404       const PetscInt *cone, *support, *ornt;
1405       DMPolytopeType  ct;
1406       PetscInt        coneSize, supportSize, q, qf, qn, qp, v, e, s;
1407 
1408       PetscCall(DMPlexGetCellType(dm, oldp, &ct));
1409       PetscCall(DMPlexGetConeSize(dm, oldp, &coneSize));
1410       PetscCall(DMPlexGetCone(dm, oldp, &cone));
1411       PetscCall(DMPlexGetConeOrientation(dm, oldp, &ornt));
1412       PetscCall(DMPlexGetSupportSize(dm, oldp, &supportSize));
1413       PetscCall(DMPlexGetSupport(dm, oldp, &support));
1414       if (dep == depth - 1) {
1415         PetscBool       hasUnsplit = PETSC_FALSE;
1416         const PetscInt  hybcell    = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1];
1417         const PetscInt *supportF;
1418 
1419         coneONew[0] = coneONew[1] = -1000;
1420         /* Split face:       copy in old face to new face to start */
1421         PetscCall(DMPlexGetSupport(sdm, newp, &supportF));
1422         PetscCall(DMPlexSetSupport(sdm, splitp, supportF));
1423         /* Split old face:   old vertices/edges in cone so no change */
1424         /* Split new face:   new vertices/edges in cone */
1425         for (q = 0; q < coneSize; ++q) {
1426           PetscCall(PetscFindInt(cone[q], numSplitPoints[dep - 1], splitPoints[dep - 1], &v));
1427           if (v < 0) {
1428             PetscCall(PetscFindInt(cone[q], numUnsplitPoints[dep - 1], unsplitPoints[dep - 1], &v));
1429             PetscCheck(v >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %" PetscInt_FMT " in split or unsplit points of depth %" PetscInt_FMT, cone[q], dep - 1);
1430             coneNew[2 + q] = DMPlexShiftPoint_Internal(cone[q], depth, depthShift) /*cone[q] + depthOffset[dep-1]*/;
1431             hasUnsplit     = PETSC_TRUE;
1432           } else {
1433             coneNew[2 + q] = v + pMaxNew[dep - 1];
1434             if (dep > 1) {
1435               const PetscInt *econe;
1436               PetscInt        econeSize, r, vs, vu;
1437 
1438               PetscCall(DMPlexGetConeSize(dm, cone[q], &econeSize));
1439               PetscCall(DMPlexGetCone(dm, cone[q], &econe));
1440               for (r = 0; r < econeSize; ++r) {
1441                 PetscCall(PetscFindInt(econe[r], numSplitPoints[dep - 2], splitPoints[dep - 2], &vs));
1442                 PetscCall(PetscFindInt(econe[r], numUnsplitPoints[dep - 2], unsplitPoints[dep - 2], &vu));
1443                 if (vs >= 0) continue;
1444                 PetscCheck(vu >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %" PetscInt_FMT " in split or unsplit points of depth %" PetscInt_FMT, econe[r], dep - 2);
1445                 hasUnsplit = PETSC_TRUE;
1446               }
1447             }
1448           }
1449         }
1450         PetscCall(DMPlexSetCone(sdm, splitp, &coneNew[2]));
1451         PetscCall(DMPlexSetConeOrientation(sdm, splitp, ornt));
1452         /* Face support */
1453         PetscInt vals[2];
1454 
1455         PetscCall(DMLabelGetValue(label, support[0], &vals[0]));
1456         if (supportSize > 1) PetscCall(DMLabelGetValue(label, support[1], &vals[1]));
1457         else vals[1] = -vals[0];
1458         PetscCheck(vals[0] * vals[1] < 0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid support labels %" PetscInt_FMT " %" PetscInt_FMT, vals[0], vals[1]);
1459 
1460         for (s = 0; s < 2; ++s) {
1461           if (s >= supportSize) {
1462             if (vals[s] < 0) {
1463               /* Ghost old face:   Replace negative side cell with cohesive cell */
1464               PetscCall(DMPlexInsertSupport(sdm, newp, s, hybcell));
1465             } else {
1466               /* Ghost new face:   Replace positive side cell with cohesive cell */
1467               PetscCall(DMPlexInsertSupport(sdm, splitp, s, hybcell));
1468             }
1469           } else {
1470             if (vals[s] < 0) {
1471               /* Split old face:   Replace negative side cell with cohesive cell */
1472               PetscCall(DMPlexInsertSupport(sdm, newp, s, hybcell));
1473             } else {
1474               /* Split new face:   Replace positive side cell with cohesive cell */
1475               PetscCall(DMPlexInsertSupport(sdm, splitp, s, hybcell));
1476             }
1477           }
1478         }
1479         /* Get orientation for cohesive face using the positive side cell */
1480         {
1481           const PetscInt *ncone, *nconeO;
1482           PetscInt        nconeSize, nc, ocell;
1483           PetscBool       flip = PETSC_FALSE;
1484 
1485           if (supportSize > 1) {
1486             ocell = vals[0] < 0 ? support[1] : support[0];
1487           } else {
1488             ocell = support[0];
1489             flip  = vals[0] < 0 ? PETSC_TRUE : PETSC_FALSE;
1490           }
1491           PetscCall(DMPlexGetConeSize(dm, ocell, &nconeSize));
1492           PetscCall(DMPlexGetCone(dm, ocell, &ncone));
1493           PetscCall(DMPlexGetConeOrientation(dm, ocell, &nconeO));
1494           for (nc = 0; nc < nconeSize; ++nc) {
1495             if (ncone[nc] == oldp) {
1496               coneONew[0] = flip ? -(nconeO[nc] + 1) : nconeO[nc];
1497               break;
1498             }
1499           }
1500           PetscCheck(nc < nconeSize, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate face %" PetscInt_FMT " in neighboring cell %" PetscInt_FMT, oldp, ocell);
1501         }
1502         /* Cohesive cell:    Old and new split face, then new cohesive faces */
1503         {
1504           const PetscInt No = DMPolytopeTypeGetNumArrangments(ct) / 2;
1505           PetscCheck((coneONew[0] >= -No) && (coneONew[0] < No), PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid %s orientation %" PetscInt_FMT, DMPolytopeTypes[ct], coneONew[0]);
1506         }
1507         const PetscInt *arr = DMPolytopeTypeGetArrangment(ct, coneONew[0]);
1508 
1509         coneNew[0]  = newp; /* Extracted negative side orientation above */
1510         coneNew[1]  = splitp;
1511         coneONew[1] = coneONew[0];
1512         for (q = 0; q < coneSize; ++q) {
1513           /* Hybrid faces must follow order from oriented end face */
1514           const PetscInt qa = arr[q * 2 + 0];
1515           const PetscInt qo = arr[q * 2 + 1];
1516           DMPolytopeType ft = dep == 2 ? DM_POLYTOPE_SEGMENT : DM_POLYTOPE_POINT;
1517 
1518           PetscCall(PetscFindInt(cone[qa], numSplitPoints[dep - 1], splitPoints[dep - 1], &v));
1519           if (v < 0) {
1520             PetscCall(PetscFindInt(cone[qa], numUnsplitPoints[dep - 1], unsplitPoints[dep - 1], &v));
1521             coneNew[2 + q] = v + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep - 1];
1522           } else {
1523             coneNew[2 + q] = v + pMaxNew[dep] + numSplitPoints[dep];
1524           }
1525           coneONew[2 + q] = DMPolytopeTypeComposeOrientation(ft, qo, ornt[qa]);
1526         }
1527         PetscCall(DMPlexSetCone(sdm, hybcell, coneNew));
1528         PetscCall(DMPlexSetConeOrientation(sdm, hybcell, coneONew));
1529         /* Label the hybrid cells on the boundary of the split */
1530         if (hasUnsplit) PetscCall(DMLabelSetValue(label, -hybcell, dim));
1531       } else if (dep == 0) {
1532         const PetscInt hybedge = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1];
1533 
1534         /* Split old vertex: Edges in old split faces and new cohesive edge */
1535         for (e = 0, qn = 0; e < supportSize; ++e) {
1536           PetscInt val;
1537 
1538           PetscCall(DMLabelGetValue(label, support[e], &val));
1539           if ((val == 1) || (val == (shift + 1))) supportNew[qn++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1540         }
1541         supportNew[qn] = hybedge;
1542         PetscCall(DMPlexSetSupport(sdm, newp, supportNew));
1543         /* Split new vertex: Edges in new split faces and new cohesive edge */
1544         for (e = 0, qp = 0; e < supportSize; ++e) {
1545           PetscInt val, edge;
1546 
1547           PetscCall(DMLabelGetValue(label, support[e], &val));
1548           if (val == 1) {
1549             PetscCall(PetscFindInt(support[e], numSplitPoints[dep + 1], splitPoints[dep + 1], &edge));
1550             PetscCheck(edge >= 0, comm, PETSC_ERR_ARG_WRONG, "Edge %" PetscInt_FMT " is not a split edge", support[e]);
1551             supportNew[qp++] = edge + pMaxNew[dep + 1];
1552           } else if (val == -(shift + 1)) {
1553             supportNew[qp++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1554           }
1555         }
1556         supportNew[qp] = hybedge;
1557         PetscCall(DMPlexSetSupport(sdm, splitp, supportNew));
1558         /* Hybrid edge:    Old and new split vertex */
1559         coneNew[0] = newp;
1560         coneNew[1] = splitp;
1561         PetscCall(DMPlexSetCone(sdm, hybedge, coneNew));
1562         for (e = 0, qf = 0; e < supportSize; ++e) {
1563           PetscInt val, edge;
1564 
1565           PetscCall(DMLabelGetValue(label, support[e], &val));
1566           if (val == 1) {
1567             PetscCall(PetscFindInt(support[e], numSplitPoints[dep + 1], splitPoints[dep + 1], &edge));
1568             PetscCheck(edge >= 0, comm, PETSC_ERR_ARG_WRONG, "Edge %" PetscInt_FMT " is not a split edge", support[e]);
1569             supportNew[qf++] = edge + pMaxNew[dep + 2] + numSplitPoints[dep + 2];
1570           }
1571         }
1572         PetscCall(DMPlexSetSupport(sdm, hybedge, supportNew));
1573       } else if (dep == dim - 2) {
1574         const PetscInt hybface = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1];
1575 
1576         /* Split old edge:   old vertices in cone so no change */
1577         /* Split new edge:   new vertices in cone */
1578         for (q = 0; q < coneSize; ++q) {
1579           PetscCall(PetscFindInt(cone[q], numSplitPoints[dep - 1], splitPoints[dep - 1], &v));
1580           if (v < 0) {
1581             PetscCall(PetscFindInt(cone[q], numUnsplitPoints[dep - 1], unsplitPoints[dep - 1], &v));
1582             PetscCheck(v >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %" PetscInt_FMT " in split or unsplit points of depth %" PetscInt_FMT, cone[q], dep - 1);
1583             coneNew[q] = DMPlexShiftPoint_Internal(cone[q], depth, depthShift) /*cone[q] + depthOffset[dep-1]*/;
1584           } else {
1585             coneNew[q] = v + pMaxNew[dep - 1];
1586           }
1587         }
1588         PetscCall(DMPlexSetCone(sdm, splitp, coneNew));
1589         /* Split old edge: Faces in positive side cells and old split faces */
1590         for (e = 0, q = 0; e < supportSize; ++e) {
1591           PetscInt val;
1592 
1593           PetscCall(DMLabelGetValue(label, support[e], &val));
1594           if (val == dim - 1) {
1595             supportNew[q++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1596           } else if (val == (shift + dim - 1)) {
1597             supportNew[q++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1598           }
1599         }
1600         supportNew[q++] = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1];
1601         PetscCall(DMPlexSetSupport(sdm, newp, supportNew));
1602         /* Split new edge: Faces in negative side cells and new split faces */
1603         for (e = 0, q = 0; e < supportSize; ++e) {
1604           PetscInt val, face;
1605 
1606           PetscCall(DMLabelGetValue(label, support[e], &val));
1607           if (val == dim - 1) {
1608             PetscCall(PetscFindInt(support[e], numSplitPoints[dep + 1], splitPoints[dep + 1], &face));
1609             PetscCheck(face >= 0, comm, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " is not a split face", support[e]);
1610             supportNew[q++] = face + pMaxNew[dep + 1];
1611           } else if (val == -(shift + dim - 1)) {
1612             supportNew[q++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1613           }
1614         }
1615         supportNew[q++] = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1];
1616         PetscCall(DMPlexSetSupport(sdm, splitp, supportNew));
1617         /* Hybrid face */
1618         coneNew[0] = newp;
1619         coneNew[1] = splitp;
1620         for (v = 0; v < coneSize; ++v) {
1621           PetscInt vertex;
1622           PetscCall(PetscFindInt(cone[v], numSplitPoints[dep - 1], splitPoints[dep - 1], &vertex));
1623           if (vertex < 0) {
1624             PetscCall(PetscFindInt(cone[v], numUnsplitPoints[dep - 1], unsplitPoints[dep - 1], &vertex));
1625             PetscCheck(vertex >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %" PetscInt_FMT " in split or unsplit points of depth %" PetscInt_FMT, cone[v], dep - 1);
1626             coneNew[2 + v] = vertex + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep - 1];
1627           } else {
1628             coneNew[2 + v] = vertex + pMaxNew[dep] + numSplitPoints[dep];
1629           }
1630         }
1631         PetscCall(DMPlexSetCone(sdm, hybface, coneNew));
1632         for (e = 0, qf = 0; e < supportSize; ++e) {
1633           PetscInt val, face;
1634 
1635           PetscCall(DMLabelGetValue(label, support[e], &val));
1636           if (val == dim - 1) {
1637             PetscCall(PetscFindInt(support[e], numSplitPoints[dep + 1], splitPoints[dep + 1], &face));
1638             PetscCheck(face >= 0, comm, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " is not a split face", support[e]);
1639             supportNew[qf++] = face + pMaxNew[dep + 2] + numSplitPoints[dep + 2];
1640           }
1641         }
1642         PetscCall(DMPlexSetSupport(sdm, hybface, supportNew));
1643       }
1644     }
1645   }
1646   for (dep = 0; dep <= depth; ++dep) {
1647     for (p = 0; p < numUnsplitPoints[dep]; ++p) {
1648       const PetscInt  oldp = unsplitPoints[dep][p];
1649       const PetscInt  newp = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*oldp + depthOffset[dep]*/;
1650       const PetscInt *cone, *support;
1651       PetscInt        coneSize, supportSize, supportSizeNew, q, qf, e, f, s;
1652 
1653       PetscCall(DMPlexGetConeSize(dm, oldp, &coneSize));
1654       PetscCall(DMPlexGetCone(dm, oldp, &cone));
1655       PetscCall(DMPlexGetSupportSize(dm, oldp, &supportSize));
1656       PetscCall(DMPlexGetSupport(dm, oldp, &support));
1657       if (dep == 0) {
1658         const PetscInt hybedge = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1] + numSplitPoints[dep];
1659 
1660         /* Unsplit vertex */
1661         PetscCall(DMPlexGetSupportSize(sdm, newp, &supportSizeNew));
1662         for (s = 0, q = 0; s < supportSize; ++s) {
1663           supportNew[q++] = DMPlexShiftPoint_Internal(support[s], depth, depthShift) /*support[s] + depthOffset[dep+1]*/;
1664           PetscCall(PetscFindInt(support[s], numSplitPoints[dep + 1], splitPoints[dep + 1], &e));
1665           if (e >= 0) supportNew[q++] = e + pMaxNew[dep + 1];
1666         }
1667         supportNew[q++] = hybedge;
1668         supportNew[q++] = hybedge;
1669         PetscCheck(q == supportSizeNew, comm, PETSC_ERR_ARG_WRONG, "Support size %" PetscInt_FMT " != %" PetscInt_FMT " for vertex %" PetscInt_FMT, q, supportSizeNew, newp);
1670         PetscCall(DMPlexSetSupport(sdm, newp, supportNew));
1671         /* Hybrid edge */
1672         coneNew[0] = newp;
1673         coneNew[1] = newp;
1674         PetscCall(DMPlexSetCone(sdm, hybedge, coneNew));
1675         for (e = 0, qf = 0; e < supportSize; ++e) {
1676           PetscInt val, edge;
1677 
1678           PetscCall(DMLabelGetValue(label, support[e], &val));
1679           if (val == 1) {
1680             PetscCall(PetscFindInt(support[e], numSplitPoints[dep + 1], splitPoints[dep + 1], &edge));
1681             PetscCheck(edge >= 0, comm, PETSC_ERR_ARG_WRONG, "Edge %" PetscInt_FMT " is not a split edge", support[e]);
1682             supportNew[qf++] = edge + pMaxNew[dep + 2] + numSplitPoints[dep + 2];
1683           } else if (val == (shift2 + 1)) {
1684             PetscCall(PetscFindInt(support[e], numUnsplitPoints[dep + 1], unsplitPoints[dep + 1], &edge));
1685             PetscCheck(edge >= 0, comm, PETSC_ERR_ARG_WRONG, "Edge %" PetscInt_FMT " is not a unsplit edge", support[e]);
1686             supportNew[qf++] = edge + pMaxNew[dep + 2] + numSplitPoints[dep + 2] + numSplitPoints[dep + 1];
1687           }
1688         }
1689         PetscCall(DMPlexSetSupport(sdm, hybedge, supportNew));
1690       } else if (dep == dim - 2) {
1691         const PetscInt hybface = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1] + numSplitPoints[dep];
1692 
1693         /* Unsplit edge: Faces into original edge, split face, and hybrid face twice */
1694         for (f = 0, qf = 0; f < supportSize; ++f) {
1695           PetscInt val, face;
1696 
1697           PetscCall(DMLabelGetValue(label, support[f], &val));
1698           if (val == dim - 1) {
1699             PetscCall(PetscFindInt(support[f], numSplitPoints[dep + 1], splitPoints[dep + 1], &face));
1700             PetscCheck(face >= 0, comm, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " is not a split face", support[f]);
1701             supportNew[qf++] = DMPlexShiftPoint_Internal(support[f], depth, depthShift) /*support[f] + depthOffset[dep+1]*/;
1702             supportNew[qf++] = face + pMaxNew[dep + 1];
1703           } else {
1704             supportNew[qf++] = DMPlexShiftPoint_Internal(support[f], depth, depthShift) /*support[f] + depthOffset[dep+1]*/;
1705           }
1706         }
1707         supportNew[qf++] = hybface;
1708         supportNew[qf++] = hybface;
1709         PetscCall(DMPlexGetSupportSize(sdm, newp, &supportSizeNew));
1710         PetscCheck(qf == supportSizeNew, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Support size for unsplit edge %" PetscInt_FMT " is %" PetscInt_FMT " != %" PetscInt_FMT, newp, qf, supportSizeNew);
1711         PetscCall(DMPlexSetSupport(sdm, newp, supportNew));
1712         /* Add hybrid face */
1713         coneNew[0] = newp;
1714         coneNew[1] = newp;
1715         PetscCall(PetscFindInt(cone[0], numUnsplitPoints[dep - 1], unsplitPoints[dep - 1], &v));
1716         PetscCheck(v >= 0, comm, PETSC_ERR_ARG_WRONG, "Vertex %" PetscInt_FMT " is not an unsplit vertex", cone[0]);
1717         coneNew[2] = v + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep - 1];
1718         PetscCall(PetscFindInt(cone[1], numUnsplitPoints[dep - 1], unsplitPoints[dep - 1], &v));
1719         PetscCheck(v >= 0, comm, PETSC_ERR_ARG_WRONG, "Vertex %" PetscInt_FMT " is not an unsplit vertex", cone[1]);
1720         coneNew[3] = v + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep - 1];
1721         PetscCall(DMPlexSetCone(sdm, hybface, coneNew));
1722         for (f = 0, qf = 0; f < supportSize; ++f) {
1723           PetscInt val, face;
1724 
1725           PetscCall(DMLabelGetValue(label, support[f], &val));
1726           if (val == dim - 1) {
1727             PetscCall(PetscFindInt(support[f], numSplitPoints[dep + 1], splitPoints[dep + 1], &face));
1728             supportNew[qf++] = face + pMaxNew[dep + 2] + numSplitPoints[dep + 2];
1729           }
1730         }
1731         PetscCall(DMPlexGetSupportSize(sdm, hybface, &supportSizeNew));
1732         PetscCheck(qf == supportSizeNew, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Support size for hybrid face %" PetscInt_FMT " is %" PetscInt_FMT " != %" PetscInt_FMT, hybface, qf, supportSizeNew);
1733         PetscCall(DMPlexSetSupport(sdm, hybface, supportNew));
1734       }
1735     }
1736   }
1737   /* Step 6b: Replace split points in negative side cones */
1738   for (sp = 0; sp < numSP; ++sp) {
1739     PetscInt        dep = values[sp];
1740     IS              pIS;
1741     PetscInt        numPoints;
1742     const PetscInt *points;
1743 
1744     if (dep >= 0) continue;
1745     PetscCall(DMLabelGetStratumIS(label, dep, &pIS));
1746     if (!pIS) continue;
1747     dep = -dep - shift;
1748     PetscCall(ISGetLocalSize(pIS, &numPoints));
1749     PetscCall(ISGetIndices(pIS, &points));
1750     for (p = 0; p < numPoints; ++p) {
1751       const PetscInt  oldp = points[p];
1752       const PetscInt  newp = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*depthOffset[dep] + oldp*/;
1753       const PetscInt *cone;
1754       PetscInt        coneSize, c;
1755       /* PetscBool       replaced = PETSC_FALSE; */
1756 
1757       /* Negative edge: replace split vertex */
1758       /* Negative cell: replace split face */
1759       PetscCall(DMPlexGetConeSize(sdm, newp, &coneSize));
1760       PetscCall(DMPlexGetCone(sdm, newp, &cone));
1761       for (c = 0; c < coneSize; ++c) {
1762         const PetscInt coldp = DMPlexShiftPointInverse_Internal(cone[c], depth, depthShift);
1763         PetscInt       csplitp, cp, val;
1764 
1765         PetscCall(DMLabelGetValue(label, coldp, &val));
1766         if (val == dep - 1) {
1767           PetscCall(PetscFindInt(coldp, numSplitPoints[dep - 1], splitPoints[dep - 1], &cp));
1768           PetscCheck(cp >= 0, comm, PETSC_ERR_ARG_WRONG, "Point %" PetscInt_FMT " is not a split point of dimension %" PetscInt_FMT, oldp, dep - 1);
1769           csplitp = pMaxNew[dep - 1] + cp;
1770           PetscCall(DMPlexInsertCone(sdm, newp, c, csplitp));
1771           /* replaced = PETSC_TRUE; */
1772         }
1773       }
1774       /* Cells with only a vertex or edge on the submesh have no replacement */
1775       /* PetscCheck(replaced,comm, PETSC_ERR_ARG_WRONG, "The cone of point %d does not contain split points", oldp); */
1776     }
1777     PetscCall(ISRestoreIndices(pIS, &points));
1778     PetscCall(ISDestroy(&pIS));
1779   }
1780   /* Step 7: Coordinates */
1781   PetscCall(DMPlexShiftCoordinates_Internal(dm, depthShift, sdm));
1782   PetscCall(DMGetCoordinateSection(sdm, &coordSection));
1783   PetscCall(DMGetCoordinatesLocal(sdm, &coordinates));
1784   PetscCall(VecGetArray(coordinates, &coords));
1785   for (v = 0; v < (numSplitPoints ? numSplitPoints[0] : 0); ++v) {
1786     const PetscInt newp   = DMPlexShiftPoint_Internal(splitPoints[0][v], depth, depthShift) /*depthOffset[0] + splitPoints[0][v]*/;
1787     const PetscInt splitp = pMaxNew[0] + v;
1788     PetscInt       dof, off, soff, d;
1789 
1790     PetscCall(PetscSectionGetDof(coordSection, newp, &dof));
1791     PetscCall(PetscSectionGetOffset(coordSection, newp, &off));
1792     PetscCall(PetscSectionGetOffset(coordSection, splitp, &soff));
1793     for (d = 0; d < dof; ++d) coords[soff + d] = coords[off + d];
1794   }
1795   PetscCall(VecRestoreArray(coordinates, &coords));
1796   /* Step 8: SF, if I can figure this out we can split the mesh in parallel */
1797   PetscCall(DMPlexShiftSF_Internal(dm, depthShift, sdm));
1798   /*   TODO We need to associate the ghost points with the correct replica */
1799   /* Step 9: Labels */
1800   PetscCall(DMPlexShiftLabels_Internal(dm, depthShift, sdm));
1801   PetscCall(DMPlexCreateVTKLabel_Internal(dm, PETSC_FALSE, sdm));
1802   PetscCall(DMGetNumLabels(sdm, &numLabels));
1803   for (dep = 0; dep <= depth; ++dep) {
1804     for (p = 0; p < numSplitPoints[dep]; ++p) {
1805       const PetscInt newp   = DMPlexShiftPoint_Internal(splitPoints[dep][p], depth, depthShift) /*depthOffset[dep] + splitPoints[dep][p]*/;
1806       const PetscInt splitp = pMaxNew[dep] + p;
1807       PetscInt       l;
1808 
1809       if (splitLabel) {
1810         const PetscInt val = 100 + dep;
1811 
1812         PetscCall(DMLabelSetValue(splitLabel, newp, val));
1813         PetscCall(DMLabelSetValue(splitLabel, splitp, -val));
1814       }
1815       for (l = 0; l < numLabels; ++l) {
1816         DMLabel     mlabel;
1817         const char *lname;
1818         PetscInt    val;
1819         PetscBool   isDepth;
1820 
1821         PetscCall(DMGetLabelName(sdm, l, &lname));
1822         PetscCall(PetscStrcmp(lname, "depth", &isDepth));
1823         if (isDepth) continue;
1824         PetscCall(DMGetLabel(sdm, lname, &mlabel));
1825         PetscCall(DMLabelGetValue(mlabel, newp, &val));
1826         if (val >= 0) PetscCall(DMLabelSetValue(mlabel, splitp, val));
1827       }
1828     }
1829   }
1830   for (sp = 0; sp < numSP; ++sp) {
1831     const PetscInt dep = values[sp];
1832 
1833     if ((dep < 0) || (dep > depth)) continue;
1834     if (splitIS[dep]) PetscCall(ISRestoreIndices(splitIS[dep], &splitPoints[dep]));
1835     PetscCall(ISDestroy(&splitIS[dep]));
1836     if (unsplitIS[dep]) PetscCall(ISRestoreIndices(unsplitIS[dep], &unsplitPoints[dep]));
1837     PetscCall(ISDestroy(&unsplitIS[dep]));
1838   }
1839   if (ghostIS) PetscCall(ISRestoreIndices(ghostIS, &ghostPoints));
1840   PetscCall(ISDestroy(&ghostIS));
1841   if (label) {
1842     PetscCall(ISRestoreIndices(valueIS, &values));
1843     PetscCall(ISDestroy(&valueIS));
1844   }
1845   for (d = 0; d <= depth; ++d) {
1846     PetscCall(DMPlexGetDepthStratum(sdm, d, NULL, &pEnd));
1847     pMaxNew[d] = pEnd - numHybridPoints[d] - numHybridPointsOld[d];
1848   }
1849   PetscCall(PetscFree3(coneNew, coneONew, supportNew));
1850   PetscCall(PetscFree5(depthMax, depthEnd, depthShift, pMaxNew, numHybridPointsOld));
1851   PetscCall(PetscFree7(splitIS, unsplitIS, numSplitPoints, numUnsplitPoints, numHybridPoints, splitPoints, unsplitPoints));
1852   PetscFunctionReturn(PETSC_SUCCESS);
1853 }
1854 
1855 /*@C
1856   DMPlexConstructCohesiveCells - Construct cohesive cells which split the face along an internal interface
1857 
1858   Collective
1859 
1860   Input Parameters:
1861 + dm - The original `DM`
1862 - label - The `DMLabel` specifying the boundary faces (this could be auto-generated)
1863 
1864   Output Parameters:
1865 + splitLabel - The `DMLabel` containing the split points, or `NULL` if no output is desired
1866 - dmSplit - The new `DM`
1867 
1868   Level: developer
1869 
1870 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMCreate()`, `DMPlexLabelCohesiveComplete()`
1871 @*/
1872 PetscErrorCode DMPlexConstructCohesiveCells(DM dm, DMLabel label, DMLabel splitLabel, DM *dmSplit)
1873 {
1874   DM       sdm;
1875   PetscInt dim;
1876 
1877   PetscFunctionBegin;
1878   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1879   PetscValidPointer(dmSplit, 4);
1880   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &sdm));
1881   PetscCall(DMSetType(sdm, DMPLEX));
1882   PetscCall(DMGetDimension(dm, &dim));
1883   PetscCall(DMSetDimension(sdm, dim));
1884   switch (dim) {
1885   case 2:
1886   case 3:
1887     PetscCall(DMPlexConstructCohesiveCells_Internal(dm, label, splitLabel, sdm));
1888     break;
1889   default:
1890     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cannot construct cohesive cells for dimension %" PetscInt_FMT, dim);
1891   }
1892   *dmSplit = sdm;
1893   PetscFunctionReturn(PETSC_SUCCESS);
1894 }
1895 
1896 /* Returns the side of the surface for a given cell with a face on the surface */
1897 static PetscErrorCode GetSurfaceSide_Static(DM dm, DM subdm, PetscInt numSubpoints, const PetscInt *subpoints, PetscInt cell, PetscInt face, PetscBool *pos)
1898 {
1899   const PetscInt *cone, *ornt;
1900   PetscInt        dim, coneSize, c;
1901 
1902   PetscFunctionBegin;
1903   *pos = PETSC_TRUE;
1904   PetscCall(DMGetDimension(dm, &dim));
1905   PetscCall(DMPlexGetConeSize(dm, cell, &coneSize));
1906   PetscCall(DMPlexGetCone(dm, cell, &cone));
1907   PetscCall(DMPlexGetConeOrientation(dm, cell, &ornt));
1908   for (c = 0; c < coneSize; ++c) {
1909     if (cone[c] == face) {
1910       PetscInt o = ornt[c];
1911 
1912       if (subdm) {
1913         const PetscInt *subcone, *subornt;
1914         PetscInt        subpoint, subface, subconeSize, sc;
1915 
1916         PetscCall(PetscFindInt(cell, numSubpoints, subpoints, &subpoint));
1917         PetscCall(PetscFindInt(face, numSubpoints, subpoints, &subface));
1918         PetscCall(DMPlexGetConeSize(subdm, subpoint, &subconeSize));
1919         PetscCall(DMPlexGetCone(subdm, subpoint, &subcone));
1920         PetscCall(DMPlexGetConeOrientation(subdm, subpoint, &subornt));
1921         for (sc = 0; sc < subconeSize; ++sc) {
1922           if (subcone[sc] == subface) {
1923             o = subornt[0];
1924             break;
1925           }
1926         }
1927         PetscCheck(sc < subconeSize, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not find subpoint %" PetscInt_FMT " (%" PetscInt_FMT ") in cone for subpoint %" PetscInt_FMT " (%" PetscInt_FMT ")", subface, face, subpoint, cell);
1928       }
1929       if (o >= 0) *pos = PETSC_TRUE;
1930       else *pos = PETSC_FALSE;
1931       break;
1932     }
1933   }
1934   PetscCheck(c != coneSize, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Cell %" PetscInt_FMT " in split face %" PetscInt_FMT " support does not have it in the cone", cell, face);
1935   PetscFunctionReturn(PETSC_SUCCESS);
1936 }
1937 
1938 static PetscErrorCode CheckFaultEdge_Private(DM dm, DMLabel label)
1939 {
1940   IS              facePosIS, faceNegIS, dimIS;
1941   const PetscInt *points;
1942   PetscInt        dim, numPoints, p, shift = 100, shift2 = 200;
1943 
1944   PetscFunctionBegin;
1945   PetscCall(DMGetDimension(dm, &dim));
1946   /* If any faces touching the fault divide cells on either side, split them */
1947   PetscCall(DMLabelGetStratumIS(label, shift + dim - 1, &facePosIS));
1948   PetscCall(DMLabelGetStratumIS(label, -(shift + dim - 1), &faceNegIS));
1949   if (!facePosIS || !faceNegIS) {
1950     PetscCall(ISDestroy(&facePosIS));
1951     PetscCall(ISDestroy(&faceNegIS));
1952     PetscFunctionReturn(PETSC_SUCCESS);
1953   }
1954   PetscCall(ISExpand(facePosIS, faceNegIS, &dimIS));
1955   PetscCall(ISDestroy(&facePosIS));
1956   PetscCall(ISDestroy(&faceNegIS));
1957   PetscCall(ISGetLocalSize(dimIS, &numPoints));
1958   PetscCall(ISGetIndices(dimIS, &points));
1959   for (p = 0; p < numPoints; ++p) {
1960     const PetscInt  point = points[p];
1961     const PetscInt *support;
1962     PetscInt        supportSize, valA, valB;
1963 
1964     PetscCall(DMPlexGetSupportSize(dm, point, &supportSize));
1965     if (supportSize != 2) continue;
1966     PetscCall(DMPlexGetSupport(dm, point, &support));
1967     PetscCall(DMLabelGetValue(label, support[0], &valA));
1968     PetscCall(DMLabelGetValue(label, support[1], &valB));
1969     if ((valA == -1) || (valB == -1)) continue;
1970     if (valA * valB > 0) continue;
1971     /* Check that this face is not incident on only unsplit faces, meaning has at least one split face */
1972     {
1973       PetscInt *closure = NULL;
1974       PetscBool split   = PETSC_FALSE;
1975       PetscInt  closureSize, cl;
1976 
1977       PetscCall(DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
1978       for (cl = 0; cl < closureSize * 2; cl += 2) {
1979         PetscCall(DMLabelGetValue(label, closure[cl], &valA));
1980         if ((valA >= 0) && (valA <= dim)) {
1981           split = PETSC_TRUE;
1982           break;
1983         }
1984       }
1985       PetscCall(DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
1986       if (!split) continue;
1987     }
1988     /* Split the face */
1989     PetscCall(DMLabelGetValue(label, point, &valA));
1990     PetscCall(DMLabelClearValue(label, point, valA));
1991     PetscCall(DMLabelSetValue(label, point, dim - 1));
1992     /* Label its closure:
1993       unmarked: label as unsplit
1994       incident: relabel as split
1995       split:    do nothing
1996     */
1997     {
1998       PetscInt *closure = NULL;
1999       PetscInt  closureSize, cl, dep;
2000 
2001       PetscCall(DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
2002       for (cl = 0; cl < closureSize * 2; cl += 2) {
2003         PetscCall(DMLabelGetValue(label, closure[cl], &valA));
2004         if (valA == -1) { /* Mark as unsplit */
2005           PetscCall(DMPlexGetPointDepth(dm, closure[cl], &dep));
2006           PetscCall(DMLabelSetValue(label, closure[cl], shift2 + dep));
2007         } else if (((valA >= shift) && (valA < shift2)) || ((valA <= -shift) && (valA > -shift2))) {
2008           PetscCall(DMPlexGetPointDepth(dm, closure[cl], &dep));
2009           PetscCall(DMLabelClearValue(label, closure[cl], valA));
2010           PetscCall(DMLabelSetValue(label, closure[cl], dep));
2011         }
2012       }
2013       PetscCall(DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
2014     }
2015   }
2016   PetscCall(ISRestoreIndices(dimIS, &points));
2017   PetscCall(ISDestroy(&dimIS));
2018   PetscFunctionReturn(PETSC_SUCCESS);
2019 }
2020 
2021 /*@
2022   DMPlexLabelCohesiveComplete - Starting with a label marking points on an internal surface, we add all other mesh pieces
2023   to complete the surface
2024 
2025   Input Parameters:
2026 + dm     - The `DM`
2027 . label  - A `DMLabel` marking the surface
2028 . blabel - A `DMLabel` marking the vertices on the boundary which will not be duplicated, or `NULL` to find them automatically
2029 . bvalue - Value of `DMLabel` marking the vertices on the boundary
2030 . flip   - Flag to flip the submesh normal and replace points on the other side
2031 - subdm  - The `DM` associated with the label, or `NULL`
2032 
2033   Output Parameter:
2034 . label - A `DMLabel` marking all surface points
2035 
2036   Level: developer
2037 
2038   Note:
2039   The vertices in blabel are called "unsplit" in the terminology from hybrid cell creation.
2040 
2041 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexConstructCohesiveCells()`, `DMPlexLabelComplete()`
2042 @*/
2043 PetscErrorCode DMPlexLabelCohesiveComplete(DM dm, DMLabel label, DMLabel blabel, PetscInt bvalue, PetscBool flip, DM subdm)
2044 {
2045   DMLabel         depthLabel;
2046   IS              dimIS, subpointIS = NULL;
2047   const PetscInt *points, *subpoints;
2048   const PetscInt  rev   = flip ? -1 : 1;
2049   PetscInt        shift = 100, shift2 = 200, shift3 = 300, dim, depth, numPoints, numSubpoints, p, val;
2050 
2051   PetscFunctionBegin;
2052   PetscCall(DMPlexGetDepth(dm, &depth));
2053   PetscCall(DMGetDimension(dm, &dim));
2054   PetscCall(DMPlexGetDepthLabel(dm, &depthLabel));
2055   if (subdm) {
2056     PetscCall(DMPlexGetSubpointIS(subdm, &subpointIS));
2057     if (subpointIS) {
2058       PetscCall(ISGetLocalSize(subpointIS, &numSubpoints));
2059       PetscCall(ISGetIndices(subpointIS, &subpoints));
2060     }
2061   }
2062   /* Mark cell on the fault, and its faces which touch the fault: cell orientation for face gives the side of the fault */
2063   PetscCall(DMLabelGetStratumIS(label, dim - 1, &dimIS));
2064   if (!dimIS) goto divide;
2065   PetscCall(ISGetLocalSize(dimIS, &numPoints));
2066   PetscCall(ISGetIndices(dimIS, &points));
2067   for (p = 0; p < numPoints; ++p) { /* Loop over fault faces */
2068     const PetscInt *support;
2069     PetscInt        supportSize, s;
2070 
2071     PetscCall(DMPlexGetSupportSize(dm, points[p], &supportSize));
2072 #if 0
2073     if (supportSize != 2) {
2074       const PetscInt *lp;
2075       PetscInt        Nlp, pind;
2076 
2077       /* Check that for a cell with a single support face, that face is in the SF */
2078       /*   THis check only works for the remote side. We would need root side information */
2079       PetscCall(PetscSFGetGraph(dm->sf, NULL, &Nlp, &lp, NULL));
2080       PetscCall(PetscFindInt(points[p], Nlp, lp, &pind));
2081       PetscCheck(pind >= 0,PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Split face %" PetscInt_FMT " has %" PetscInt_FMT " != 2 supports, and the face is not shared with another process", points[p], supportSize);
2082     }
2083 #endif
2084     PetscCall(DMPlexGetSupport(dm, points[p], &support));
2085     for (s = 0; s < supportSize; ++s) {
2086       const PetscInt *cone;
2087       PetscInt        coneSize, c;
2088       PetscBool       pos;
2089 
2090       PetscCall(GetSurfaceSide_Static(dm, subdm, numSubpoints, subpoints, support[s], points[p], &pos));
2091       if (pos) PetscCall(DMLabelSetValue(label, support[s], rev * (shift + dim)));
2092       else PetscCall(DMLabelSetValue(label, support[s], -rev * (shift + dim)));
2093       if (rev < 0) pos = !pos ? PETSC_TRUE : PETSC_FALSE;
2094       /* Put faces touching the fault in the label */
2095       PetscCall(DMPlexGetConeSize(dm, support[s], &coneSize));
2096       PetscCall(DMPlexGetCone(dm, support[s], &cone));
2097       for (c = 0; c < coneSize; ++c) {
2098         const PetscInt point = cone[c];
2099 
2100         PetscCall(DMLabelGetValue(label, point, &val));
2101         if (val == -1) {
2102           PetscInt *closure = NULL;
2103           PetscInt  closureSize, cl;
2104 
2105           PetscCall(DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
2106           for (cl = 0; cl < closureSize * 2; cl += 2) {
2107             const PetscInt clp  = closure[cl];
2108             PetscInt       bval = -1;
2109 
2110             PetscCall(DMLabelGetValue(label, clp, &val));
2111             if (blabel) PetscCall(DMLabelGetValue(blabel, clp, &bval));
2112             if ((val >= 0) && (val < dim - 1) && (bval < 0)) {
2113               PetscCall(DMLabelSetValue(label, point, pos == PETSC_TRUE ? shift + dim - 1 : -(shift + dim - 1)));
2114               break;
2115             }
2116           }
2117           PetscCall(DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
2118         }
2119       }
2120     }
2121   }
2122   PetscCall(ISRestoreIndices(dimIS, &points));
2123   PetscCall(ISDestroy(&dimIS));
2124   /* Mark boundary points as unsplit */
2125   if (blabel) {
2126     IS bdIS;
2127 
2128     PetscCall(DMLabelGetStratumIS(blabel, bvalue, &bdIS));
2129     PetscCall(ISGetLocalSize(bdIS, &numPoints));
2130     PetscCall(ISGetIndices(bdIS, &points));
2131     for (p = 0; p < numPoints; ++p) {
2132       const PetscInt point = points[p];
2133       PetscInt       val, bval;
2134 
2135       PetscCall(DMLabelGetValue(blabel, point, &bval));
2136       if (bval >= 0) {
2137         PetscCall(DMLabelGetValue(label, point, &val));
2138         if ((val < 0) || (val > dim)) {
2139           /* This could be a point added from splitting a vertex on an adjacent fault, otherwise its just wrong */
2140           PetscCall(DMLabelClearValue(blabel, point, bval));
2141         }
2142       }
2143     }
2144     for (p = 0; p < numPoints; ++p) {
2145       const PetscInt point = points[p];
2146       PetscInt       val, bval;
2147 
2148       PetscCall(DMLabelGetValue(blabel, point, &bval));
2149       if (bval >= 0) {
2150         const PetscInt *cone, *support;
2151         PetscInt        coneSize, supportSize, s, valA, valB, valE;
2152 
2153         /* Mark as unsplit */
2154         PetscCall(DMLabelGetValue(label, point, &val));
2155         PetscCheck(val >= 0 && val <= dim, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %" PetscInt_FMT " has label value %" PetscInt_FMT ", should be part of the fault", point, val);
2156         PetscCall(DMLabelClearValue(label, point, val));
2157         PetscCall(DMLabelSetValue(label, point, shift2 + val));
2158         /* Check for cross-edge
2159              A cross-edge has endpoints which are both on the boundary of the surface, but the edge itself is not. */
2160         if (val != 0) continue;
2161         PetscCall(DMPlexGetSupport(dm, point, &support));
2162         PetscCall(DMPlexGetSupportSize(dm, point, &supportSize));
2163         for (s = 0; s < supportSize; ++s) {
2164           PetscCall(DMPlexGetCone(dm, support[s], &cone));
2165           PetscCall(DMPlexGetConeSize(dm, support[s], &coneSize));
2166           PetscCheck(coneSize == 2, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Edge %" PetscInt_FMT " has %" PetscInt_FMT " vertices != 2", support[s], coneSize);
2167           PetscCall(DMLabelGetValue(blabel, cone[0], &valA));
2168           PetscCall(DMLabelGetValue(blabel, cone[1], &valB));
2169           PetscCall(DMLabelGetValue(blabel, support[s], &valE));
2170           if ((valE < 0) && (valA >= 0) && (valB >= 0) && (cone[0] != cone[1])) PetscCall(DMLabelSetValue(blabel, support[s], 2));
2171         }
2172       }
2173     }
2174     PetscCall(ISRestoreIndices(bdIS, &points));
2175     PetscCall(ISDestroy(&bdIS));
2176   }
2177   /* Mark ghost fault cells */
2178   {
2179     PetscSF         sf;
2180     const PetscInt *leaves;
2181     PetscInt        Nl, l;
2182 
2183     PetscCall(DMGetPointSF(dm, &sf));
2184     PetscCall(PetscSFGetGraph(sf, NULL, &Nl, &leaves, NULL));
2185     PetscCall(DMLabelGetStratumIS(label, dim - 1, &dimIS));
2186     if (!dimIS) goto divide;
2187     PetscCall(ISGetLocalSize(dimIS, &numPoints));
2188     PetscCall(ISGetIndices(dimIS, &points));
2189     if (Nl > 0) {
2190       for (p = 0; p < numPoints; ++p) {
2191         const PetscInt point = points[p];
2192         PetscInt       val;
2193 
2194         PetscCall(PetscFindInt(point, Nl, leaves, &l));
2195         if (l >= 0) {
2196           PetscInt *closure = NULL;
2197           PetscInt  closureSize, cl;
2198 
2199           PetscCall(DMLabelGetValue(label, point, &val));
2200           PetscCheck((val == dim - 1) || (val == shift2 + dim - 1), PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %" PetscInt_FMT " has label value %" PetscInt_FMT ", should be a fault face", point, val);
2201           PetscCall(DMLabelClearValue(label, point, val));
2202           PetscCall(DMLabelSetValue(label, point, shift3 + val));
2203           PetscCall(DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
2204           for (cl = 2; cl < closureSize * 2; cl += 2) {
2205             const PetscInt clp = closure[cl];
2206 
2207             PetscCall(DMLabelGetValue(label, clp, &val));
2208             PetscCheck(val != -1, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %" PetscInt_FMT " is missing from label, but is in the closure of a fault face", point);
2209             PetscCall(DMLabelClearValue(label, clp, val));
2210             PetscCall(DMLabelSetValue(label, clp, shift3 + val));
2211           }
2212           PetscCall(DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
2213         }
2214       }
2215     }
2216     PetscCall(ISRestoreIndices(dimIS, &points));
2217     PetscCall(ISDestroy(&dimIS));
2218   }
2219 divide:
2220   if (subpointIS) PetscCall(ISRestoreIndices(subpointIS, &subpoints));
2221   PetscCall(DMPlexLabelFaultHalo(dm, label));
2222   PetscCall(CheckFaultEdge_Private(dm, label));
2223   PetscFunctionReturn(PETSC_SUCCESS);
2224 }
2225 
2226 /* Check that no cell have all vertices on the fault */
2227 PetscErrorCode DMPlexCheckValidSubmesh_Private(DM dm, DMLabel label, DM subdm)
2228 {
2229   IS              subpointIS;
2230   const PetscInt *dmpoints;
2231   PetscInt        defaultValue, cStart, cEnd, c, vStart, vEnd;
2232 
2233   PetscFunctionBegin;
2234   if (!label) PetscFunctionReturn(PETSC_SUCCESS);
2235   PetscCall(DMLabelGetDefaultValue(label, &defaultValue));
2236   PetscCall(DMPlexGetSubpointIS(subdm, &subpointIS));
2237   if (!subpointIS) PetscFunctionReturn(PETSC_SUCCESS);
2238   PetscCall(DMPlexGetHeightStratum(subdm, 0, &cStart, &cEnd));
2239   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
2240   PetscCall(ISGetIndices(subpointIS, &dmpoints));
2241   for (c = cStart; c < cEnd; ++c) {
2242     PetscBool invalidCell = PETSC_TRUE;
2243     PetscInt *closure     = NULL;
2244     PetscInt  closureSize, cl;
2245 
2246     PetscCall(DMPlexGetTransitiveClosure(dm, dmpoints[c], PETSC_TRUE, &closureSize, &closure));
2247     for (cl = 0; cl < closureSize * 2; cl += 2) {
2248       PetscInt value = 0;
2249 
2250       if ((closure[cl] < vStart) || (closure[cl] >= vEnd)) continue;
2251       PetscCall(DMLabelGetValue(label, closure[cl], &value));
2252       if (value == defaultValue) {
2253         invalidCell = PETSC_FALSE;
2254         break;
2255       }
2256     }
2257     PetscCall(DMPlexRestoreTransitiveClosure(dm, dmpoints[c], PETSC_TRUE, &closureSize, &closure));
2258     if (invalidCell) {
2259       PetscCall(ISRestoreIndices(subpointIS, &dmpoints));
2260       PetscCall(ISDestroy(&subpointIS));
2261       PetscCall(DMDestroy(&subdm));
2262       SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Ambiguous submesh. Cell %" PetscInt_FMT " has all of its vertices on the submesh.", dmpoints[c]);
2263     }
2264   }
2265   PetscCall(ISRestoreIndices(subpointIS, &dmpoints));
2266   PetscFunctionReturn(PETSC_SUCCESS);
2267 }
2268 
2269 /*@
2270   DMPlexCreateHybridMesh - Create a mesh with hybrid cells along an internal interface
2271 
2272   Collective
2273 
2274   Input Parameters:
2275 + dm - The original `DM`
2276 . label - The label specifying the interface vertices
2277 . bdlabel - The optional label specifying the interface boundary vertices
2278 - bdvalue - Value of optional label specifying the interface boundary vertices
2279 
2280   Output Parameters:
2281 + hybridLabel - The label fully marking the interface, or `NULL` if no output is desired
2282 . splitLabel - The label containing the split points, or `NULL` if no output is desired
2283 . dmInterface - The new interface `DM`, or `NULL`
2284 - dmHybrid - The new `DM` with cohesive cells
2285 
2286   Level: developer
2287 
2288   Note:
2289   The hybridLabel indicates what parts of the original mesh impinged on the division surface. For points
2290   directly on the division surface, they are labeled with their dimension, so an edge 7 on the division surface would be
2291   7 (1) in hybridLabel. For points that impinge from the positive side, they are labeled with 100+dim, so an edge 6 with
2292   one vertex 3 on the surface would be 6 (101) and 3 (0) in hybridLabel. If an edge 9 from the negative side of the
2293   surface also hits vertex 3, it would be 9 (-101) in hybridLabel.
2294 
2295   The splitLabel indicates what points in the new hybrid mesh were the result of splitting points in the original
2296   mesh. The label value is $\pm 100+dim$ for each point. For example, if two edges 10 and 14 in the hybrid resulting from
2297   splitting an edge in the original mesh, you would have 10 (101) and 14 (-101) in the splitLabel.
2298 
2299   The dmInterface is a `DM` built from the original division surface. It has a label which can be retrieved using
2300   `DMPlexGetSubpointMap()` which maps each point back to the point in the surface of the original mesh.
2301 
2302 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexConstructCohesiveCells()`, `DMPlexLabelCohesiveComplete()`, `DMPlexGetSubpointMap()`, `DMCreate()`
2303 @*/
2304 PetscErrorCode DMPlexCreateHybridMesh(DM dm, DMLabel label, DMLabel bdlabel, PetscInt bdvalue, DMLabel *hybridLabel, DMLabel *splitLabel, DM *dmInterface, DM *dmHybrid)
2305 {
2306   DM       idm;
2307   DMLabel  subpointMap, hlabel, slabel = NULL;
2308   PetscInt dim;
2309 
2310   PetscFunctionBegin;
2311   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2312   if (label) PetscValidPointer(label, 2);
2313   if (bdlabel) PetscValidPointer(bdlabel, 3);
2314   if (hybridLabel) PetscValidPointer(hybridLabel, 5);
2315   if (splitLabel) PetscValidPointer(splitLabel, 6);
2316   if (dmInterface) PetscValidPointer(dmInterface, 7);
2317   PetscValidPointer(dmHybrid, 8);
2318   PetscCall(DMGetDimension(dm, &dim));
2319   PetscCall(DMPlexCreateSubmesh(dm, label, 1, PETSC_FALSE, &idm));
2320   PetscCall(DMPlexCheckValidSubmesh_Private(dm, label, idm));
2321   PetscCall(DMPlexOrient(idm));
2322   PetscCall(DMPlexGetSubpointMap(idm, &subpointMap));
2323   PetscCall(DMLabelDuplicate(subpointMap, &hlabel));
2324   PetscCall(DMLabelClearStratum(hlabel, dim));
2325   if (splitLabel) {
2326     const char *name;
2327     char        sname[PETSC_MAX_PATH_LEN];
2328 
2329     PetscCall(PetscObjectGetName((PetscObject)hlabel, &name));
2330     PetscCall(PetscStrncpy(sname, name, sizeof(sname)));
2331     PetscCall(PetscStrlcat(sname, " split", sizeof(sname)));
2332     PetscCall(DMLabelCreate(PETSC_COMM_SELF, sname, &slabel));
2333   }
2334   PetscCall(DMPlexLabelCohesiveComplete(dm, hlabel, bdlabel, bdvalue, PETSC_FALSE, idm));
2335   if (dmInterface) {
2336     *dmInterface = idm;
2337   } else PetscCall(DMDestroy(&idm));
2338   PetscCall(DMPlexConstructCohesiveCells(dm, hlabel, slabel, dmHybrid));
2339   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_TRUE, *dmHybrid));
2340   if (hybridLabel) *hybridLabel = hlabel;
2341   else PetscCall(DMLabelDestroy(&hlabel));
2342   if (splitLabel) *splitLabel = slabel;
2343   {
2344     DM      cdm;
2345     DMLabel ctLabel;
2346 
2347     /* We need to somehow share the celltype label with the coordinate dm */
2348     PetscCall(DMGetCoordinateDM(*dmHybrid, &cdm));
2349     PetscCall(DMPlexGetCellTypeLabel(*dmHybrid, &ctLabel));
2350     PetscCall(DMSetLabel(cdm, ctLabel));
2351   }
2352   PetscFunctionReturn(PETSC_SUCCESS);
2353 }
2354 
2355 /* Here we need the explicit assumption that:
2356 
2357      For any marked cell, the marked vertices constitute a single face
2358 */
2359 static PetscErrorCode DMPlexMarkSubmesh_Uninterpolated(DM dm, DMLabel vertexLabel, PetscInt value, DMLabel subpointMap, PetscInt *numFaces, PetscInt *nFV, DM subdm)
2360 {
2361   IS              subvertexIS = NULL;
2362   const PetscInt *subvertices;
2363   PetscInt       *pStart, *pEnd, pSize;
2364   PetscInt        depth, dim, d, numSubVerticesInitial = 0, v;
2365 
2366   PetscFunctionBegin;
2367   *numFaces = 0;
2368   *nFV      = 0;
2369   PetscCall(DMPlexGetDepth(dm, &depth));
2370   PetscCall(DMGetDimension(dm, &dim));
2371   pSize = PetscMax(depth, dim) + 1;
2372   PetscCall(PetscMalloc2(pSize, &pStart, pSize, &pEnd));
2373   for (d = 0; d <= depth; ++d) PetscCall(DMPlexGetSimplexOrBoxCells(dm, depth - d, &pStart[d], &pEnd[d]));
2374   /* Loop over initial vertices and mark all faces in the collective star() */
2375   if (vertexLabel) PetscCall(DMLabelGetStratumIS(vertexLabel, value, &subvertexIS));
2376   if (subvertexIS) {
2377     PetscCall(ISGetSize(subvertexIS, &numSubVerticesInitial));
2378     PetscCall(ISGetIndices(subvertexIS, &subvertices));
2379   }
2380   for (v = 0; v < numSubVerticesInitial; ++v) {
2381     const PetscInt vertex = subvertices[v];
2382     PetscInt      *star   = NULL;
2383     PetscInt       starSize, s, numCells = 0, c;
2384 
2385     PetscCall(DMPlexGetTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star));
2386     for (s = 0; s < starSize * 2; s += 2) {
2387       const PetscInt point = star[s];
2388       if ((point >= pStart[depth]) && (point < pEnd[depth])) star[numCells++] = point;
2389     }
2390     for (c = 0; c < numCells; ++c) {
2391       const PetscInt cell    = star[c];
2392       PetscInt      *closure = NULL;
2393       PetscInt       closureSize, cl;
2394       PetscInt       cellLoc, numCorners = 0, faceSize = 0;
2395 
2396       PetscCall(DMLabelGetValue(subpointMap, cell, &cellLoc));
2397       if (cellLoc == 2) continue;
2398       PetscCheck(cellLoc < 0, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Cell %" PetscInt_FMT " has dimension %" PetscInt_FMT " in the surface label", cell, cellLoc);
2399       PetscCall(DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
2400       for (cl = 0; cl < closureSize * 2; cl += 2) {
2401         const PetscInt point = closure[cl];
2402         PetscInt       vertexLoc;
2403 
2404         if ((point >= pStart[0]) && (point < pEnd[0])) {
2405           ++numCorners;
2406           PetscCall(DMLabelGetValue(vertexLabel, point, &vertexLoc));
2407           if (vertexLoc == value) closure[faceSize++] = point;
2408         }
2409       }
2410       if (!(*nFV)) PetscCall(DMPlexGetNumFaceVertices(dm, dim, numCorners, nFV));
2411       PetscCheck(faceSize <= *nFV, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Invalid submesh: Too many vertices %" PetscInt_FMT " of an element on the surface", faceSize);
2412       if (faceSize == *nFV) {
2413         const PetscInt *cells = NULL;
2414         PetscInt        numCells, nc;
2415 
2416         ++(*numFaces);
2417         for (cl = 0; cl < faceSize; ++cl) PetscCall(DMLabelSetValue(subpointMap, closure[cl], 0));
2418         PetscCall(DMPlexGetJoin(dm, faceSize, closure, &numCells, &cells));
2419         for (nc = 0; nc < numCells; ++nc) PetscCall(DMLabelSetValue(subpointMap, cells[nc], 2));
2420         PetscCall(DMPlexRestoreJoin(dm, faceSize, closure, &numCells, &cells));
2421       }
2422       PetscCall(DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
2423     }
2424     PetscCall(DMPlexRestoreTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star));
2425   }
2426   if (subvertexIS) PetscCall(ISRestoreIndices(subvertexIS, &subvertices));
2427   PetscCall(ISDestroy(&subvertexIS));
2428   PetscCall(PetscFree2(pStart, pEnd));
2429   PetscFunctionReturn(PETSC_SUCCESS);
2430 }
2431 
2432 static PetscErrorCode DMPlexMarkSubmesh_Interpolated(DM dm, DMLabel vertexLabel, PetscInt value, PetscBool markedFaces, DMLabel subpointMap, DM subdm)
2433 {
2434   IS              subvertexIS = NULL;
2435   const PetscInt *subvertices;
2436   PetscInt       *pStart, *pEnd;
2437   PetscInt        dim, d, numSubVerticesInitial = 0, v;
2438 
2439   PetscFunctionBegin;
2440   PetscCall(DMGetDimension(dm, &dim));
2441   PetscCall(PetscMalloc2(dim + 1, &pStart, dim + 1, &pEnd));
2442   for (d = 0; d <= dim; ++d) PetscCall(DMPlexGetSimplexOrBoxCells(dm, dim - d, &pStart[d], &pEnd[d]));
2443   /* Loop over initial vertices and mark all faces in the collective star() */
2444   if (vertexLabel) {
2445     PetscCall(DMLabelGetStratumIS(vertexLabel, value, &subvertexIS));
2446     if (subvertexIS) {
2447       PetscCall(ISGetSize(subvertexIS, &numSubVerticesInitial));
2448       PetscCall(ISGetIndices(subvertexIS, &subvertices));
2449     }
2450   }
2451   for (v = 0; v < numSubVerticesInitial; ++v) {
2452     const PetscInt vertex = subvertices[v];
2453     PetscInt      *star   = NULL;
2454     PetscInt       starSize, s, numFaces = 0, f;
2455 
2456     PetscCall(DMPlexGetTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star));
2457     for (s = 0; s < starSize * 2; s += 2) {
2458       const PetscInt point = star[s];
2459       PetscInt       faceLoc;
2460 
2461       if ((point >= pStart[dim - 1]) && (point < pEnd[dim - 1])) {
2462         if (markedFaces) {
2463           PetscCall(DMLabelGetValue(vertexLabel, point, &faceLoc));
2464           if (faceLoc < 0) continue;
2465         }
2466         star[numFaces++] = point;
2467       }
2468     }
2469     for (f = 0; f < numFaces; ++f) {
2470       const PetscInt face    = star[f];
2471       PetscInt      *closure = NULL;
2472       PetscInt       closureSize, c;
2473       PetscInt       faceLoc;
2474 
2475       PetscCall(DMLabelGetValue(subpointMap, face, &faceLoc));
2476       if (faceLoc == dim - 1) continue;
2477       PetscCheck(faceLoc < 0, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Face %" PetscInt_FMT " has dimension %" PetscInt_FMT " in the surface label", face, faceLoc);
2478       PetscCall(DMPlexGetTransitiveClosure(dm, face, PETSC_TRUE, &closureSize, &closure));
2479       for (c = 0; c < closureSize * 2; c += 2) {
2480         const PetscInt point = closure[c];
2481         PetscInt       vertexLoc;
2482 
2483         if ((point >= pStart[0]) && (point < pEnd[0])) {
2484           PetscCall(DMLabelGetValue(vertexLabel, point, &vertexLoc));
2485           if (vertexLoc != value) break;
2486         }
2487       }
2488       if (c == closureSize * 2) {
2489         const PetscInt *support;
2490         PetscInt        supportSize, s;
2491 
2492         for (c = 0; c < closureSize * 2; c += 2) {
2493           const PetscInt point = closure[c];
2494 
2495           for (d = 0; d < dim; ++d) {
2496             if ((point >= pStart[d]) && (point < pEnd[d])) {
2497               PetscCall(DMLabelSetValue(subpointMap, point, d));
2498               break;
2499             }
2500           }
2501         }
2502         PetscCall(DMPlexGetSupportSize(dm, face, &supportSize));
2503         PetscCall(DMPlexGetSupport(dm, face, &support));
2504         for (s = 0; s < supportSize; ++s) PetscCall(DMLabelSetValue(subpointMap, support[s], dim));
2505       }
2506       PetscCall(DMPlexRestoreTransitiveClosure(dm, face, PETSC_TRUE, &closureSize, &closure));
2507     }
2508     PetscCall(DMPlexRestoreTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star));
2509   }
2510   if (subvertexIS) PetscCall(ISRestoreIndices(subvertexIS, &subvertices));
2511   PetscCall(ISDestroy(&subvertexIS));
2512   PetscCall(PetscFree2(pStart, pEnd));
2513   PetscFunctionReturn(PETSC_SUCCESS);
2514 }
2515 
2516 static PetscErrorCode DMPlexMarkCohesiveSubmesh_Uninterpolated(DM dm, PetscBool hasLagrange, const char labelname[], PetscInt value, DMLabel subpointMap, PetscInt *numFaces, PetscInt *nFV, PetscInt *subCells[], DM subdm)
2517 {
2518   DMLabel         label = NULL;
2519   const PetscInt *cone;
2520   PetscInt        dim, cMax, cEnd, c, subc = 0, p, coneSize = -1;
2521 
2522   PetscFunctionBegin;
2523   *numFaces = 0;
2524   *nFV      = 0;
2525   if (labelname) PetscCall(DMGetLabel(dm, labelname, &label));
2526   *subCells = NULL;
2527   PetscCall(DMGetDimension(dm, &dim));
2528   PetscCall(DMPlexGetTensorPrismBounds_Internal(dm, dim, &cMax, &cEnd));
2529   if (cMax < 0) PetscFunctionReturn(PETSC_SUCCESS);
2530   if (label) {
2531     for (c = cMax; c < cEnd; ++c) {
2532       PetscInt val;
2533 
2534       PetscCall(DMLabelGetValue(label, c, &val));
2535       if (val == value) {
2536         ++(*numFaces);
2537         PetscCall(DMPlexGetConeSize(dm, c, &coneSize));
2538       }
2539     }
2540   } else {
2541     *numFaces = cEnd - cMax;
2542     PetscCall(DMPlexGetConeSize(dm, cMax, &coneSize));
2543   }
2544   PetscCall(PetscMalloc1(*numFaces * 2, subCells));
2545   if (!(*numFaces)) PetscFunctionReturn(PETSC_SUCCESS);
2546   *nFV = hasLagrange ? coneSize / 3 : coneSize / 2;
2547   for (c = cMax; c < cEnd; ++c) {
2548     const PetscInt *cells;
2549     PetscInt        numCells;
2550 
2551     if (label) {
2552       PetscInt val;
2553 
2554       PetscCall(DMLabelGetValue(label, c, &val));
2555       if (val != value) continue;
2556     }
2557     PetscCall(DMPlexGetCone(dm, c, &cone));
2558     for (p = 0; p < *nFV; ++p) PetscCall(DMLabelSetValue(subpointMap, cone[p], 0));
2559     /* Negative face */
2560     PetscCall(DMPlexGetJoin(dm, *nFV, cone, &numCells, &cells));
2561     /* Not true in parallel
2562     PetscCheck(numCells == 2,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive cells should separate two cells"); */
2563     for (p = 0; p < numCells; ++p) {
2564       PetscCall(DMLabelSetValue(subpointMap, cells[p], 2));
2565       (*subCells)[subc++] = cells[p];
2566     }
2567     PetscCall(DMPlexRestoreJoin(dm, *nFV, cone, &numCells, &cells));
2568     /* Positive face is not included */
2569   }
2570   PetscFunctionReturn(PETSC_SUCCESS);
2571 }
2572 
2573 static PetscErrorCode DMPlexMarkCohesiveSubmesh_Interpolated(DM dm, DMLabel label, PetscInt value, DMLabel subpointMap, DM subdm)
2574 {
2575   PetscInt *pStart, *pEnd;
2576   PetscInt  dim, cMax, cEnd, c, d;
2577 
2578   PetscFunctionBegin;
2579   PetscCall(DMGetDimension(dm, &dim));
2580   PetscCall(DMPlexGetTensorPrismBounds_Internal(dm, dim, &cMax, &cEnd));
2581   if (cMax < 0) PetscFunctionReturn(PETSC_SUCCESS);
2582   PetscCall(PetscMalloc2(dim + 1, &pStart, dim + 1, &pEnd));
2583   for (d = 0; d <= dim; ++d) PetscCall(DMPlexGetDepthStratum(dm, d, &pStart[d], &pEnd[d]));
2584   for (c = cMax; c < cEnd; ++c) {
2585     const PetscInt *cone;
2586     PetscInt       *closure = NULL;
2587     PetscInt        fconeSize, coneSize, closureSize, cl, val;
2588 
2589     if (label) {
2590       PetscCall(DMLabelGetValue(label, c, &val));
2591       if (val != value) continue;
2592     }
2593     PetscCall(DMPlexGetConeSize(dm, c, &coneSize));
2594     PetscCall(DMPlexGetCone(dm, c, &cone));
2595     PetscCall(DMPlexGetConeSize(dm, cone[0], &fconeSize));
2596     PetscCheck(coneSize == (fconeSize ? fconeSize : 1) + 2, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive cells should separate two cells");
2597     /* Negative face */
2598     PetscCall(DMPlexGetTransitiveClosure(dm, cone[0], PETSC_TRUE, &closureSize, &closure));
2599     for (cl = 0; cl < closureSize * 2; cl += 2) {
2600       const PetscInt point = closure[cl];
2601 
2602       for (d = 0; d <= dim; ++d) {
2603         if ((point >= pStart[d]) && (point < pEnd[d])) {
2604           PetscCall(DMLabelSetValue(subpointMap, point, d));
2605           break;
2606         }
2607       }
2608     }
2609     PetscCall(DMPlexRestoreTransitiveClosure(dm, cone[0], PETSC_TRUE, &closureSize, &closure));
2610     /* Cells -- positive face is not included */
2611     for (cl = 0; cl < 1; ++cl) {
2612       const PetscInt *support;
2613       PetscInt        supportSize, s;
2614 
2615       PetscCall(DMPlexGetSupportSize(dm, cone[cl], &supportSize));
2616       /* PetscCheck(supportSize == 2,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive faces should separate two cells"); */
2617       PetscCall(DMPlexGetSupport(dm, cone[cl], &support));
2618       for (s = 0; s < supportSize; ++s) PetscCall(DMLabelSetValue(subpointMap, support[s], dim));
2619     }
2620   }
2621   PetscCall(PetscFree2(pStart, pEnd));
2622   PetscFunctionReturn(PETSC_SUCCESS);
2623 }
2624 
2625 static PetscErrorCode DMPlexGetFaceOrientation(DM dm, PetscInt cell, PetscInt numCorners, PetscInt indices[], PetscInt oppositeVertex, PetscInt origVertices[], PetscInt faceVertices[], PetscBool *posOriented)
2626 {
2627   MPI_Comm       comm;
2628   PetscBool      posOrient = PETSC_FALSE;
2629   const PetscInt debug     = 0;
2630   PetscInt       cellDim, faceSize, f;
2631 
2632   PetscFunctionBegin;
2633   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
2634   PetscCall(DMGetDimension(dm, &cellDim));
2635   if (debug) PetscCall(PetscPrintf(comm, "cellDim: %" PetscInt_FMT " numCorners: %" PetscInt_FMT "\n", cellDim, numCorners));
2636 
2637   if (cellDim == 1 && numCorners == 2) {
2638     /* Triangle */
2639     faceSize  = numCorners - 1;
2640     posOrient = !(oppositeVertex % 2) ? PETSC_TRUE : PETSC_FALSE;
2641   } else if (cellDim == 2 && numCorners == 3) {
2642     /* Triangle */
2643     faceSize  = numCorners - 1;
2644     posOrient = !(oppositeVertex % 2) ? PETSC_TRUE : PETSC_FALSE;
2645   } else if (cellDim == 3 && numCorners == 4) {
2646     /* Tetrahedron */
2647     faceSize  = numCorners - 1;
2648     posOrient = (oppositeVertex % 2) ? PETSC_TRUE : PETSC_FALSE;
2649   } else if (cellDim == 1 && numCorners == 3) {
2650     /* Quadratic line */
2651     faceSize  = 1;
2652     posOrient = PETSC_TRUE;
2653   } else if (cellDim == 2 && numCorners == 4) {
2654     /* Quads */
2655     faceSize = 2;
2656     if ((indices[1] > indices[0]) && (indices[1] - indices[0] == 1)) {
2657       posOrient = PETSC_TRUE;
2658     } else if ((indices[0] == 3) && (indices[1] == 0)) {
2659       posOrient = PETSC_TRUE;
2660     } else {
2661       if (((indices[0] > indices[1]) && (indices[0] - indices[1] == 1)) || ((indices[0] == 0) && (indices[1] == 3))) {
2662         posOrient = PETSC_FALSE;
2663       } else SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid quad crossedge");
2664     }
2665   } else if (cellDim == 2 && numCorners == 6) {
2666     /* Quadratic triangle (I hate this) */
2667     /* Edges are determined by the first 2 vertices (corners of edges) */
2668     const PetscInt faceSizeTri = 3;
2669     PetscInt       sortedIndices[3], i, iFace;
2670     PetscBool      found                    = PETSC_FALSE;
2671     PetscInt       faceVerticesTriSorted[9] = {
2672       0, 3, 4, /* bottom */
2673       1, 4, 5, /* right */
2674       2, 3, 5, /* left */
2675     };
2676     PetscInt faceVerticesTri[9] = {
2677       0, 3, 4, /* bottom */
2678       1, 4, 5, /* right */
2679       2, 5, 3, /* left */
2680     };
2681 
2682     for (i = 0; i < faceSizeTri; ++i) sortedIndices[i] = indices[i];
2683     PetscCall(PetscSortInt(faceSizeTri, sortedIndices));
2684     for (iFace = 0; iFace < 3; ++iFace) {
2685       const PetscInt ii = iFace * faceSizeTri;
2686       PetscInt       fVertex, cVertex;
2687 
2688       if ((sortedIndices[0] == faceVerticesTriSorted[ii + 0]) && (sortedIndices[1] == faceVerticesTriSorted[ii + 1])) {
2689         for (fVertex = 0; fVertex < faceSizeTri; ++fVertex) {
2690           for (cVertex = 0; cVertex < faceSizeTri; ++cVertex) {
2691             if (indices[cVertex] == faceVerticesTri[ii + fVertex]) {
2692               faceVertices[fVertex] = origVertices[cVertex];
2693               break;
2694             }
2695           }
2696         }
2697         found = PETSC_TRUE;
2698         break;
2699       }
2700     }
2701     PetscCheck(found, comm, PETSC_ERR_ARG_WRONG, "Invalid tri crossface");
2702     if (posOriented) *posOriented = PETSC_TRUE;
2703     PetscFunctionReturn(PETSC_SUCCESS);
2704   } else if (cellDim == 2 && numCorners == 9) {
2705     /* Quadratic quad (I hate this) */
2706     /* Edges are determined by the first 2 vertices (corners of edges) */
2707     const PetscInt faceSizeQuad = 3;
2708     PetscInt       sortedIndices[3], i, iFace;
2709     PetscBool      found                      = PETSC_FALSE;
2710     PetscInt       faceVerticesQuadSorted[12] = {
2711       0, 1, 4, /* bottom */
2712       1, 2, 5, /* right */
2713       2, 3, 6, /* top */
2714       0, 3, 7, /* left */
2715     };
2716     PetscInt faceVerticesQuad[12] = {
2717       0, 1, 4, /* bottom */
2718       1, 2, 5, /* right */
2719       2, 3, 6, /* top */
2720       3, 0, 7, /* left */
2721     };
2722 
2723     for (i = 0; i < faceSizeQuad; ++i) sortedIndices[i] = indices[i];
2724     PetscCall(PetscSortInt(faceSizeQuad, sortedIndices));
2725     for (iFace = 0; iFace < 4; ++iFace) {
2726       const PetscInt ii = iFace * faceSizeQuad;
2727       PetscInt       fVertex, cVertex;
2728 
2729       if ((sortedIndices[0] == faceVerticesQuadSorted[ii + 0]) && (sortedIndices[1] == faceVerticesQuadSorted[ii + 1])) {
2730         for (fVertex = 0; fVertex < faceSizeQuad; ++fVertex) {
2731           for (cVertex = 0; cVertex < faceSizeQuad; ++cVertex) {
2732             if (indices[cVertex] == faceVerticesQuad[ii + fVertex]) {
2733               faceVertices[fVertex] = origVertices[cVertex];
2734               break;
2735             }
2736           }
2737         }
2738         found = PETSC_TRUE;
2739         break;
2740       }
2741     }
2742     PetscCheck(found, comm, PETSC_ERR_ARG_WRONG, "Invalid quad crossface");
2743     if (posOriented) *posOriented = PETSC_TRUE;
2744     PetscFunctionReturn(PETSC_SUCCESS);
2745   } else if (cellDim == 3 && numCorners == 8) {
2746     /* Hexes
2747        A hex is two oriented quads with the normal of the first
2748        pointing up at the second.
2749 
2750           7---6
2751          /|  /|
2752         4---5 |
2753         | 1-|-2
2754         |/  |/
2755         0---3
2756 
2757         Faces are determined by the first 4 vertices (corners of faces) */
2758     const PetscInt faceSizeHex = 4;
2759     PetscInt       sortedIndices[4], i, iFace;
2760     PetscBool      found                     = PETSC_FALSE;
2761     PetscInt       faceVerticesHexSorted[24] = {
2762       0, 1, 2, 3, /* bottom */
2763       4, 5, 6, 7, /* top */
2764       0, 3, 4, 5, /* front */
2765       2, 3, 5, 6, /* right */
2766       1, 2, 6, 7, /* back */
2767       0, 1, 4, 7, /* left */
2768     };
2769     PetscInt faceVerticesHex[24] = {
2770       1, 2, 3, 0, /* bottom */
2771       4, 5, 6, 7, /* top */
2772       0, 3, 5, 4, /* front */
2773       3, 2, 6, 5, /* right */
2774       2, 1, 7, 6, /* back */
2775       1, 0, 4, 7, /* left */
2776     };
2777 
2778     for (i = 0; i < faceSizeHex; ++i) sortedIndices[i] = indices[i];
2779     PetscCall(PetscSortInt(faceSizeHex, sortedIndices));
2780     for (iFace = 0; iFace < 6; ++iFace) {
2781       const PetscInt ii = iFace * faceSizeHex;
2782       PetscInt       fVertex, cVertex;
2783 
2784       if ((sortedIndices[0] == faceVerticesHexSorted[ii + 0]) && (sortedIndices[1] == faceVerticesHexSorted[ii + 1]) && (sortedIndices[2] == faceVerticesHexSorted[ii + 2]) && (sortedIndices[3] == faceVerticesHexSorted[ii + 3])) {
2785         for (fVertex = 0; fVertex < faceSizeHex; ++fVertex) {
2786           for (cVertex = 0; cVertex < faceSizeHex; ++cVertex) {
2787             if (indices[cVertex] == faceVerticesHex[ii + fVertex]) {
2788               faceVertices[fVertex] = origVertices[cVertex];
2789               break;
2790             }
2791           }
2792         }
2793         found = PETSC_TRUE;
2794         break;
2795       }
2796     }
2797     PetscCheck(found, comm, PETSC_ERR_ARG_WRONG, "Invalid hex crossface");
2798     if (posOriented) *posOriented = PETSC_TRUE;
2799     PetscFunctionReturn(PETSC_SUCCESS);
2800   } else if (cellDim == 3 && numCorners == 10) {
2801     /* Quadratic tet */
2802     /* Faces are determined by the first 3 vertices (corners of faces) */
2803     const PetscInt faceSizeTet = 6;
2804     PetscInt       sortedIndices[6], i, iFace;
2805     PetscBool      found                     = PETSC_FALSE;
2806     PetscInt       faceVerticesTetSorted[24] = {
2807       0, 1, 2, 6, 7, 8, /* bottom */
2808       0, 3, 4, 6, 7, 9, /* front */
2809       1, 4, 5, 7, 8, 9, /* right */
2810       2, 3, 5, 6, 8, 9, /* left */
2811     };
2812     PetscInt faceVerticesTet[24] = {
2813       0, 1, 2, 6, 7, 8, /* bottom */
2814       0, 4, 3, 6, 7, 9, /* front */
2815       1, 5, 4, 7, 8, 9, /* right */
2816       2, 3, 5, 8, 6, 9, /* left */
2817     };
2818 
2819     for (i = 0; i < faceSizeTet; ++i) sortedIndices[i] = indices[i];
2820     PetscCall(PetscSortInt(faceSizeTet, sortedIndices));
2821     for (iFace = 0; iFace < 4; ++iFace) {
2822       const PetscInt ii = iFace * faceSizeTet;
2823       PetscInt       fVertex, cVertex;
2824 
2825       if ((sortedIndices[0] == faceVerticesTetSorted[ii + 0]) && (sortedIndices[1] == faceVerticesTetSorted[ii + 1]) && (sortedIndices[2] == faceVerticesTetSorted[ii + 2]) && (sortedIndices[3] == faceVerticesTetSorted[ii + 3])) {
2826         for (fVertex = 0; fVertex < faceSizeTet; ++fVertex) {
2827           for (cVertex = 0; cVertex < faceSizeTet; ++cVertex) {
2828             if (indices[cVertex] == faceVerticesTet[ii + fVertex]) {
2829               faceVertices[fVertex] = origVertices[cVertex];
2830               break;
2831             }
2832           }
2833         }
2834         found = PETSC_TRUE;
2835         break;
2836       }
2837     }
2838     PetscCheck(found, comm, PETSC_ERR_ARG_WRONG, "Invalid tet crossface");
2839     if (posOriented) *posOriented = PETSC_TRUE;
2840     PetscFunctionReturn(PETSC_SUCCESS);
2841   } else if (cellDim == 3 && numCorners == 27) {
2842     /* Quadratic hexes (I hate this)
2843        A hex is two oriented quads with the normal of the first
2844        pointing up at the second.
2845 
2846          7---6
2847         /|  /|
2848        4---5 |
2849        | 3-|-2
2850        |/  |/
2851        0---1
2852 
2853        Faces are determined by the first 4 vertices (corners of faces) */
2854     const PetscInt faceSizeQuadHex = 9;
2855     PetscInt       sortedIndices[9], i, iFace;
2856     PetscBool      found                         = PETSC_FALSE;
2857     PetscInt       faceVerticesQuadHexSorted[54] = {
2858       0, 1, 2, 3, 8,  9,  10, 11, 24, /* bottom */
2859       4, 5, 6, 7, 12, 13, 14, 15, 25, /* top */
2860       0, 1, 4, 5, 8,  12, 16, 17, 22, /* front */
2861       1, 2, 5, 6, 9,  13, 17, 18, 21, /* right */
2862       2, 3, 6, 7, 10, 14, 18, 19, 23, /* back */
2863       0, 3, 4, 7, 11, 15, 16, 19, 20, /* left */
2864     };
2865     PetscInt faceVerticesQuadHex[54] = {
2866       3, 2, 1, 0, 10, 9,  8,  11, 24, /* bottom */
2867       4, 5, 6, 7, 12, 13, 14, 15, 25, /* top */
2868       0, 1, 5, 4, 8,  17, 12, 16, 22, /* front */
2869       1, 2, 6, 5, 9,  18, 13, 17, 21, /* right */
2870       2, 3, 7, 6, 10, 19, 14, 18, 23, /* back */
2871       3, 0, 4, 7, 11, 16, 15, 19, 20  /* left */
2872     };
2873 
2874     for (i = 0; i < faceSizeQuadHex; ++i) sortedIndices[i] = indices[i];
2875     PetscCall(PetscSortInt(faceSizeQuadHex, sortedIndices));
2876     for (iFace = 0; iFace < 6; ++iFace) {
2877       const PetscInt ii = iFace * faceSizeQuadHex;
2878       PetscInt       fVertex, cVertex;
2879 
2880       if ((sortedIndices[0] == faceVerticesQuadHexSorted[ii + 0]) && (sortedIndices[1] == faceVerticesQuadHexSorted[ii + 1]) && (sortedIndices[2] == faceVerticesQuadHexSorted[ii + 2]) && (sortedIndices[3] == faceVerticesQuadHexSorted[ii + 3])) {
2881         for (fVertex = 0; fVertex < faceSizeQuadHex; ++fVertex) {
2882           for (cVertex = 0; cVertex < faceSizeQuadHex; ++cVertex) {
2883             if (indices[cVertex] == faceVerticesQuadHex[ii + fVertex]) {
2884               faceVertices[fVertex] = origVertices[cVertex];
2885               break;
2886             }
2887           }
2888         }
2889         found = PETSC_TRUE;
2890         break;
2891       }
2892     }
2893     PetscCheck(found, comm, PETSC_ERR_ARG_WRONG, "Invalid hex crossface");
2894     if (posOriented) *posOriented = PETSC_TRUE;
2895     PetscFunctionReturn(PETSC_SUCCESS);
2896   } else SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Unknown cell type for faceOrientation().");
2897   if (!posOrient) {
2898     if (debug) PetscCall(PetscPrintf(comm, "  Reversing initial face orientation\n"));
2899     for (f = 0; f < faceSize; ++f) faceVertices[f] = origVertices[faceSize - 1 - f];
2900   } else {
2901     if (debug) PetscCall(PetscPrintf(comm, "  Keeping initial face orientation\n"));
2902     for (f = 0; f < faceSize; ++f) faceVertices[f] = origVertices[f];
2903   }
2904   if (posOriented) *posOriented = posOrient;
2905   PetscFunctionReturn(PETSC_SUCCESS);
2906 }
2907 
2908 /*@
2909   DMPlexGetOrientedFace - Given a cell and a face, as a set of vertices, return the oriented face, as a set of vertices,
2910   in faceVertices. The orientation is such that the face normal points out of the cell
2911 
2912   Not Collective
2913 
2914   Input Parameters:
2915 + dm           - The original mesh
2916 . cell         - The cell mesh point
2917 . faceSize     - The number of vertices on the face
2918 . face         - The face vertices
2919 . numCorners   - The number of vertices on the cell
2920 . indices      - Local numbering of face vertices in cell cone
2921 - origVertices - Original face vertices
2922 
2923   Output Parameters:
2924 + faceVertices - The face vertices properly oriented
2925 - posOriented  - `PETSC_TRUE` if the face was oriented with outward normal
2926 
2927   Level: developer
2928 
2929 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexGetCone()`
2930 @*/
2931 PetscErrorCode DMPlexGetOrientedFace(DM dm, PetscInt cell, PetscInt faceSize, const PetscInt face[], PetscInt numCorners, PetscInt indices[], PetscInt origVertices[], PetscInt faceVertices[], PetscBool *posOriented)
2932 {
2933   const PetscInt *cone = NULL;
2934   PetscInt        coneSize, v, f, v2;
2935   PetscInt        oppositeVertex = -1;
2936 
2937   PetscFunctionBegin;
2938   PetscCall(DMPlexGetConeSize(dm, cell, &coneSize));
2939   PetscCall(DMPlexGetCone(dm, cell, &cone));
2940   for (v = 0, v2 = 0; v < coneSize; ++v) {
2941     PetscBool found = PETSC_FALSE;
2942 
2943     for (f = 0; f < faceSize; ++f) {
2944       if (face[f] == cone[v]) {
2945         found = PETSC_TRUE;
2946         break;
2947       }
2948     }
2949     if (found) {
2950       indices[v2]      = v;
2951       origVertices[v2] = cone[v];
2952       ++v2;
2953     } else {
2954       oppositeVertex = v;
2955     }
2956   }
2957   PetscCall(DMPlexGetFaceOrientation(dm, cell, numCorners, indices, oppositeVertex, origVertices, faceVertices, posOriented));
2958   PetscFunctionReturn(PETSC_SUCCESS);
2959 }
2960 
2961 /*
2962   DMPlexInsertFace_Internal - Puts a face into the mesh
2963 
2964   Not Collective
2965 
2966   Input Parameters:
2967   + dm              - The `DMPLEX`
2968   . numFaceVertex   - The number of vertices in the face
2969   . faceVertices    - The vertices in the face for `dm`
2970   . subfaceVertices - The vertices in the face for subdm
2971   . numCorners      - The number of vertices in the `cell`
2972   . cell            - A cell in `dm` containing the face
2973   . subcell         - A cell in subdm containing the face
2974   . firstFace       - First face in the mesh
2975   - newFacePoint    - Next face in the mesh
2976 
2977   Output Parameters:
2978   . newFacePoint - Contains next face point number on input, updated on output
2979 
2980   Level: developer
2981 */
2982 static PetscErrorCode DMPlexInsertFace_Internal(DM dm, DM subdm, PetscInt numFaceVertices, const PetscInt faceVertices[], const PetscInt subfaceVertices[], PetscInt numCorners, PetscInt cell, PetscInt subcell, PetscInt firstFace, PetscInt *newFacePoint)
2983 {
2984   MPI_Comm        comm;
2985   DM_Plex        *submesh = (DM_Plex *)subdm->data;
2986   const PetscInt *faces;
2987   PetscInt        numFaces, coneSize;
2988 
2989   PetscFunctionBegin;
2990   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
2991   PetscCall(DMPlexGetConeSize(subdm, subcell, &coneSize));
2992   PetscCheck(coneSize == 1, comm, PETSC_ERR_ARG_OUTOFRANGE, "Cone size of cell %" PetscInt_FMT " is %" PetscInt_FMT " != 1", cell, coneSize);
2993 #if 0
2994   /* Cannot use this because support() has not been constructed yet */
2995   PetscCall(DMPlexGetJoin(subdm, numFaceVertices, subfaceVertices, &numFaces, &faces));
2996 #else
2997   {
2998     PetscInt f;
2999 
3000     numFaces = 0;
3001     PetscCall(DMGetWorkArray(subdm, 1, MPIU_INT, (void **)&faces));
3002     for (f = firstFace; f < *newFacePoint; ++f) {
3003       PetscInt dof, off, d;
3004 
3005       PetscCall(PetscSectionGetDof(submesh->coneSection, f, &dof));
3006       PetscCall(PetscSectionGetOffset(submesh->coneSection, f, &off));
3007       /* Yes, I know this is quadratic, but I expect the sizes to be <5 */
3008       for (d = 0; d < dof; ++d) {
3009         const PetscInt p = submesh->cones[off + d];
3010         PetscInt       v;
3011 
3012         for (v = 0; v < numFaceVertices; ++v) {
3013           if (subfaceVertices[v] == p) break;
3014         }
3015         if (v == numFaceVertices) break;
3016       }
3017       if (d == dof) {
3018         numFaces               = 1;
3019         ((PetscInt *)faces)[0] = f;
3020       }
3021     }
3022   }
3023 #endif
3024   PetscCheck(numFaces <= 1, comm, PETSC_ERR_ARG_WRONG, "Vertex set had %" PetscInt_FMT " faces, not one", numFaces);
3025   if (numFaces == 1) {
3026     /* Add the other cell neighbor for this face */
3027     PetscCall(DMPlexSetCone(subdm, subcell, faces));
3028   } else {
3029     PetscInt *indices, *origVertices, *orientedVertices, *orientedSubVertices, v, ov;
3030     PetscBool posOriented;
3031 
3032     PetscCall(DMGetWorkArray(subdm, 4 * numFaceVertices * sizeof(PetscInt), MPIU_INT, &orientedVertices));
3033     origVertices        = &orientedVertices[numFaceVertices];
3034     indices             = &orientedVertices[numFaceVertices * 2];
3035     orientedSubVertices = &orientedVertices[numFaceVertices * 3];
3036     PetscCall(DMPlexGetOrientedFace(dm, cell, numFaceVertices, faceVertices, numCorners, indices, origVertices, orientedVertices, &posOriented));
3037     /* TODO: I know that routine should return a permutation, not the indices */
3038     for (v = 0; v < numFaceVertices; ++v) {
3039       const PetscInt vertex = faceVertices[v], subvertex = subfaceVertices[v];
3040       for (ov = 0; ov < numFaceVertices; ++ov) {
3041         if (orientedVertices[ov] == vertex) {
3042           orientedSubVertices[ov] = subvertex;
3043           break;
3044         }
3045       }
3046       PetscCheck(ov != numFaceVertices, comm, PETSC_ERR_PLIB, "Could not find face vertex %" PetscInt_FMT " in orientated set", vertex);
3047     }
3048     PetscCall(DMPlexSetCone(subdm, *newFacePoint, orientedSubVertices));
3049     PetscCall(DMPlexSetCone(subdm, subcell, newFacePoint));
3050     PetscCall(DMRestoreWorkArray(subdm, 4 * numFaceVertices * sizeof(PetscInt), MPIU_INT, &orientedVertices));
3051     ++(*newFacePoint);
3052   }
3053 #if 0
3054   PetscCall(DMPlexRestoreJoin(subdm, numFaceVertices, subfaceVertices, &numFaces, &faces));
3055 #else
3056   PetscCall(DMRestoreWorkArray(subdm, 1, MPIU_INT, (void **)&faces));
3057 #endif
3058   PetscFunctionReturn(PETSC_SUCCESS);
3059 }
3060 
3061 static PetscErrorCode DMPlexCreateSubmesh_Uninterpolated(DM dm, DMLabel vertexLabel, PetscInt value, DM subdm)
3062 {
3063   MPI_Comm        comm;
3064   DMLabel         subpointMap;
3065   IS              subvertexIS, subcellIS;
3066   const PetscInt *subVertices, *subCells;
3067   PetscInt        numSubVertices, firstSubVertex, numSubCells;
3068   PetscInt       *subface, maxConeSize, numSubFaces = 0, firstSubFace, newFacePoint, nFV = 0;
3069   PetscInt        vStart, vEnd, c, f;
3070 
3071   PetscFunctionBegin;
3072   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
3073   /* Create subpointMap which marks the submesh */
3074   PetscCall(DMLabelCreate(PETSC_COMM_SELF, "subpoint_map", &subpointMap));
3075   PetscCall(DMPlexSetSubpointMap(subdm, subpointMap));
3076   PetscCall(DMLabelDestroy(&subpointMap));
3077   if (vertexLabel) PetscCall(DMPlexMarkSubmesh_Uninterpolated(dm, vertexLabel, value, subpointMap, &numSubFaces, &nFV, subdm));
3078   /* Setup chart */
3079   PetscCall(DMLabelGetStratumSize(subpointMap, 0, &numSubVertices));
3080   PetscCall(DMLabelGetStratumSize(subpointMap, 2, &numSubCells));
3081   PetscCall(DMPlexSetChart(subdm, 0, numSubCells + numSubFaces + numSubVertices));
3082   PetscCall(DMPlexSetVTKCellHeight(subdm, 1));
3083   /* Set cone sizes */
3084   firstSubVertex = numSubCells;
3085   firstSubFace   = numSubCells + numSubVertices;
3086   newFacePoint   = firstSubFace;
3087   PetscCall(DMLabelGetStratumIS(subpointMap, 0, &subvertexIS));
3088   if (subvertexIS) PetscCall(ISGetIndices(subvertexIS, &subVertices));
3089   PetscCall(DMLabelGetStratumIS(subpointMap, 2, &subcellIS));
3090   if (subcellIS) PetscCall(ISGetIndices(subcellIS, &subCells));
3091   for (c = 0; c < numSubCells; ++c) PetscCall(DMPlexSetConeSize(subdm, c, 1));
3092   for (f = firstSubFace; f < firstSubFace + numSubFaces; ++f) PetscCall(DMPlexSetConeSize(subdm, f, nFV));
3093   PetscCall(DMSetUp(subdm));
3094   /* Create face cones */
3095   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
3096   PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, NULL));
3097   PetscCall(DMGetWorkArray(subdm, maxConeSize, MPIU_INT, (void **)&subface));
3098   for (c = 0; c < numSubCells; ++c) {
3099     const PetscInt cell    = subCells[c];
3100     const PetscInt subcell = c;
3101     PetscInt      *closure = NULL;
3102     PetscInt       closureSize, cl, numCorners = 0, faceSize = 0;
3103 
3104     PetscCall(DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
3105     for (cl = 0; cl < closureSize * 2; cl += 2) {
3106       const PetscInt point = closure[cl];
3107       PetscInt       subVertex;
3108 
3109       if ((point >= vStart) && (point < vEnd)) {
3110         ++numCorners;
3111         PetscCall(PetscFindInt(point, numSubVertices, subVertices, &subVertex));
3112         if (subVertex >= 0) {
3113           closure[faceSize] = point;
3114           subface[faceSize] = firstSubVertex + subVertex;
3115           ++faceSize;
3116         }
3117       }
3118     }
3119     PetscCheck(faceSize <= nFV, comm, PETSC_ERR_ARG_WRONG, "Invalid submesh: Too many vertices %" PetscInt_FMT " of an element on the surface", faceSize);
3120     if (faceSize == nFV) PetscCall(DMPlexInsertFace_Internal(dm, subdm, faceSize, closure, subface, numCorners, cell, subcell, firstSubFace, &newFacePoint));
3121     PetscCall(DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
3122   }
3123   PetscCall(DMRestoreWorkArray(subdm, maxConeSize, MPIU_INT, (void **)&subface));
3124   PetscCall(DMPlexSymmetrize(subdm));
3125   PetscCall(DMPlexStratify(subdm));
3126   /* Build coordinates */
3127   {
3128     PetscSection coordSection, subCoordSection;
3129     Vec          coordinates, subCoordinates;
3130     PetscScalar *coords, *subCoords;
3131     PetscInt     numComp, coordSize, v;
3132     const char  *name;
3133 
3134     PetscCall(DMGetCoordinateSection(dm, &coordSection));
3135     PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
3136     PetscCall(DMGetCoordinateSection(subdm, &subCoordSection));
3137     PetscCall(PetscSectionSetNumFields(subCoordSection, 1));
3138     PetscCall(PetscSectionGetFieldComponents(coordSection, 0, &numComp));
3139     PetscCall(PetscSectionSetFieldComponents(subCoordSection, 0, numComp));
3140     PetscCall(PetscSectionSetChart(subCoordSection, firstSubVertex, firstSubVertex + numSubVertices));
3141     for (v = 0; v < numSubVertices; ++v) {
3142       const PetscInt vertex    = subVertices[v];
3143       const PetscInt subvertex = firstSubVertex + v;
3144       PetscInt       dof;
3145 
3146       PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3147       PetscCall(PetscSectionSetDof(subCoordSection, subvertex, dof));
3148       PetscCall(PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof));
3149     }
3150     PetscCall(PetscSectionSetUp(subCoordSection));
3151     PetscCall(PetscSectionGetStorageSize(subCoordSection, &coordSize));
3152     PetscCall(VecCreate(PETSC_COMM_SELF, &subCoordinates));
3153     PetscCall(PetscObjectGetName((PetscObject)coordinates, &name));
3154     PetscCall(PetscObjectSetName((PetscObject)subCoordinates, name));
3155     PetscCall(VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE));
3156     PetscCall(VecSetType(subCoordinates, VECSTANDARD));
3157     if (coordSize) {
3158       PetscCall(VecGetArray(coordinates, &coords));
3159       PetscCall(VecGetArray(subCoordinates, &subCoords));
3160       for (v = 0; v < numSubVertices; ++v) {
3161         const PetscInt vertex    = subVertices[v];
3162         const PetscInt subvertex = firstSubVertex + v;
3163         PetscInt       dof, off, sdof, soff, d;
3164 
3165         PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3166         PetscCall(PetscSectionGetOffset(coordSection, vertex, &off));
3167         PetscCall(PetscSectionGetDof(subCoordSection, subvertex, &sdof));
3168         PetscCall(PetscSectionGetOffset(subCoordSection, subvertex, &soff));
3169         PetscCheck(dof == sdof, comm, PETSC_ERR_PLIB, "Coordinate dimension %" PetscInt_FMT " on subvertex %" PetscInt_FMT ", vertex %" PetscInt_FMT " should be %" PetscInt_FMT, sdof, subvertex, vertex, dof);
3170         for (d = 0; d < dof; ++d) subCoords[soff + d] = coords[off + d];
3171       }
3172       PetscCall(VecRestoreArray(coordinates, &coords));
3173       PetscCall(VecRestoreArray(subCoordinates, &subCoords));
3174     }
3175     PetscCall(DMSetCoordinatesLocal(subdm, subCoordinates));
3176     PetscCall(VecDestroy(&subCoordinates));
3177   }
3178   /* Cleanup */
3179   if (subvertexIS) PetscCall(ISRestoreIndices(subvertexIS, &subVertices));
3180   PetscCall(ISDestroy(&subvertexIS));
3181   if (subcellIS) PetscCall(ISRestoreIndices(subcellIS, &subCells));
3182   PetscCall(ISDestroy(&subcellIS));
3183   PetscFunctionReturn(PETSC_SUCCESS);
3184 }
3185 
3186 /* TODO: Fix this to properly propagate up error conditions it may find */
3187 static inline PetscInt DMPlexFilterPoint_Internal(PetscInt point, PetscInt firstSubPoint, PetscInt numSubPoints, const PetscInt subPoints[])
3188 {
3189   PetscInt       subPoint;
3190   PetscErrorCode ierr;
3191 
3192   ierr = PetscFindInt(point, numSubPoints, subPoints, &subPoint);
3193   if (ierr) return -1;
3194   return subPoint < 0 ? subPoint : firstSubPoint + subPoint;
3195 }
3196 
3197 /* TODO: Fix this to properly propagate up error conditions it may find */
3198 static inline PetscInt DMPlexFilterPointPerm_Internal(PetscInt point, PetscInt firstSubPoint, PetscInt numSubPoints, const PetscInt subPoints[], const PetscInt subIndices[])
3199 {
3200   PetscInt       subPoint;
3201   PetscErrorCode ierr;
3202 
3203   ierr = PetscFindInt(point, numSubPoints, subPoints, &subPoint);
3204   if (ierr) return -1;
3205   return subPoint < 0 ? subPoint : firstSubPoint + (subIndices ? subIndices[subPoint] : subPoint);
3206 }
3207 
3208 static PetscErrorCode DMPlexFilterLabels_Internal(DM dm, const PetscInt numSubPoints[], const PetscInt *subpoints[], const PetscInt firstSubPoint[], DM subdm)
3209 {
3210   DMLabel  depthLabel;
3211   PetscInt Nl, l, d;
3212 
3213   PetscFunctionBegin;
3214   // Reset depth label for fast lookup
3215   PetscCall(DMPlexGetDepthLabel(dm, &depthLabel));
3216   PetscCall(DMLabelMakeAllInvalid_Internal(depthLabel));
3217   PetscCall(DMGetNumLabels(dm, &Nl));
3218   for (l = 0; l < Nl; ++l) {
3219     DMLabel         label, newlabel;
3220     const char     *lname;
3221     PetscBool       isDepth, isDim, isCelltype, isVTK;
3222     IS              valueIS;
3223     const PetscInt *values;
3224     PetscInt        Nv, v;
3225 
3226     PetscCall(DMGetLabelName(dm, l, &lname));
3227     PetscCall(PetscStrcmp(lname, "depth", &isDepth));
3228     PetscCall(PetscStrcmp(lname, "dim", &isDim));
3229     PetscCall(PetscStrcmp(lname, "celltype", &isCelltype));
3230     PetscCall(PetscStrcmp(lname, "vtk", &isVTK));
3231     if (isDepth || isDim || isCelltype || isVTK) continue;
3232     PetscCall(DMCreateLabel(subdm, lname));
3233     PetscCall(DMGetLabel(dm, lname, &label));
3234     PetscCall(DMGetLabel(subdm, lname, &newlabel));
3235     PetscCall(DMLabelGetDefaultValue(label, &v));
3236     PetscCall(DMLabelSetDefaultValue(newlabel, v));
3237     PetscCall(DMLabelGetValueIS(label, &valueIS));
3238     PetscCall(ISGetLocalSize(valueIS, &Nv));
3239     PetscCall(ISGetIndices(valueIS, &values));
3240     for (v = 0; v < Nv; ++v) {
3241       IS              pointIS;
3242       const PetscInt *points;
3243       PetscInt        Np, p;
3244 
3245       PetscCall(DMLabelGetStratumIS(label, values[v], &pointIS));
3246       PetscCall(ISGetLocalSize(pointIS, &Np));
3247       PetscCall(ISGetIndices(pointIS, &points));
3248       for (p = 0; p < Np; ++p) {
3249         const PetscInt point = points[p];
3250         PetscInt       subp;
3251 
3252         PetscCall(DMPlexGetPointDepth(dm, point, &d));
3253         subp = DMPlexFilterPoint_Internal(point, firstSubPoint[d], numSubPoints[d], subpoints[d]);
3254         if (subp >= 0) PetscCall(DMLabelSetValue(newlabel, subp, values[v]));
3255       }
3256       PetscCall(ISRestoreIndices(pointIS, &points));
3257       PetscCall(ISDestroy(&pointIS));
3258     }
3259     PetscCall(ISRestoreIndices(valueIS, &values));
3260     PetscCall(ISDestroy(&valueIS));
3261   }
3262   PetscFunctionReturn(PETSC_SUCCESS);
3263 }
3264 
3265 static PetscErrorCode DMPlexCreateSubmeshGeneric_Interpolated(DM dm, DMLabel label, PetscInt value, PetscBool markedFaces, PetscBool isCohesive, PetscInt cellHeight, DM subdm)
3266 {
3267   MPI_Comm         comm;
3268   DMLabel          subpointMap;
3269   IS              *subpointIS;
3270   const PetscInt **subpoints;
3271   PetscInt        *numSubPoints, *firstSubPoint, *coneNew, *orntNew;
3272   PetscInt         totSubPoints = 0, maxConeSize, dim, sdim, cdim, p, d, v;
3273   PetscMPIInt      rank;
3274 
3275   PetscFunctionBegin;
3276   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
3277   PetscCallMPI(MPI_Comm_rank(comm, &rank));
3278   /* Create subpointMap which marks the submesh */
3279   PetscCall(DMLabelCreate(PETSC_COMM_SELF, "subpoint_map", &subpointMap));
3280   PetscCall(DMPlexSetSubpointMap(subdm, subpointMap));
3281   if (cellHeight) {
3282     if (isCohesive) PetscCall(DMPlexMarkCohesiveSubmesh_Interpolated(dm, label, value, subpointMap, subdm));
3283     else PetscCall(DMPlexMarkSubmesh_Interpolated(dm, label, value, markedFaces, subpointMap, subdm));
3284   } else {
3285     DMLabel         depth;
3286     IS              pointIS;
3287     const PetscInt *points;
3288     PetscInt        numPoints = 0;
3289 
3290     PetscCall(DMPlexGetDepthLabel(dm, &depth));
3291     PetscCall(DMLabelGetStratumIS(label, value, &pointIS));
3292     if (pointIS) {
3293       PetscCall(ISGetIndices(pointIS, &points));
3294       PetscCall(ISGetLocalSize(pointIS, &numPoints));
3295     }
3296     for (p = 0; p < numPoints; ++p) {
3297       PetscInt *closure = NULL;
3298       PetscInt  closureSize, c, pdim;
3299 
3300       PetscCall(DMPlexGetTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closure));
3301       for (c = 0; c < closureSize * 2; c += 2) {
3302         PetscCall(DMLabelGetValue(depth, closure[c], &pdim));
3303         PetscCall(DMLabelSetValue(subpointMap, closure[c], pdim));
3304       }
3305       PetscCall(DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closure));
3306     }
3307     if (pointIS) PetscCall(ISRestoreIndices(pointIS, &points));
3308     PetscCall(ISDestroy(&pointIS));
3309   }
3310   /* Setup chart */
3311   PetscCall(DMGetDimension(dm, &dim));
3312   PetscCall(DMGetCoordinateDim(dm, &cdim));
3313   PetscCall(PetscMalloc4(dim + 1, &numSubPoints, dim + 1, &firstSubPoint, dim + 1, &subpointIS, dim + 1, &subpoints));
3314   for (d = 0; d <= dim; ++d) {
3315     PetscCall(DMLabelGetStratumSize(subpointMap, d, &numSubPoints[d]));
3316     totSubPoints += numSubPoints[d];
3317   }
3318   // Determine submesh dimension
3319   PetscCall(DMGetDimension(subdm, &sdim));
3320   if (sdim > 0) {
3321     // Calling function knows what dimension to use, and we include neighboring cells as well
3322     sdim = dim;
3323   } else {
3324     // We reset the subdimension based on what is being selected
3325     PetscInt lsdim;
3326     for (lsdim = dim; lsdim >= 0; --lsdim)
3327       if (numSubPoints[lsdim]) break;
3328     PetscCallMPI(MPI_Allreduce(&lsdim, &sdim, 1, MPIU_INT, MPIU_MAX, comm));
3329     PetscCall(DMSetDimension(subdm, sdim));
3330     PetscCall(DMSetCoordinateDim(subdm, cdim));
3331   }
3332   PetscCall(DMPlexSetChart(subdm, 0, totSubPoints));
3333   PetscCall(DMPlexSetVTKCellHeight(subdm, cellHeight));
3334   /* Set cone sizes */
3335   firstSubPoint[sdim] = 0;
3336   firstSubPoint[0]    = firstSubPoint[sdim] + numSubPoints[sdim];
3337   if (sdim > 1) firstSubPoint[sdim - 1] = firstSubPoint[0] + numSubPoints[0];
3338   if (sdim > 2) firstSubPoint[sdim - 2] = firstSubPoint[sdim - 1] + numSubPoints[sdim - 1];
3339   for (d = 0; d <= sdim; ++d) {
3340     PetscCall(DMLabelGetStratumIS(subpointMap, d, &subpointIS[d]));
3341     if (subpointIS[d]) PetscCall(ISGetIndices(subpointIS[d], &subpoints[d]));
3342   }
3343   /* We do not want this label automatically computed, instead we compute it here */
3344   PetscCall(DMCreateLabel(subdm, "celltype"));
3345   for (d = 0; d <= sdim; ++d) {
3346     for (p = 0; p < numSubPoints[d]; ++p) {
3347       const PetscInt  point    = subpoints[d][p];
3348       const PetscInt  subpoint = firstSubPoint[d] + p;
3349       const PetscInt *cone;
3350       PetscInt        coneSize;
3351 
3352       PetscCall(DMPlexGetConeSize(dm, point, &coneSize));
3353       if (cellHeight && (d == sdim)) {
3354         PetscInt coneSizeNew, c, val;
3355 
3356         PetscCall(DMPlexGetCone(dm, point, &cone));
3357         for (c = 0, coneSizeNew = 0; c < coneSize; ++c) {
3358           PetscCall(DMLabelGetValue(subpointMap, cone[c], &val));
3359           if (val >= 0) coneSizeNew++;
3360         }
3361         PetscCall(DMPlexSetConeSize(subdm, subpoint, coneSizeNew));
3362         PetscCall(DMPlexSetCellType(subdm, subpoint, DM_POLYTOPE_FV_GHOST));
3363       } else {
3364         DMPolytopeType ct;
3365 
3366         PetscCall(DMPlexSetConeSize(subdm, subpoint, coneSize));
3367         PetscCall(DMPlexGetCellType(dm, point, &ct));
3368         PetscCall(DMPlexSetCellType(subdm, subpoint, ct));
3369       }
3370     }
3371   }
3372   PetscCall(DMLabelDestroy(&subpointMap));
3373   PetscCall(DMSetUp(subdm));
3374   /* Set cones */
3375   PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, NULL));
3376   PetscCall(PetscMalloc2(maxConeSize, &coneNew, maxConeSize, &orntNew));
3377   for (d = 0; d <= sdim; ++d) {
3378     for (p = 0; p < numSubPoints[d]; ++p) {
3379       const PetscInt  point    = subpoints[d][p];
3380       const PetscInt  subpoint = firstSubPoint[d] + p;
3381       const PetscInt *cone, *ornt;
3382       PetscInt        coneSize, subconeSize, coneSizeNew, c, subc, fornt = 0;
3383 
3384       if (d == sdim - 1) {
3385         const PetscInt *support, *cone, *ornt;
3386         PetscInt        supportSize, coneSize, s, subc;
3387 
3388         PetscCall(DMPlexGetSupport(dm, point, &support));
3389         PetscCall(DMPlexGetSupportSize(dm, point, &supportSize));
3390         for (s = 0; s < supportSize; ++s) {
3391           PetscBool isHybrid = PETSC_FALSE;
3392 
3393           PetscCall(DMPlexCellIsHybrid_Internal(dm, support[s], &isHybrid));
3394           if (!isHybrid) continue;
3395           PetscCall(PetscFindInt(support[s], numSubPoints[d + 1], subpoints[d + 1], &subc));
3396           if (subc >= 0) {
3397             const PetscInt ccell = subpoints[d + 1][subc];
3398 
3399             PetscCall(DMPlexGetCone(dm, ccell, &cone));
3400             PetscCall(DMPlexGetConeSize(dm, ccell, &coneSize));
3401             PetscCall(DMPlexGetConeOrientation(dm, ccell, &ornt));
3402             for (c = 0; c < coneSize; ++c) {
3403               if (cone[c] == point) {
3404                 fornt = ornt[c];
3405                 break;
3406               }
3407             }
3408             break;
3409           }
3410         }
3411       }
3412       PetscCall(DMPlexGetConeSize(dm, point, &coneSize));
3413       PetscCall(DMPlexGetConeSize(subdm, subpoint, &subconeSize));
3414       PetscCall(DMPlexGetCone(dm, point, &cone));
3415       PetscCall(DMPlexGetConeOrientation(dm, point, &ornt));
3416       for (c = 0, coneSizeNew = 0; c < coneSize; ++c) {
3417         PetscCall(PetscFindInt(cone[c], numSubPoints[d - 1], subpoints[d - 1], &subc));
3418         if (subc >= 0) {
3419           coneNew[coneSizeNew] = firstSubPoint[d - 1] + subc;
3420           orntNew[coneSizeNew] = ornt[c];
3421           ++coneSizeNew;
3422         }
3423       }
3424       PetscCheck(coneSizeNew == subconeSize, comm, PETSC_ERR_PLIB, "Number of cone points located %" PetscInt_FMT " does not match subcone size %" PetscInt_FMT, coneSizeNew, subconeSize);
3425       PetscCall(DMPlexSetCone(subdm, subpoint, coneNew));
3426       PetscCall(DMPlexSetConeOrientation(subdm, subpoint, orntNew));
3427       if (fornt < 0) PetscCall(DMPlexOrientPoint(subdm, subpoint, fornt));
3428     }
3429   }
3430   PetscCall(PetscFree2(coneNew, orntNew));
3431   PetscCall(DMPlexSymmetrize(subdm));
3432   PetscCall(DMPlexStratify(subdm));
3433   /* Build coordinates */
3434   {
3435     PetscSection coordSection, subCoordSection;
3436     Vec          coordinates, subCoordinates;
3437     PetscScalar *coords, *subCoords;
3438     PetscInt     cdim, numComp, coordSize;
3439     const char  *name;
3440 
3441     PetscCall(DMGetCoordinateDim(dm, &cdim));
3442     PetscCall(DMGetCoordinateSection(dm, &coordSection));
3443     PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
3444     PetscCall(DMGetCoordinateSection(subdm, &subCoordSection));
3445     PetscCall(PetscSectionSetNumFields(subCoordSection, 1));
3446     PetscCall(PetscSectionGetFieldComponents(coordSection, 0, &numComp));
3447     PetscCall(PetscSectionSetFieldComponents(subCoordSection, 0, numComp));
3448     PetscCall(PetscSectionSetChart(subCoordSection, firstSubPoint[0], firstSubPoint[0] + numSubPoints[0]));
3449     for (v = 0; v < numSubPoints[0]; ++v) {
3450       const PetscInt vertex    = subpoints[0][v];
3451       const PetscInt subvertex = firstSubPoint[0] + v;
3452       PetscInt       dof;
3453 
3454       PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3455       PetscCall(PetscSectionSetDof(subCoordSection, subvertex, dof));
3456       PetscCall(PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof));
3457     }
3458     PetscCall(PetscSectionSetUp(subCoordSection));
3459     PetscCall(PetscSectionGetStorageSize(subCoordSection, &coordSize));
3460     PetscCall(VecCreate(PETSC_COMM_SELF, &subCoordinates));
3461     PetscCall(PetscObjectGetName((PetscObject)coordinates, &name));
3462     PetscCall(PetscObjectSetName((PetscObject)subCoordinates, name));
3463     PetscCall(VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE));
3464     PetscCall(VecSetBlockSize(subCoordinates, cdim));
3465     PetscCall(VecSetType(subCoordinates, VECSTANDARD));
3466     PetscCall(VecGetArray(coordinates, &coords));
3467     PetscCall(VecGetArray(subCoordinates, &subCoords));
3468     for (v = 0; v < numSubPoints[0]; ++v) {
3469       const PetscInt vertex    = subpoints[0][v];
3470       const PetscInt subvertex = firstSubPoint[0] + v;
3471       PetscInt       dof, off, sdof, soff, d;
3472 
3473       PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3474       PetscCall(PetscSectionGetOffset(coordSection, vertex, &off));
3475       PetscCall(PetscSectionGetDof(subCoordSection, subvertex, &sdof));
3476       PetscCall(PetscSectionGetOffset(subCoordSection, subvertex, &soff));
3477       PetscCheck(dof == sdof, comm, PETSC_ERR_PLIB, "Coordinate dimension %" PetscInt_FMT " on subvertex %" PetscInt_FMT ", vertex %" PetscInt_FMT " should be %" PetscInt_FMT, sdof, subvertex, vertex, dof);
3478       for (d = 0; d < dof; ++d) subCoords[soff + d] = coords[off + d];
3479     }
3480     PetscCall(VecRestoreArray(coordinates, &coords));
3481     PetscCall(VecRestoreArray(subCoordinates, &subCoords));
3482     PetscCall(DMSetCoordinatesLocal(subdm, subCoordinates));
3483     PetscCall(VecDestroy(&subCoordinates));
3484   }
3485   /* Build SF: We need this complexity because subpoints might not be selected on the owning process */
3486   {
3487     PetscSF            sfPoint, sfPointSub;
3488     IS                 subpIS;
3489     const PetscSFNode *remotePoints;
3490     PetscSFNode       *sremotePoints = NULL, *newLocalPoints = NULL, *newOwners = NULL;
3491     const PetscInt    *localPoints, *subpoints, *rootdegree;
3492     PetscInt          *slocalPoints = NULL, *sortedPoints = NULL, *sortedIndices = NULL;
3493     PetscInt           numRoots, numLeaves, numSubpoints = 0, numSubroots, numSubleaves = 0, l, sl = 0, ll = 0, pStart, pEnd, p;
3494     PetscMPIInt        rank, size;
3495 
3496     PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
3497     PetscCallMPI(MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size));
3498     PetscCall(DMGetPointSF(dm, &sfPoint));
3499     PetscCall(DMGetPointSF(subdm, &sfPointSub));
3500     PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
3501     PetscCall(DMPlexGetChart(subdm, NULL, &numSubroots));
3502     PetscCall(DMPlexGetSubpointIS(subdm, &subpIS));
3503     if (subpIS) {
3504       PetscBool sorted = PETSC_TRUE;
3505 
3506       PetscCall(ISGetIndices(subpIS, &subpoints));
3507       PetscCall(ISGetLocalSize(subpIS, &numSubpoints));
3508       for (p = 1; p < numSubpoints; ++p) sorted = sorted && (subpoints[p] >= subpoints[p - 1]) ? PETSC_TRUE : PETSC_FALSE;
3509       if (!sorted) {
3510         PetscCall(PetscMalloc2(numSubpoints, &sortedPoints, numSubpoints, &sortedIndices));
3511         for (p = 0; p < numSubpoints; ++p) sortedIndices[p] = p;
3512         PetscCall(PetscArraycpy(sortedPoints, subpoints, numSubpoints));
3513         PetscCall(PetscSortIntWithArray(numSubpoints, sortedPoints, sortedIndices));
3514       }
3515     }
3516     PetscCall(PetscSFGetGraph(sfPoint, &numRoots, &numLeaves, &localPoints, &remotePoints));
3517     if (numRoots >= 0) {
3518       PetscCall(PetscSFComputeDegreeBegin(sfPoint, &rootdegree));
3519       PetscCall(PetscSFComputeDegreeEnd(sfPoint, &rootdegree));
3520       PetscCall(PetscMalloc2(pEnd - pStart, &newLocalPoints, numRoots, &newOwners));
3521       for (p = 0; p < pEnd - pStart; ++p) {
3522         newLocalPoints[p].rank  = -2;
3523         newLocalPoints[p].index = -2;
3524       }
3525       /* Set subleaves */
3526       for (l = 0; l < numLeaves; ++l) {
3527         const PetscInt point    = localPoints[l];
3528         const PetscInt subpoint = DMPlexFilterPointPerm_Internal(point, 0, numSubpoints, sortedPoints ? sortedPoints : subpoints, sortedIndices);
3529 
3530         if (subpoint < 0) continue;
3531         newLocalPoints[point - pStart].rank  = rank;
3532         newLocalPoints[point - pStart].index = subpoint;
3533         ++numSubleaves;
3534       }
3535       /* Must put in owned subpoints */
3536       for (p = pStart; p < pEnd; ++p) {
3537         newOwners[p - pStart].rank  = -3;
3538         newOwners[p - pStart].index = -3;
3539       }
3540       for (p = 0; p < numSubpoints; ++p) {
3541         /* Hold on to currently owned points */
3542         if (rootdegree[subpoints[p] - pStart]) newOwners[subpoints[p] - pStart].rank = rank + size;
3543         else newOwners[subpoints[p] - pStart].rank = rank;
3544         newOwners[subpoints[p] - pStart].index = p;
3545       }
3546       PetscCall(PetscSFReduceBegin(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC));
3547       PetscCall(PetscSFReduceEnd(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC));
3548       for (p = pStart; p < pEnd; ++p)
3549         if (newOwners[p - pStart].rank >= size) newOwners[p - pStart].rank -= size;
3550       PetscCall(PetscSFBcastBegin(sfPoint, MPIU_2INT, newOwners, newLocalPoints, MPI_REPLACE));
3551       PetscCall(PetscSFBcastEnd(sfPoint, MPIU_2INT, newOwners, newLocalPoints, MPI_REPLACE));
3552       PetscCall(PetscMalloc1(numSubleaves, &slocalPoints));
3553       PetscCall(PetscMalloc1(numSubleaves, &sremotePoints));
3554       for (l = 0; l < numLeaves; ++l) {
3555         const PetscInt point    = localPoints[l];
3556         const PetscInt subpoint = DMPlexFilterPointPerm_Internal(point, 0, numSubpoints, sortedPoints ? sortedPoints : subpoints, sortedIndices);
3557 
3558         if (subpoint < 0) continue;
3559         if (newLocalPoints[point].rank == rank) {
3560           ++ll;
3561           continue;
3562         }
3563         slocalPoints[sl]        = subpoint;
3564         sremotePoints[sl].rank  = newLocalPoints[point].rank;
3565         sremotePoints[sl].index = newLocalPoints[point].index;
3566         PetscCheck(sremotePoints[sl].rank >= 0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote rank for local point %" PetscInt_FMT, point);
3567         PetscCheck(sremotePoints[sl].index >= 0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote subpoint for local point %" PetscInt_FMT, point);
3568         ++sl;
3569       }
3570       PetscCheck(sl + ll == numSubleaves, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatch in number of subleaves %" PetscInt_FMT " + %" PetscInt_FMT " != %" PetscInt_FMT, sl, ll, numSubleaves);
3571       PetscCall(PetscFree2(newLocalPoints, newOwners));
3572       PetscCall(PetscSFSetGraph(sfPointSub, numSubroots, sl, slocalPoints, PETSC_OWN_POINTER, sremotePoints, PETSC_OWN_POINTER));
3573     }
3574     if (subpIS) PetscCall(ISRestoreIndices(subpIS, &subpoints));
3575     PetscCall(PetscFree2(sortedPoints, sortedIndices));
3576   }
3577   /* Filter labels */
3578   PetscCall(DMPlexFilterLabels_Internal(dm, numSubPoints, subpoints, firstSubPoint, subdm));
3579   /* Cleanup */
3580   for (d = 0; d <= sdim; ++d) {
3581     if (subpointIS[d]) PetscCall(ISRestoreIndices(subpointIS[d], &subpoints[d]));
3582     PetscCall(ISDestroy(&subpointIS[d]));
3583   }
3584   PetscCall(PetscFree4(numSubPoints, firstSubPoint, subpointIS, subpoints));
3585   PetscFunctionReturn(PETSC_SUCCESS);
3586 }
3587 
3588 static PetscErrorCode DMPlexCreateSubmesh_Interpolated(DM dm, DMLabel vertexLabel, PetscInt value, PetscBool markedFaces, DM subdm)
3589 {
3590   PetscFunctionBegin;
3591   PetscCall(DMPlexCreateSubmeshGeneric_Interpolated(dm, vertexLabel, value, markedFaces, PETSC_FALSE, 1, subdm));
3592   PetscFunctionReturn(PETSC_SUCCESS);
3593 }
3594 
3595 /*@
3596   DMPlexCreateSubmesh - Extract a hypersurface from the mesh using vertices defined by a label
3597 
3598   Input Parameters:
3599 + dm           - The original mesh
3600 . vertexLabel  - The `DMLabel` marking points contained in the surface
3601 . value        - The label value to use
3602 - markedFaces  - `PETSC_TRUE` if surface faces are marked in addition to vertices, `PETSC_FALSE` if only vertices are marked
3603 
3604   Output Parameter:
3605 . subdm - The surface mesh
3606 
3607   Level: developer
3608 
3609   Note:
3610   This function produces a `DMLabel` mapping original points in the submesh to their depth. This can be obtained using `DMPlexGetSubpointMap()`.
3611 
3612 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexGetSubpointMap()`, `DMGetLabel()`, `DMLabelSetValue()`
3613 @*/
3614 PetscErrorCode DMPlexCreateSubmesh(DM dm, DMLabel vertexLabel, PetscInt value, PetscBool markedFaces, DM *subdm)
3615 {
3616   DMPlexInterpolatedFlag interpolated;
3617   PetscInt               dim, cdim;
3618 
3619   PetscFunctionBegin;
3620   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3621   PetscValidPointer(subdm, 5);
3622   PetscCall(DMGetDimension(dm, &dim));
3623   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), subdm));
3624   PetscCall(DMSetType(*subdm, DMPLEX));
3625   PetscCall(DMSetDimension(*subdm, dim - 1));
3626   PetscCall(DMGetCoordinateDim(dm, &cdim));
3627   PetscCall(DMSetCoordinateDim(*subdm, cdim));
3628   PetscCall(DMPlexIsInterpolated(dm, &interpolated));
3629   PetscCheck(interpolated != DMPLEX_INTERPOLATED_PARTIAL, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Not for partially interpolated meshes");
3630   if (interpolated) {
3631     PetscCall(DMPlexCreateSubmesh_Interpolated(dm, vertexLabel, value, markedFaces, *subdm));
3632   } else {
3633     PetscCall(DMPlexCreateSubmesh_Uninterpolated(dm, vertexLabel, value, *subdm));
3634   }
3635   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_TRUE, *subdm));
3636   PetscFunctionReturn(PETSC_SUCCESS);
3637 }
3638 
3639 static PetscErrorCode DMPlexCreateCohesiveSubmesh_Uninterpolated(DM dm, PetscBool hasLagrange, const char label[], PetscInt value, DM subdm)
3640 {
3641   MPI_Comm        comm;
3642   DMLabel         subpointMap;
3643   IS              subvertexIS;
3644   const PetscInt *subVertices;
3645   PetscInt        numSubVertices, firstSubVertex, numSubCells, *subCells = NULL;
3646   PetscInt       *subface, maxConeSize, numSubFaces, firstSubFace, newFacePoint, nFV;
3647   PetscInt        c, f;
3648 
3649   PetscFunctionBegin;
3650   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
3651   /* Create subpointMap which marks the submesh */
3652   PetscCall(DMLabelCreate(PETSC_COMM_SELF, "subpoint_map", &subpointMap));
3653   PetscCall(DMPlexSetSubpointMap(subdm, subpointMap));
3654   PetscCall(DMLabelDestroy(&subpointMap));
3655   PetscCall(DMPlexMarkCohesiveSubmesh_Uninterpolated(dm, hasLagrange, label, value, subpointMap, &numSubFaces, &nFV, &subCells, subdm));
3656   /* Setup chart */
3657   PetscCall(DMLabelGetStratumSize(subpointMap, 0, &numSubVertices));
3658   PetscCall(DMLabelGetStratumSize(subpointMap, 2, &numSubCells));
3659   PetscCall(DMPlexSetChart(subdm, 0, numSubCells + numSubFaces + numSubVertices));
3660   PetscCall(DMPlexSetVTKCellHeight(subdm, 1));
3661   /* Set cone sizes */
3662   firstSubVertex = numSubCells;
3663   firstSubFace   = numSubCells + numSubVertices;
3664   newFacePoint   = firstSubFace;
3665   PetscCall(DMLabelGetStratumIS(subpointMap, 0, &subvertexIS));
3666   if (subvertexIS) PetscCall(ISGetIndices(subvertexIS, &subVertices));
3667   for (c = 0; c < numSubCells; ++c) PetscCall(DMPlexSetConeSize(subdm, c, 1));
3668   for (f = firstSubFace; f < firstSubFace + numSubFaces; ++f) PetscCall(DMPlexSetConeSize(subdm, f, nFV));
3669   PetscCall(DMSetUp(subdm));
3670   /* Create face cones */
3671   PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, NULL));
3672   PetscCall(DMGetWorkArray(subdm, maxConeSize, MPIU_INT, (void **)&subface));
3673   for (c = 0; c < numSubCells; ++c) {
3674     const PetscInt  cell    = subCells[c];
3675     const PetscInt  subcell = c;
3676     const PetscInt *cone, *cells;
3677     PetscBool       isHybrid = PETSC_FALSE;
3678     PetscInt        numCells, subVertex, p, v;
3679 
3680     PetscCall(DMPlexCellIsHybrid_Internal(dm, cell, &isHybrid));
3681     if (!isHybrid) continue;
3682     PetscCall(DMPlexGetCone(dm, cell, &cone));
3683     for (v = 0; v < nFV; ++v) {
3684       PetscCall(PetscFindInt(cone[v], numSubVertices, subVertices, &subVertex));
3685       subface[v] = firstSubVertex + subVertex;
3686     }
3687     PetscCall(DMPlexSetCone(subdm, newFacePoint, subface));
3688     PetscCall(DMPlexSetCone(subdm, subcell, &newFacePoint));
3689     PetscCall(DMPlexGetJoin(dm, nFV, cone, &numCells, &cells));
3690     /* Not true in parallel
3691     PetscCheck(numCells == 2,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive cells should separate two cells"); */
3692     for (p = 0; p < numCells; ++p) {
3693       PetscInt  negsubcell;
3694       PetscBool isHybrid = PETSC_FALSE;
3695 
3696       PetscCall(DMPlexCellIsHybrid_Internal(dm, cells[p], &isHybrid));
3697       if (isHybrid) continue;
3698       /* I know this is a crap search */
3699       for (negsubcell = 0; negsubcell < numSubCells; ++negsubcell) {
3700         if (subCells[negsubcell] == cells[p]) break;
3701       }
3702       PetscCheck(negsubcell != numSubCells, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not find negative face neighbor for cohesive cell %" PetscInt_FMT, cell);
3703       PetscCall(DMPlexSetCone(subdm, negsubcell, &newFacePoint));
3704     }
3705     PetscCall(DMPlexRestoreJoin(dm, nFV, cone, &numCells, &cells));
3706     ++newFacePoint;
3707   }
3708   PetscCall(DMRestoreWorkArray(subdm, maxConeSize, MPIU_INT, (void **)&subface));
3709   PetscCall(DMPlexSymmetrize(subdm));
3710   PetscCall(DMPlexStratify(subdm));
3711   /* Build coordinates */
3712   {
3713     PetscSection coordSection, subCoordSection;
3714     Vec          coordinates, subCoordinates;
3715     PetscScalar *coords, *subCoords;
3716     PetscInt     cdim, numComp, coordSize, v;
3717     const char  *name;
3718 
3719     PetscCall(DMGetCoordinateDim(dm, &cdim));
3720     PetscCall(DMGetCoordinateSection(dm, &coordSection));
3721     PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
3722     PetscCall(DMGetCoordinateSection(subdm, &subCoordSection));
3723     PetscCall(PetscSectionSetNumFields(subCoordSection, 1));
3724     PetscCall(PetscSectionGetFieldComponents(coordSection, 0, &numComp));
3725     PetscCall(PetscSectionSetFieldComponents(subCoordSection, 0, numComp));
3726     PetscCall(PetscSectionSetChart(subCoordSection, firstSubVertex, firstSubVertex + numSubVertices));
3727     for (v = 0; v < numSubVertices; ++v) {
3728       const PetscInt vertex    = subVertices[v];
3729       const PetscInt subvertex = firstSubVertex + v;
3730       PetscInt       dof;
3731 
3732       PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3733       PetscCall(PetscSectionSetDof(subCoordSection, subvertex, dof));
3734       PetscCall(PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof));
3735     }
3736     PetscCall(PetscSectionSetUp(subCoordSection));
3737     PetscCall(PetscSectionGetStorageSize(subCoordSection, &coordSize));
3738     PetscCall(VecCreate(PETSC_COMM_SELF, &subCoordinates));
3739     PetscCall(PetscObjectGetName((PetscObject)coordinates, &name));
3740     PetscCall(PetscObjectSetName((PetscObject)subCoordinates, name));
3741     PetscCall(VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE));
3742     PetscCall(VecSetBlockSize(subCoordinates, cdim));
3743     PetscCall(VecSetType(subCoordinates, VECSTANDARD));
3744     PetscCall(VecGetArray(coordinates, &coords));
3745     PetscCall(VecGetArray(subCoordinates, &subCoords));
3746     for (v = 0; v < numSubVertices; ++v) {
3747       const PetscInt vertex    = subVertices[v];
3748       const PetscInt subvertex = firstSubVertex + v;
3749       PetscInt       dof, off, sdof, soff, d;
3750 
3751       PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3752       PetscCall(PetscSectionGetOffset(coordSection, vertex, &off));
3753       PetscCall(PetscSectionGetDof(subCoordSection, subvertex, &sdof));
3754       PetscCall(PetscSectionGetOffset(subCoordSection, subvertex, &soff));
3755       PetscCheck(dof == sdof, comm, PETSC_ERR_PLIB, "Coordinate dimension %" PetscInt_FMT " on subvertex %" PetscInt_FMT ", vertex %" PetscInt_FMT " should be %" PetscInt_FMT, sdof, subvertex, vertex, dof);
3756       for (d = 0; d < dof; ++d) subCoords[soff + d] = coords[off + d];
3757     }
3758     PetscCall(VecRestoreArray(coordinates, &coords));
3759     PetscCall(VecRestoreArray(subCoordinates, &subCoords));
3760     PetscCall(DMSetCoordinatesLocal(subdm, subCoordinates));
3761     PetscCall(VecDestroy(&subCoordinates));
3762   }
3763   /* Build SF */
3764   CHKMEMQ;
3765   {
3766     PetscSF            sfPoint, sfPointSub;
3767     const PetscSFNode *remotePoints;
3768     PetscSFNode       *sremotePoints, *newLocalPoints, *newOwners;
3769     const PetscInt    *localPoints;
3770     PetscInt          *slocalPoints;
3771     PetscInt           numRoots, numLeaves, numSubRoots = numSubCells + numSubFaces + numSubVertices, numSubLeaves = 0, l, sl, ll, pStart, pEnd, p, vStart, vEnd;
3772     PetscMPIInt        rank;
3773 
3774     PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
3775     PetscCall(DMGetPointSF(dm, &sfPoint));
3776     PetscCall(DMGetPointSF(subdm, &sfPointSub));
3777     PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
3778     PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
3779     PetscCall(PetscSFGetGraph(sfPoint, &numRoots, &numLeaves, &localPoints, &remotePoints));
3780     if (numRoots >= 0) {
3781       /* Only vertices should be shared */
3782       PetscCall(PetscMalloc2(pEnd - pStart, &newLocalPoints, numRoots, &newOwners));
3783       for (p = 0; p < pEnd - pStart; ++p) {
3784         newLocalPoints[p].rank  = -2;
3785         newLocalPoints[p].index = -2;
3786       }
3787       /* Set subleaves */
3788       for (l = 0; l < numLeaves; ++l) {
3789         const PetscInt point    = localPoints[l];
3790         const PetscInt subPoint = DMPlexFilterPoint_Internal(point, firstSubVertex, numSubVertices, subVertices);
3791 
3792         PetscCheck(!(point < vStart) || !(point >= vEnd), PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Should not be mapping anything but vertices, %" PetscInt_FMT, point);
3793         if (subPoint < 0) continue;
3794         newLocalPoints[point - pStart].rank  = rank;
3795         newLocalPoints[point - pStart].index = subPoint;
3796         ++numSubLeaves;
3797       }
3798       /* Must put in owned subpoints */
3799       for (p = pStart; p < pEnd; ++p) {
3800         const PetscInt subPoint = DMPlexFilterPoint_Internal(p, firstSubVertex, numSubVertices, subVertices);
3801 
3802         if (subPoint < 0) {
3803           newOwners[p - pStart].rank  = -3;
3804           newOwners[p - pStart].index = -3;
3805         } else {
3806           newOwners[p - pStart].rank  = rank;
3807           newOwners[p - pStart].index = subPoint;
3808         }
3809       }
3810       PetscCall(PetscSFReduceBegin(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC));
3811       PetscCall(PetscSFReduceEnd(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC));
3812       PetscCall(PetscSFBcastBegin(sfPoint, MPIU_2INT, newOwners, newLocalPoints, MPI_REPLACE));
3813       PetscCall(PetscSFBcastEnd(sfPoint, MPIU_2INT, newOwners, newLocalPoints, MPI_REPLACE));
3814       PetscCall(PetscMalloc1(numSubLeaves, &slocalPoints));
3815       PetscCall(PetscMalloc1(numSubLeaves, &sremotePoints));
3816       for (l = 0, sl = 0, ll = 0; l < numLeaves; ++l) {
3817         const PetscInt point    = localPoints[l];
3818         const PetscInt subPoint = DMPlexFilterPoint_Internal(point, firstSubVertex, numSubVertices, subVertices);
3819 
3820         if (subPoint < 0) continue;
3821         if (newLocalPoints[point].rank == rank) {
3822           ++ll;
3823           continue;
3824         }
3825         slocalPoints[sl]        = subPoint;
3826         sremotePoints[sl].rank  = newLocalPoints[point].rank;
3827         sremotePoints[sl].index = newLocalPoints[point].index;
3828         PetscCheck(sremotePoints[sl].rank >= 0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote rank for local point %" PetscInt_FMT, point);
3829         PetscCheck(sremotePoints[sl].index >= 0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote subpoint for local point %" PetscInt_FMT, point);
3830         ++sl;
3831       }
3832       PetscCall(PetscFree2(newLocalPoints, newOwners));
3833       PetscCheck(sl + ll == numSubLeaves, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatch in number of subleaves %" PetscInt_FMT " + %" PetscInt_FMT " != %" PetscInt_FMT, sl, ll, numSubLeaves);
3834       PetscCall(PetscSFSetGraph(sfPointSub, numSubRoots, sl, slocalPoints, PETSC_OWN_POINTER, sremotePoints, PETSC_OWN_POINTER));
3835     }
3836   }
3837   CHKMEMQ;
3838   /* Cleanup */
3839   if (subvertexIS) PetscCall(ISRestoreIndices(subvertexIS, &subVertices));
3840   PetscCall(ISDestroy(&subvertexIS));
3841   PetscCall(PetscFree(subCells));
3842   PetscFunctionReturn(PETSC_SUCCESS);
3843 }
3844 
3845 static PetscErrorCode DMPlexCreateCohesiveSubmesh_Interpolated(DM dm, const char labelname[], PetscInt value, DM subdm)
3846 {
3847   DMLabel label = NULL;
3848 
3849   PetscFunctionBegin;
3850   if (labelname) PetscCall(DMGetLabel(dm, labelname, &label));
3851   PetscCall(DMPlexCreateSubmeshGeneric_Interpolated(dm, label, value, PETSC_FALSE, PETSC_TRUE, 1, subdm));
3852   PetscFunctionReturn(PETSC_SUCCESS);
3853 }
3854 
3855 /*@C
3856   DMPlexCreateCohesiveSubmesh - Extract from a mesh with cohesive cells the hypersurface defined by one face of the cells. Optionally, a label can be given to restrict the cells.
3857 
3858   Input Parameters:
3859 + dm          - The original mesh
3860 . hasLagrange - The mesh has Lagrange unknowns in the cohesive cells
3861 . label       - A label name, or `NULL`
3862 - value  - A label value
3863 
3864   Output Parameter:
3865 . subdm - The surface mesh
3866 
3867   Level: developer
3868 
3869   Note:
3870   This function produces a `DMLabel` mapping original points in the submesh to their depth. This can be obtained using `DMPlexGetSubpointMap()`.
3871 
3872 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexGetSubpointMap()`, `DMPlexCreateSubmesh()`
3873 @*/
3874 PetscErrorCode DMPlexCreateCohesiveSubmesh(DM dm, PetscBool hasLagrange, const char label[], PetscInt value, DM *subdm)
3875 {
3876   PetscInt dim, cdim, depth;
3877 
3878   PetscFunctionBegin;
3879   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3880   PetscValidPointer(subdm, 5);
3881   PetscCall(DMGetDimension(dm, &dim));
3882   PetscCall(DMPlexGetDepth(dm, &depth));
3883   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), subdm));
3884   PetscCall(DMSetType(*subdm, DMPLEX));
3885   PetscCall(DMSetDimension(*subdm, dim - 1));
3886   PetscCall(DMGetCoordinateDim(dm, &cdim));
3887   PetscCall(DMSetCoordinateDim(*subdm, cdim));
3888   if (depth == dim) {
3889     PetscCall(DMPlexCreateCohesiveSubmesh_Interpolated(dm, label, value, *subdm));
3890   } else {
3891     PetscCall(DMPlexCreateCohesiveSubmesh_Uninterpolated(dm, hasLagrange, label, value, *subdm));
3892   }
3893   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_TRUE, *subdm));
3894   PetscFunctionReturn(PETSC_SUCCESS);
3895 }
3896 
3897 /*@
3898   DMPlexFilter - Extract a subset of mesh cells defined by a label as a separate mesh
3899 
3900   Input Parameters:
3901 + dm        - The original mesh
3902 . cellLabel - The `DMLabel` marking cells contained in the new mesh
3903 - value     - The label value to use
3904 
3905   Output Parameter:
3906 . subdm - The new mesh
3907 
3908   Level: developer
3909 
3910   Note:
3911   This function produces a `DMLabel` mapping original points in the submesh to their depth. This can be obtained using `DMPlexGetSubpointMap()`.
3912 
3913 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexGetSubpointMap()`, `DMGetLabel()`, `DMLabelSetValue()`, `DMPlexCreateSubmesh()`
3914 @*/
3915 PetscErrorCode DMPlexFilter(DM dm, DMLabel cellLabel, PetscInt value, DM *subdm)
3916 {
3917   PetscInt dim, overlap;
3918 
3919   PetscFunctionBegin;
3920   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3921   PetscValidPointer(subdm, 4);
3922   PetscCall(DMGetDimension(dm, &dim));
3923   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), subdm));
3924   PetscCall(DMSetType(*subdm, DMPLEX));
3925   /* Extract submesh in place, could be empty on some procs, could have inconsistency if procs do not both extract a shared cell */
3926   PetscCall(DMPlexCreateSubmeshGeneric_Interpolated(dm, cellLabel, value, PETSC_FALSE, PETSC_FALSE, 0, *subdm));
3927   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_TRUE, *subdm));
3928   // It is possible to obtain a surface mesh where some faces are in SF
3929   //   We should either mark the mesh as having an overlap, or delete these from the SF
3930   PetscCall(DMPlexGetOverlap(dm, &overlap));
3931   if (!overlap) {
3932     PetscSF         sf;
3933     const PetscInt *leaves;
3934     PetscInt        cStart, cEnd, Nl;
3935     PetscBool       hasSubcell = PETSC_FALSE, ghasSubcell;
3936 
3937     PetscCall(DMPlexGetHeightStratum(*subdm, 0, &cStart, &cEnd));
3938     PetscCall(DMGetPointSF(*subdm, &sf));
3939     PetscCall(PetscSFGetGraph(sf, NULL, &Nl, &leaves, NULL));
3940     for (PetscInt l = 0; l < Nl; ++l) {
3941       const PetscInt point = leaves ? leaves[l] : l;
3942 
3943       if (point >= cStart && point < cEnd) {
3944         hasSubcell = PETSC_TRUE;
3945         break;
3946       }
3947     }
3948     PetscCall(MPIU_Allreduce(&hasSubcell, &ghasSubcell, 1, MPIU_BOOL, MPI_LOR, PetscObjectComm((PetscObject)dm)));
3949     if (ghasSubcell) PetscCall(DMPlexSetOverlap(*subdm, NULL, 1));
3950   }
3951   PetscFunctionReturn(PETSC_SUCCESS);
3952 }
3953 
3954 /*@
3955   DMPlexGetSubpointMap - Returns a `DMLabel` with point dimension as values
3956 
3957   Input Parameter:
3958 . dm - The submesh `DM`
3959 
3960   Output Parameter:
3961 . subpointMap - The `DMLabel` of all the points from the original mesh in this submesh, or `NULL` if this is not a submesh
3962 
3963   Level: developer
3964 
3965 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateSubmesh()`, `DMPlexGetSubpointIS()`
3966 @*/
3967 PetscErrorCode DMPlexGetSubpointMap(DM dm, DMLabel *subpointMap)
3968 {
3969   PetscFunctionBegin;
3970   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3971   PetscValidPointer(subpointMap, 2);
3972   *subpointMap = ((DM_Plex *)dm->data)->subpointMap;
3973   PetscFunctionReturn(PETSC_SUCCESS);
3974 }
3975 
3976 /*@
3977   DMPlexSetSubpointMap - Sets the `DMLabel` with point dimension as values
3978 
3979   Input Parameters:
3980 + dm - The submesh `DM`
3981 - subpointMap - The `DMLabel` of all the points from the original mesh in this submesh
3982 
3983   Level: developer
3984 
3985   Note:
3986   Should normally not be called by the user, since it is set in `DMPlexCreateSubmesh()`
3987 
3988 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateSubmesh()`, `DMPlexGetSubpointIS()`
3989 @*/
3990 PetscErrorCode DMPlexSetSubpointMap(DM dm, DMLabel subpointMap)
3991 {
3992   DM_Plex *mesh = (DM_Plex *)dm->data;
3993   DMLabel  tmp;
3994 
3995   PetscFunctionBegin;
3996   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3997   tmp               = mesh->subpointMap;
3998   mesh->subpointMap = subpointMap;
3999   PetscCall(PetscObjectReference((PetscObject)mesh->subpointMap));
4000   PetscCall(DMLabelDestroy(&tmp));
4001   PetscFunctionReturn(PETSC_SUCCESS);
4002 }
4003 
4004 static PetscErrorCode DMPlexCreateSubpointIS_Internal(DM dm, IS *subpointIS)
4005 {
4006   DMLabel  spmap;
4007   PetscInt depth, d;
4008 
4009   PetscFunctionBegin;
4010   PetscCall(DMPlexGetSubpointMap(dm, &spmap));
4011   PetscCall(DMPlexGetDepth(dm, &depth));
4012   if (spmap && depth >= 0) {
4013     DM_Plex  *mesh = (DM_Plex *)dm->data;
4014     PetscInt *points, *depths;
4015     PetscInt  pStart, pEnd, p, off;
4016 
4017     PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
4018     PetscCheck(!pStart, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Submeshes must start the point numbering at 0, not %" PetscInt_FMT, pStart);
4019     PetscCall(PetscMalloc1(pEnd, &points));
4020     PetscCall(DMGetWorkArray(dm, depth + 1, MPIU_INT, &depths));
4021     depths[0] = depth;
4022     depths[1] = 0;
4023     for (d = 2; d <= depth; ++d) depths[d] = depth + 1 - d;
4024     for (d = 0, off = 0; d <= depth; ++d) {
4025       const PetscInt dep = depths[d];
4026       PetscInt       depStart, depEnd, n;
4027 
4028       PetscCall(DMPlexGetDepthStratum(dm, dep, &depStart, &depEnd));
4029       PetscCall(DMLabelGetStratumSize(spmap, dep, &n));
4030       if (((d < 2) && (depth > 1)) || (d == 1)) { /* Only check vertices and cells for now since the map is broken for others */
4031         PetscCheck(n == depEnd - depStart, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "The number of mapped submesh points %" PetscInt_FMT " at depth %" PetscInt_FMT " should be %" PetscInt_FMT, n, dep, depEnd - depStart);
4032       } else {
4033         if (!n) {
4034           if (d == 0) {
4035             /* Missing cells */
4036             for (p = 0; p < depEnd - depStart; ++p, ++off) points[off] = -1;
4037           } else {
4038             /* Missing faces */
4039             for (p = 0; p < depEnd - depStart; ++p, ++off) points[off] = PETSC_MAX_INT;
4040           }
4041         }
4042       }
4043       if (n) {
4044         IS              is;
4045         const PetscInt *opoints;
4046 
4047         PetscCall(DMLabelGetStratumIS(spmap, dep, &is));
4048         PetscCall(ISGetIndices(is, &opoints));
4049         for (p = 0; p < n; ++p, ++off) points[off] = opoints[p];
4050         PetscCall(ISRestoreIndices(is, &opoints));
4051         PetscCall(ISDestroy(&is));
4052       }
4053     }
4054     PetscCall(DMRestoreWorkArray(dm, depth + 1, MPIU_INT, &depths));
4055     PetscCheck(off == pEnd, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "The number of mapped submesh points %" PetscInt_FMT " should be %" PetscInt_FMT, off, pEnd);
4056     PetscCall(ISCreateGeneral(PETSC_COMM_SELF, pEnd, points, PETSC_OWN_POINTER, subpointIS));
4057     PetscCall(PetscObjectStateGet((PetscObject)spmap, &mesh->subpointState));
4058   }
4059   PetscFunctionReturn(PETSC_SUCCESS);
4060 }
4061 
4062 /*@
4063   DMPlexGetSubpointIS - Returns an `IS` covering the entire subdm chart with the original points as data
4064 
4065   Input Parameter:
4066 . dm - The submesh `DM`
4067 
4068   Output Parameter:
4069 . subpointIS - The `IS` of all the points from the original mesh in this submesh, or `NULL` if this is not a submesh
4070 
4071   Level: developer
4072 
4073   Note:
4074   This `IS` is guaranteed to be sorted by the construction of the submesh
4075 
4076 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMPlexCreateSubmesh()`, `DMPlexGetSubpointMap()`
4077 @*/
4078 PetscErrorCode DMPlexGetSubpointIS(DM dm, IS *subpointIS)
4079 {
4080   DM_Plex         *mesh = (DM_Plex *)dm->data;
4081   DMLabel          spmap;
4082   PetscObjectState state;
4083 
4084   PetscFunctionBegin;
4085   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4086   PetscValidPointer(subpointIS, 2);
4087   PetscCall(DMPlexGetSubpointMap(dm, &spmap));
4088   PetscCall(PetscObjectStateGet((PetscObject)spmap, &state));
4089   if (state != mesh->subpointState || !mesh->subpointIS) PetscCall(DMPlexCreateSubpointIS_Internal(dm, &mesh->subpointIS));
4090   *subpointIS = mesh->subpointIS;
4091   PetscFunctionReturn(PETSC_SUCCESS);
4092 }
4093 
4094 /*@
4095   DMGetEnclosureRelation - Get the relationship between `dmA` and `dmB`
4096 
4097   Input Parameters:
4098 + dmA - The first `DM`
4099 - dmB - The second `DM`
4100 
4101   Output Parameter:
4102 . rel - The relation of `dmA` to `dmB`
4103 
4104   Level: intermediate
4105 
4106 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMGetEnclosurePoint()`
4107 @*/
4108 PetscErrorCode DMGetEnclosureRelation(DM dmA, DM dmB, DMEnclosureType *rel)
4109 {
4110   DM       plexA, plexB, sdm;
4111   DMLabel  spmap;
4112   PetscInt pStartA, pEndA, pStartB, pEndB, NpA, NpB;
4113 
4114   PetscFunctionBegin;
4115   PetscValidPointer(rel, 3);
4116   *rel = DM_ENC_NONE;
4117   if (!dmA || !dmB) PetscFunctionReturn(PETSC_SUCCESS);
4118   PetscValidHeaderSpecific(dmA, DM_CLASSID, 1);
4119   PetscValidHeaderSpecific(dmB, DM_CLASSID, 2);
4120   if (dmA == dmB) {
4121     *rel = DM_ENC_EQUALITY;
4122     PetscFunctionReturn(PETSC_SUCCESS);
4123   }
4124   PetscCall(DMConvert(dmA, DMPLEX, &plexA));
4125   PetscCall(DMConvert(dmB, DMPLEX, &plexB));
4126   PetscCall(DMPlexGetChart(plexA, &pStartA, &pEndA));
4127   PetscCall(DMPlexGetChart(plexB, &pStartB, &pEndB));
4128   /* Assumption 1: subDMs have smaller charts than the DMs that they originate from
4129     - The degenerate case of a subdomain which includes all of the domain on some process can be treated as equality */
4130   if ((pStartA == pStartB) && (pEndA == pEndB)) {
4131     *rel = DM_ENC_EQUALITY;
4132     goto end;
4133   }
4134   NpA = pEndA - pStartA;
4135   NpB = pEndB - pStartB;
4136   if (NpA == NpB) goto end;
4137   sdm = NpA > NpB ? plexB : plexA; /* The other is the original, enclosing dm */
4138   PetscCall(DMPlexGetSubpointMap(sdm, &spmap));
4139   if (!spmap) goto end;
4140   /* TODO Check the space mapped to by subpointMap is same size as dm */
4141   if (NpA > NpB) {
4142     *rel = DM_ENC_SUPERMESH;
4143   } else {
4144     *rel = DM_ENC_SUBMESH;
4145   }
4146 end:
4147   PetscCall(DMDestroy(&plexA));
4148   PetscCall(DMDestroy(&plexB));
4149   PetscFunctionReturn(PETSC_SUCCESS);
4150 }
4151 
4152 /*@
4153   DMGetEnclosurePoint - Get the point `pA` in `dmA` which corresponds to the point `pB` in `dmB`
4154 
4155   Input Parameters:
4156 + dmA   - The first `DM`
4157 . dmB   - The second `DM`
4158 . etype - The type of enclosure relation that `dmA` has to `dmB`
4159 - pB    - A point of `dmB`
4160 
4161   Output Parameter:
4162 . pA    - The corresponding point of `dmA`
4163 
4164   Level: intermediate
4165 
4166 .seealso: [](chapter_unstructured), `DM`, `DMPLEX`, `DMGetEnclosureRelation()`
4167 @*/
4168 PetscErrorCode DMGetEnclosurePoint(DM dmA, DM dmB, DMEnclosureType etype, PetscInt pB, PetscInt *pA)
4169 {
4170   DM              sdm;
4171   IS              subpointIS;
4172   const PetscInt *subpoints;
4173   PetscInt        numSubpoints;
4174 
4175   PetscFunctionBegin;
4176   /* TODO Cache the IS, making it look like an index */
4177   switch (etype) {
4178   case DM_ENC_SUPERMESH:
4179     sdm = dmB;
4180     PetscCall(DMPlexGetSubpointIS(sdm, &subpointIS));
4181     PetscCall(ISGetIndices(subpointIS, &subpoints));
4182     *pA = subpoints[pB];
4183     PetscCall(ISRestoreIndices(subpointIS, &subpoints));
4184     break;
4185   case DM_ENC_SUBMESH:
4186     sdm = dmA;
4187     PetscCall(DMPlexGetSubpointIS(sdm, &subpointIS));
4188     PetscCall(ISGetLocalSize(subpointIS, &numSubpoints));
4189     PetscCall(ISGetIndices(subpointIS, &subpoints));
4190     PetscCall(PetscFindInt(pB, numSubpoints, subpoints, pA));
4191     if (*pA < 0) {
4192       PetscCall(DMViewFromOptions(dmA, NULL, "-dm_enc_A_view"));
4193       PetscCall(DMViewFromOptions(dmB, NULL, "-dm_enc_B_view"));
4194       SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Point %" PetscInt_FMT " not found in submesh", pB);
4195     }
4196     PetscCall(ISRestoreIndices(subpointIS, &subpoints));
4197     break;
4198   case DM_ENC_EQUALITY:
4199   case DM_ENC_NONE:
4200     *pA = pB;
4201     break;
4202   case DM_ENC_UNKNOWN: {
4203     DMEnclosureType enc;
4204 
4205     PetscCall(DMGetEnclosureRelation(dmA, dmB, &enc));
4206     PetscCall(DMGetEnclosurePoint(dmA, dmB, enc, pB, pA));
4207   } break;
4208   default:
4209     SETERRQ(PetscObjectComm((PetscObject)dmA), PETSC_ERR_ARG_OUTOFRANGE, "Invalid enclosure type %d", (int)etype);
4210   }
4211   PetscFunctionReturn(PETSC_SUCCESS);
4212 }
4213