xref: /petsc/src/dm/impls/plex/plexsubmesh.c (revision 0ea77eda237b1068ff0d8bfa28c3463dc2087695)
1 #include <petsc/private/dmpleximpl.h>  /*I      "petscdmplex.h"    I*/
2 #include <petsc/private/dmlabelimpl.h> /*I      "petscdmlabel.h"   I*/
3 #include <petscsf.h>
4 
5 static PetscErrorCode DMPlexCellIsHybrid_Internal(DM dm, PetscInt p, PetscBool *isHybrid) {
6   DMPolytopeType ct;
7 
8   PetscFunctionBegin;
9   PetscCall(DMPlexGetCellType(dm, p, &ct));
10   switch (ct) {
11   case DM_POLYTOPE_POINT_PRISM_TENSOR:
12   case DM_POLYTOPE_SEG_PRISM_TENSOR:
13   case DM_POLYTOPE_TRI_PRISM_TENSOR:
14   case DM_POLYTOPE_QUAD_PRISM_TENSOR: *isHybrid = PETSC_TRUE;
15   default: *isHybrid = PETSC_FALSE;
16   }
17   PetscFunctionReturn(0);
18 }
19 
20 static PetscErrorCode DMPlexGetTensorPrismBounds_Internal(DM dm, PetscInt dim, PetscInt *cStart, PetscInt *cEnd) {
21   DMLabel ctLabel;
22 
23   PetscFunctionBegin;
24   if (cStart) *cStart = -1;
25   if (cEnd) *cEnd = -1;
26   PetscCall(DMPlexGetCellTypeLabel(dm, &ctLabel));
27   switch (dim) {
28   case 1: PetscCall(DMLabelGetStratumBounds(ctLabel, DM_POLYTOPE_POINT_PRISM_TENSOR, cStart, cEnd)); break;
29   case 2: PetscCall(DMLabelGetStratumBounds(ctLabel, DM_POLYTOPE_SEG_PRISM_TENSOR, cStart, cEnd)); break;
30   case 3:
31     PetscCall(DMLabelGetStratumBounds(ctLabel, DM_POLYTOPE_TRI_PRISM_TENSOR, cStart, cEnd));
32     if (*cStart < 0) PetscCall(DMLabelGetStratumBounds(ctLabel, DM_POLYTOPE_QUAD_PRISM_TENSOR, cStart, cEnd));
33     break;
34   default: PetscFunctionReturn(0);
35   }
36   PetscFunctionReturn(0);
37 }
38 
39 static PetscErrorCode DMPlexMarkBoundaryFaces_Internal(DM dm, PetscInt val, PetscInt cellHeight, DMLabel label) {
40   PetscInt fStart, fEnd, f;
41 
42   PetscFunctionBegin;
43   PetscCall(DMPlexGetHeightStratum(dm, cellHeight + 1, &fStart, &fEnd));
44   for (f = fStart; f < fEnd; ++f) {
45     PetscInt supportSize;
46 
47     PetscCall(DMPlexGetSupportSize(dm, f, &supportSize));
48     if (supportSize == 1) {
49       if (val < 0) {
50         PetscInt *closure = NULL;
51         PetscInt  clSize, cl, cval;
52 
53         PetscCall(DMPlexGetTransitiveClosure(dm, f, PETSC_TRUE, &clSize, &closure));
54         for (cl = 0; cl < clSize * 2; cl += 2) {
55           PetscCall(DMLabelGetValue(label, closure[cl], &cval));
56           if (cval < 0) continue;
57           PetscCall(DMLabelSetValue(label, f, cval));
58           break;
59         }
60         if (cl == clSize * 2) PetscCall(DMLabelSetValue(label, f, 1));
61         PetscCall(DMPlexRestoreTransitiveClosure(dm, f, PETSC_TRUE, &clSize, &closure));
62       } else {
63         PetscCall(DMLabelSetValue(label, f, val));
64       }
65     }
66   }
67   PetscFunctionReturn(0);
68 }
69 
70 /*@
71   DMPlexMarkBoundaryFaces - Mark all faces on the boundary
72 
73   Not Collective
74 
75   Input Parameters:
76 + dm - The original DM
77 - val - The marker value, or PETSC_DETERMINE to use some value in the closure (or 1 if none are found)
78 
79   Output Parameter:
80 . label - The DMLabel marking boundary faces with the given value
81 
82   Level: developer
83 
84 .seealso: `DMLabelCreate()`, `DMCreateLabel()`
85 @*/
86 PetscErrorCode DMPlexMarkBoundaryFaces(DM dm, PetscInt val, DMLabel label) {
87   DMPlexInterpolatedFlag flg;
88 
89   PetscFunctionBegin;
90   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
91   PetscCall(DMPlexIsInterpolated(dm, &flg));
92   PetscCheck(flg == DMPLEX_INTERPOLATED_FULL, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM is not fully interpolated on this rank");
93   PetscCall(DMPlexMarkBoundaryFaces_Internal(dm, val, 0, label));
94   PetscFunctionReturn(0);
95 }
96 
97 static PetscErrorCode DMPlexLabelComplete_Internal(DM dm, DMLabel label, PetscBool completeCells) {
98   IS              valueIS;
99   PetscSF         sfPoint;
100   const PetscInt *values;
101   PetscInt        numValues, v, cStart, cEnd, nroots;
102 
103   PetscFunctionBegin;
104   PetscCall(DMLabelGetNumValues(label, &numValues));
105   PetscCall(DMLabelGetValueIS(label, &valueIS));
106   PetscCall(DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd));
107   PetscCall(ISGetIndices(valueIS, &values));
108   for (v = 0; v < numValues; ++v) {
109     IS              pointIS;
110     const PetscInt *points;
111     PetscInt        numPoints, p;
112 
113     PetscCall(DMLabelGetStratumSize(label, values[v], &numPoints));
114     PetscCall(DMLabelGetStratumIS(label, values[v], &pointIS));
115     PetscCall(ISGetIndices(pointIS, &points));
116     for (p = 0; p < numPoints; ++p) {
117       PetscInt  q       = points[p];
118       PetscInt *closure = NULL;
119       PetscInt  closureSize, c;
120 
121       if (cStart <= q && q < cEnd && !completeCells) { /* skip cells */
122         continue;
123       }
124       PetscCall(DMPlexGetTransitiveClosure(dm, q, PETSC_TRUE, &closureSize, &closure));
125       for (c = 0; c < closureSize * 2; c += 2) PetscCall(DMLabelSetValue(label, closure[c], values[v]));
126       PetscCall(DMPlexRestoreTransitiveClosure(dm, q, PETSC_TRUE, &closureSize, &closure));
127     }
128     PetscCall(ISRestoreIndices(pointIS, &points));
129     PetscCall(ISDestroy(&pointIS));
130   }
131   PetscCall(ISRestoreIndices(valueIS, &values));
132   PetscCall(ISDestroy(&valueIS));
133   PetscCall(DMGetPointSF(dm, &sfPoint));
134   PetscCall(PetscSFGetGraph(sfPoint, &nroots, NULL, NULL, NULL));
135   if (nroots >= 0) {
136     DMLabel         lblRoots, lblLeaves;
137     IS              valueIS, pointIS;
138     const PetscInt *values;
139     PetscInt        numValues, v;
140 
141     /* Pull point contributions from remote leaves into local roots */
142     PetscCall(DMLabelGather(label, sfPoint, &lblLeaves));
143     PetscCall(DMLabelGetValueIS(lblLeaves, &valueIS));
144     PetscCall(ISGetLocalSize(valueIS, &numValues));
145     PetscCall(ISGetIndices(valueIS, &values));
146     for (v = 0; v < numValues; ++v) {
147       const PetscInt value = values[v];
148 
149       PetscCall(DMLabelGetStratumIS(lblLeaves, value, &pointIS));
150       PetscCall(DMLabelInsertIS(label, pointIS, value));
151       PetscCall(ISDestroy(&pointIS));
152     }
153     PetscCall(ISRestoreIndices(valueIS, &values));
154     PetscCall(ISDestroy(&valueIS));
155     PetscCall(DMLabelDestroy(&lblLeaves));
156     /* Push point contributions from roots into remote leaves */
157     PetscCall(DMLabelDistribute(label, sfPoint, &lblRoots));
158     PetscCall(DMLabelGetValueIS(lblRoots, &valueIS));
159     PetscCall(ISGetLocalSize(valueIS, &numValues));
160     PetscCall(ISGetIndices(valueIS, &values));
161     for (v = 0; v < numValues; ++v) {
162       const PetscInt value = values[v];
163 
164       PetscCall(DMLabelGetStratumIS(lblRoots, value, &pointIS));
165       PetscCall(DMLabelInsertIS(label, pointIS, value));
166       PetscCall(ISDestroy(&pointIS));
167     }
168     PetscCall(ISRestoreIndices(valueIS, &values));
169     PetscCall(ISDestroy(&valueIS));
170     PetscCall(DMLabelDestroy(&lblRoots));
171   }
172   PetscFunctionReturn(0);
173 }
174 
175 /*@
176   DMPlexLabelComplete - Starting with a label marking points on a surface, we add the transitive closure to the surface
177 
178   Input Parameters:
179 + dm - The DM
180 - label - A DMLabel marking the surface points
181 
182   Output Parameter:
183 . label - A DMLabel marking all surface points in the transitive closure
184 
185   Level: developer
186 
187 .seealso: `DMPlexLabelCohesiveComplete()`
188 @*/
189 PetscErrorCode DMPlexLabelComplete(DM dm, DMLabel label) {
190   PetscFunctionBegin;
191   PetscCall(DMPlexLabelComplete_Internal(dm, label, PETSC_TRUE));
192   PetscFunctionReturn(0);
193 }
194 
195 /*@
196   DMPlexLabelAddCells - Starting with a label marking points on a surface, we add a cell for each point
197 
198   Input Parameters:
199 + dm - The DM
200 - label - A DMLabel marking the surface points
201 
202   Output Parameter:
203 . label - A DMLabel incorporating cells
204 
205   Level: developer
206 
207   Note: The cells allow FEM boundary conditions to be applied using the cell geometry
208 
209 .seealso: `DMPlexLabelAddFaceCells()`, `DMPlexLabelComplete()`, `DMPlexLabelCohesiveComplete()`
210 @*/
211 PetscErrorCode DMPlexLabelAddCells(DM dm, DMLabel label) {
212   IS              valueIS;
213   const PetscInt *values;
214   PetscInt        numValues, v, cStart, cEnd;
215 
216   PetscFunctionBegin;
217   PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd));
218   PetscCall(DMLabelGetNumValues(label, &numValues));
219   PetscCall(DMLabelGetValueIS(label, &valueIS));
220   PetscCall(ISGetIndices(valueIS, &values));
221   for (v = 0; v < numValues; ++v) {
222     IS              pointIS;
223     const PetscInt *points;
224     PetscInt        numPoints, p;
225 
226     PetscCall(DMLabelGetStratumSize(label, values[v], &numPoints));
227     PetscCall(DMLabelGetStratumIS(label, values[v], &pointIS));
228     PetscCall(ISGetIndices(pointIS, &points));
229     for (p = 0; p < numPoints; ++p) {
230       PetscInt *closure = NULL;
231       PetscInt  closureSize, cl;
232 
233       PetscCall(DMPlexGetTransitiveClosure(dm, points[p], PETSC_FALSE, &closureSize, &closure));
234       for (cl = closureSize - 1; cl > 0; --cl) {
235         const PetscInt cell = closure[cl * 2];
236         if ((cell >= cStart) && (cell < cEnd)) {
237           PetscCall(DMLabelSetValue(label, cell, values[v]));
238           break;
239         }
240       }
241       PetscCall(DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_FALSE, &closureSize, &closure));
242     }
243     PetscCall(ISRestoreIndices(pointIS, &points));
244     PetscCall(ISDestroy(&pointIS));
245   }
246   PetscCall(ISRestoreIndices(valueIS, &values));
247   PetscCall(ISDestroy(&valueIS));
248   PetscFunctionReturn(0);
249 }
250 
251 /*@
252   DMPlexLabelAddFaceCells - Starting with a label marking faces on a surface, we add a cell for each face
253 
254   Input Parameters:
255 + dm - The DM
256 - label - A DMLabel marking the surface points
257 
258   Output Parameter:
259 . label - A DMLabel incorporating cells
260 
261   Level: developer
262 
263   Note: The cells allow FEM boundary conditions to be applied using the cell geometry
264 
265 .seealso: `DMPlexLabelAddCells()`, `DMPlexLabelComplete()`, `DMPlexLabelCohesiveComplete()`
266 @*/
267 PetscErrorCode DMPlexLabelAddFaceCells(DM dm, DMLabel label) {
268   IS              valueIS;
269   const PetscInt *values;
270   PetscInt        numValues, v, cStart, cEnd, fStart, fEnd;
271 
272   PetscFunctionBegin;
273   PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd));
274   PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
275   PetscCall(DMLabelGetNumValues(label, &numValues));
276   PetscCall(DMLabelGetValueIS(label, &valueIS));
277   PetscCall(ISGetIndices(valueIS, &values));
278   for (v = 0; v < numValues; ++v) {
279     IS              pointIS;
280     const PetscInt *points;
281     PetscInt        numPoints, p;
282 
283     PetscCall(DMLabelGetStratumSize(label, values[v], &numPoints));
284     PetscCall(DMLabelGetStratumIS(label, values[v], &pointIS));
285     PetscCall(ISGetIndices(pointIS, &points));
286     for (p = 0; p < numPoints; ++p) {
287       const PetscInt face    = points[p];
288       PetscInt      *closure = NULL;
289       PetscInt       closureSize, cl;
290 
291       if ((face < fStart) || (face >= fEnd)) continue;
292       PetscCall(DMPlexGetTransitiveClosure(dm, face, PETSC_FALSE, &closureSize, &closure));
293       for (cl = closureSize - 1; cl > 0; --cl) {
294         const PetscInt cell = closure[cl * 2];
295         if ((cell >= cStart) && (cell < cEnd)) {
296           PetscCall(DMLabelSetValue(label, cell, values[v]));
297           break;
298         }
299       }
300       PetscCall(DMPlexRestoreTransitiveClosure(dm, face, PETSC_FALSE, &closureSize, &closure));
301     }
302     PetscCall(ISRestoreIndices(pointIS, &points));
303     PetscCall(ISDestroy(&pointIS));
304   }
305   PetscCall(ISRestoreIndices(valueIS, &values));
306   PetscCall(ISDestroy(&valueIS));
307   PetscFunctionReturn(0);
308 }
309 
310 /*@
311   DMPlexLabelClearCells - Remove cells from a label
312 
313   Input Parameters:
314 + dm - The DM
315 - label - A DMLabel marking surface points and their adjacent cells
316 
317   Output Parameter:
318 . label - A DMLabel without cells
319 
320   Level: developer
321 
322   Note: This undoes DMPlexLabelAddCells() or DMPlexLabelAddFaceCells()
323 
324 .seealso: `DMPlexLabelComplete()`, `DMPlexLabelCohesiveComplete()`, `DMPlexLabelAddCells()`
325 @*/
326 PetscErrorCode DMPlexLabelClearCells(DM dm, DMLabel label) {
327   IS              valueIS;
328   const PetscInt *values;
329   PetscInt        numValues, v, cStart, cEnd;
330 
331   PetscFunctionBegin;
332   PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd));
333   PetscCall(DMLabelGetNumValues(label, &numValues));
334   PetscCall(DMLabelGetValueIS(label, &valueIS));
335   PetscCall(ISGetIndices(valueIS, &values));
336   for (v = 0; v < numValues; ++v) {
337     IS              pointIS;
338     const PetscInt *points;
339     PetscInt        numPoints, p;
340 
341     PetscCall(DMLabelGetStratumSize(label, values[v], &numPoints));
342     PetscCall(DMLabelGetStratumIS(label, values[v], &pointIS));
343     PetscCall(ISGetIndices(pointIS, &points));
344     for (p = 0; p < numPoints; ++p) {
345       PetscInt point = points[p];
346 
347       if (point >= cStart && point < cEnd) PetscCall(DMLabelClearValue(label, point, values[v]));
348     }
349     PetscCall(ISRestoreIndices(pointIS, &points));
350     PetscCall(ISDestroy(&pointIS));
351   }
352   PetscCall(ISRestoreIndices(valueIS, &values));
353   PetscCall(ISDestroy(&valueIS));
354   PetscFunctionReturn(0);
355 }
356 
357 /* take (oldEnd, added) pairs, ordered by height and convert them to (oldstart, newstart) pairs, ordered by ascending
358  * index (skipping first, which is (0,0)) */
359 static inline PetscErrorCode DMPlexShiftPointSetUp_Internal(PetscInt depth, PetscInt depthShift[]) {
360   PetscInt d, off = 0;
361 
362   PetscFunctionBegin;
363   /* sort by (oldend): yes this is an O(n^2) sort, we expect depth <= 3 */
364   for (d = 0; d < depth; d++) {
365     PetscInt firstd     = d;
366     PetscInt firstStart = depthShift[2 * d];
367     PetscInt e;
368 
369     for (e = d + 1; e <= depth; e++) {
370       if (depthShift[2 * e] < firstStart) {
371         firstd     = e;
372         firstStart = depthShift[2 * d];
373       }
374     }
375     if (firstd != d) {
376       PetscInt swap[2];
377 
378       e                     = firstd;
379       swap[0]               = depthShift[2 * d];
380       swap[1]               = depthShift[2 * d + 1];
381       depthShift[2 * d]     = depthShift[2 * e];
382       depthShift[2 * d + 1] = depthShift[2 * e + 1];
383       depthShift[2 * e]     = swap[0];
384       depthShift[2 * e + 1] = swap[1];
385     }
386   }
387   /* convert (oldstart, added) to (oldstart, newstart) */
388   for (d = 0; d <= depth; d++) {
389     off += depthShift[2 * d + 1];
390     depthShift[2 * d + 1] = depthShift[2 * d] + off;
391   }
392   PetscFunctionReturn(0);
393 }
394 
395 /* depthShift is a list of (old, new) pairs */
396 static inline PetscInt DMPlexShiftPoint_Internal(PetscInt p, PetscInt depth, PetscInt depthShift[]) {
397   PetscInt d;
398   PetscInt newOff = 0;
399 
400   for (d = 0; d <= depth; d++) {
401     if (p < depthShift[2 * d]) return p + newOff;
402     else newOff = depthShift[2 * d + 1] - depthShift[2 * d];
403   }
404   return p + newOff;
405 }
406 
407 /* depthShift is a list of (old, new) pairs */
408 static inline PetscInt DMPlexShiftPointInverse_Internal(PetscInt p, PetscInt depth, PetscInt depthShift[]) {
409   PetscInt d;
410   PetscInt newOff = 0;
411 
412   for (d = 0; d <= depth; d++) {
413     if (p < depthShift[2 * d + 1]) return p + newOff;
414     else newOff = depthShift[2 * d] - depthShift[2 * d + 1];
415   }
416   return p + newOff;
417 }
418 
419 static PetscErrorCode DMPlexShiftSizes_Internal(DM dm, PetscInt depthShift[], DM dmNew) {
420   PetscInt depth = 0, d, pStart, pEnd, p;
421   DMLabel  depthLabel;
422 
423   PetscFunctionBegin;
424   PetscCall(DMPlexGetDepth(dm, &depth));
425   if (depth < 0) PetscFunctionReturn(0);
426   /* Step 1: Expand chart */
427   PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
428   pEnd = DMPlexShiftPoint_Internal(pEnd, depth, depthShift);
429   PetscCall(DMPlexSetChart(dmNew, pStart, pEnd));
430   PetscCall(DMCreateLabel(dmNew, "depth"));
431   PetscCall(DMPlexGetDepthLabel(dmNew, &depthLabel));
432   PetscCall(DMCreateLabel(dmNew, "celltype"));
433   /* Step 2: Set cone and support sizes */
434   for (d = 0; d <= depth; ++d) {
435     PetscInt pStartNew, pEndNew;
436     IS       pIS;
437 
438     PetscCall(DMPlexGetDepthStratum(dm, d, &pStart, &pEnd));
439     pStartNew = DMPlexShiftPoint_Internal(pStart, depth, depthShift);
440     pEndNew   = DMPlexShiftPoint_Internal(pEnd, depth, depthShift);
441     PetscCall(ISCreateStride(PETSC_COMM_SELF, pEndNew - pStartNew, pStartNew, 1, &pIS));
442     PetscCall(DMLabelSetStratumIS(depthLabel, d, pIS));
443     PetscCall(ISDestroy(&pIS));
444     for (p = pStart; p < pEnd; ++p) {
445       PetscInt       newp = DMPlexShiftPoint_Internal(p, depth, depthShift);
446       PetscInt       size;
447       DMPolytopeType ct;
448 
449       PetscCall(DMPlexGetConeSize(dm, p, &size));
450       PetscCall(DMPlexSetConeSize(dmNew, newp, size));
451       PetscCall(DMPlexGetSupportSize(dm, p, &size));
452       PetscCall(DMPlexSetSupportSize(dmNew, newp, size));
453       PetscCall(DMPlexGetCellType(dm, p, &ct));
454       PetscCall(DMPlexSetCellType(dmNew, newp, ct));
455     }
456   }
457   PetscFunctionReturn(0);
458 }
459 
460 static PetscErrorCode DMPlexShiftPoints_Internal(DM dm, PetscInt depthShift[], DM dmNew) {
461   PetscInt *newpoints;
462   PetscInt  depth = 0, maxConeSize, maxSupportSize, maxConeSizeNew, maxSupportSizeNew, pStart, pEnd, p;
463 
464   PetscFunctionBegin;
465   PetscCall(DMPlexGetDepth(dm, &depth));
466   if (depth < 0) PetscFunctionReturn(0);
467   PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize));
468   PetscCall(DMPlexGetMaxSizes(dmNew, &maxConeSizeNew, &maxSupportSizeNew));
469   PetscCall(PetscMalloc1(PetscMax(PetscMax(maxConeSize, maxSupportSize), PetscMax(maxConeSizeNew, maxSupportSizeNew)), &newpoints));
470   /* Step 5: Set cones and supports */
471   PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
472   for (p = pStart; p < pEnd; ++p) {
473     const PetscInt *points = NULL, *orientations = NULL;
474     PetscInt        size, sizeNew, i, newp = DMPlexShiftPoint_Internal(p, depth, depthShift);
475 
476     PetscCall(DMPlexGetConeSize(dm, p, &size));
477     PetscCall(DMPlexGetCone(dm, p, &points));
478     PetscCall(DMPlexGetConeOrientation(dm, p, &orientations));
479     for (i = 0; i < size; ++i) newpoints[i] = DMPlexShiftPoint_Internal(points[i], depth, depthShift);
480     PetscCall(DMPlexSetCone(dmNew, newp, newpoints));
481     PetscCall(DMPlexSetConeOrientation(dmNew, newp, orientations));
482     PetscCall(DMPlexGetSupportSize(dm, p, &size));
483     PetscCall(DMPlexGetSupportSize(dmNew, newp, &sizeNew));
484     PetscCall(DMPlexGetSupport(dm, p, &points));
485     for (i = 0; i < size; ++i) newpoints[i] = DMPlexShiftPoint_Internal(points[i], depth, depthShift);
486     for (i = size; i < sizeNew; ++i) newpoints[i] = 0;
487     PetscCall(DMPlexSetSupport(dmNew, newp, newpoints));
488   }
489   PetscCall(PetscFree(newpoints));
490   PetscFunctionReturn(0);
491 }
492 
493 static PetscErrorCode DMPlexShiftCoordinates_Internal(DM dm, PetscInt depthShift[], DM dmNew) {
494   PetscSection coordSection, newCoordSection;
495   Vec          coordinates, newCoordinates;
496   PetscScalar *coords, *newCoords;
497   PetscInt     coordSize, sStart, sEnd;
498   PetscInt     dim, depth = 0, cStart, cEnd, cStartNew, cEndNew, c, vStart, vEnd, vStartNew, vEndNew, v;
499   PetscBool    hasCells;
500 
501   PetscFunctionBegin;
502   PetscCall(DMGetCoordinateDim(dm, &dim));
503   PetscCall(DMSetCoordinateDim(dmNew, dim));
504   PetscCall(DMPlexGetDepth(dm, &depth));
505   /* Step 8: Convert coordinates */
506   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
507   PetscCall(DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd));
508   PetscCall(DMPlexGetDepthStratum(dmNew, 0, &vStartNew, &vEndNew));
509   PetscCall(DMPlexGetHeightStratum(dmNew, 0, &cStartNew, &cEndNew));
510   PetscCall(DMGetCoordinateSection(dm, &coordSection));
511   PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dm), &newCoordSection));
512   PetscCall(PetscSectionSetNumFields(newCoordSection, 1));
513   PetscCall(PetscSectionSetFieldComponents(newCoordSection, 0, dim));
514   PetscCall(PetscSectionGetChart(coordSection, &sStart, &sEnd));
515   hasCells = sStart == cStart ? PETSC_TRUE : PETSC_FALSE;
516   PetscCall(PetscSectionSetChart(newCoordSection, hasCells ? cStartNew : vStartNew, vEndNew));
517   if (hasCells) {
518     for (c = cStart; c < cEnd; ++c) {
519       PetscInt cNew = DMPlexShiftPoint_Internal(c, depth, depthShift), dof;
520 
521       PetscCall(PetscSectionGetDof(coordSection, c, &dof));
522       PetscCall(PetscSectionSetDof(newCoordSection, cNew, dof));
523       PetscCall(PetscSectionSetFieldDof(newCoordSection, cNew, 0, dof));
524     }
525   }
526   for (v = vStartNew; v < vEndNew; ++v) {
527     PetscCall(PetscSectionSetDof(newCoordSection, v, dim));
528     PetscCall(PetscSectionSetFieldDof(newCoordSection, v, 0, dim));
529   }
530   PetscCall(PetscSectionSetUp(newCoordSection));
531   PetscCall(DMSetCoordinateSection(dmNew, PETSC_DETERMINE, newCoordSection));
532   PetscCall(PetscSectionGetStorageSize(newCoordSection, &coordSize));
533   PetscCall(VecCreate(PETSC_COMM_SELF, &newCoordinates));
534   PetscCall(PetscObjectSetName((PetscObject)newCoordinates, "coordinates"));
535   PetscCall(VecSetSizes(newCoordinates, coordSize, PETSC_DETERMINE));
536   PetscCall(VecSetBlockSize(newCoordinates, dim));
537   PetscCall(VecSetType(newCoordinates, VECSTANDARD));
538   PetscCall(DMSetCoordinatesLocal(dmNew, newCoordinates));
539   PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
540   PetscCall(VecGetArray(coordinates, &coords));
541   PetscCall(VecGetArray(newCoordinates, &newCoords));
542   if (hasCells) {
543     for (c = cStart; c < cEnd; ++c) {
544       PetscInt cNew = DMPlexShiftPoint_Internal(c, depth, depthShift), dof, off, noff, d;
545 
546       PetscCall(PetscSectionGetDof(coordSection, c, &dof));
547       PetscCall(PetscSectionGetOffset(coordSection, c, &off));
548       PetscCall(PetscSectionGetOffset(newCoordSection, cNew, &noff));
549       for (d = 0; d < dof; ++d) newCoords[noff + d] = coords[off + d];
550     }
551   }
552   for (v = vStart; v < vEnd; ++v) {
553     PetscInt dof, off, noff, d;
554 
555     PetscCall(PetscSectionGetDof(coordSection, v, &dof));
556     PetscCall(PetscSectionGetOffset(coordSection, v, &off));
557     PetscCall(PetscSectionGetOffset(newCoordSection, DMPlexShiftPoint_Internal(v, depth, depthShift), &noff));
558     for (d = 0; d < dof; ++d) newCoords[noff + d] = coords[off + d];
559   }
560   PetscCall(VecRestoreArray(coordinates, &coords));
561   PetscCall(VecRestoreArray(newCoordinates, &newCoords));
562   PetscCall(VecDestroy(&newCoordinates));
563   PetscCall(PetscSectionDestroy(&newCoordSection));
564   PetscFunctionReturn(0);
565 }
566 
567 static PetscErrorCode DMPlexShiftSF_Single(DM dm, PetscInt depthShift[], PetscSF sf, PetscSF sfNew) {
568   const PetscSFNode *remotePoints;
569   PetscSFNode       *gremotePoints;
570   const PetscInt    *localPoints;
571   PetscInt          *glocalPoints, *newLocation, *newRemoteLocation;
572   PetscInt           numRoots, numLeaves, l, pStart, pEnd, depth = 0, totShift = 0;
573 
574   PetscFunctionBegin;
575   PetscCall(DMPlexGetDepth(dm, &depth));
576   PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
577   PetscCall(PetscSFGetGraph(sf, &numRoots, &numLeaves, &localPoints, &remotePoints));
578   totShift = DMPlexShiftPoint_Internal(pEnd, depth, depthShift) - pEnd;
579   if (numRoots >= 0) {
580     PetscCall(PetscMalloc2(numRoots, &newLocation, pEnd - pStart, &newRemoteLocation));
581     for (l = 0; l < numRoots; ++l) newLocation[l] = DMPlexShiftPoint_Internal(l, depth, depthShift);
582     PetscCall(PetscSFBcastBegin(sf, MPIU_INT, newLocation, newRemoteLocation, MPI_REPLACE));
583     PetscCall(PetscSFBcastEnd(sf, MPIU_INT, newLocation, newRemoteLocation, MPI_REPLACE));
584     PetscCall(PetscMalloc1(numLeaves, &glocalPoints));
585     PetscCall(PetscMalloc1(numLeaves, &gremotePoints));
586     for (l = 0; l < numLeaves; ++l) {
587       glocalPoints[l]        = DMPlexShiftPoint_Internal(localPoints[l], depth, depthShift);
588       gremotePoints[l].rank  = remotePoints[l].rank;
589       gremotePoints[l].index = newRemoteLocation[localPoints[l]];
590     }
591     PetscCall(PetscFree2(newLocation, newRemoteLocation));
592     PetscCall(PetscSFSetGraph(sfNew, numRoots + totShift, numLeaves, glocalPoints, PETSC_OWN_POINTER, gremotePoints, PETSC_OWN_POINTER));
593   }
594   PetscFunctionReturn(0);
595 }
596 
597 static PetscErrorCode DMPlexShiftSF_Internal(DM dm, PetscInt depthShift[], DM dmNew) {
598   PetscSF   sfPoint, sfPointNew;
599   PetscBool useNatural;
600 
601   PetscFunctionBegin;
602   /* Step 9: Convert pointSF */
603   PetscCall(DMGetPointSF(dm, &sfPoint));
604   PetscCall(DMGetPointSF(dmNew, &sfPointNew));
605   PetscCall(DMPlexShiftSF_Single(dm, depthShift, sfPoint, sfPointNew));
606   /* Step 9b: Convert naturalSF */
607   PetscCall(DMGetUseNatural(dm, &useNatural));
608   if (useNatural) {
609     PetscSF sfNat, sfNatNew;
610 
611     PetscCall(DMSetUseNatural(dmNew, useNatural));
612     PetscCall(DMGetNaturalSF(dm, &sfNat));
613     PetscCall(DMGetNaturalSF(dmNew, &sfNatNew));
614     PetscCall(DMPlexShiftSF_Single(dm, depthShift, sfNat, sfNatNew));
615   }
616   PetscFunctionReturn(0);
617 }
618 
619 static PetscErrorCode DMPlexShiftLabels_Internal(DM dm, PetscInt depthShift[], DM dmNew) {
620   PetscInt depth = 0, numLabels, l;
621 
622   PetscFunctionBegin;
623   PetscCall(DMPlexGetDepth(dm, &depth));
624   /* Step 10: Convert labels */
625   PetscCall(DMGetNumLabels(dm, &numLabels));
626   for (l = 0; l < numLabels; ++l) {
627     DMLabel         label, newlabel;
628     const char     *lname;
629     PetscBool       isDepth, isDim;
630     IS              valueIS;
631     const PetscInt *values;
632     PetscInt        numValues, val;
633 
634     PetscCall(DMGetLabelName(dm, l, &lname));
635     PetscCall(PetscStrcmp(lname, "depth", &isDepth));
636     if (isDepth) continue;
637     PetscCall(PetscStrcmp(lname, "dim", &isDim));
638     if (isDim) continue;
639     PetscCall(DMCreateLabel(dmNew, lname));
640     PetscCall(DMGetLabel(dm, lname, &label));
641     PetscCall(DMGetLabel(dmNew, lname, &newlabel));
642     PetscCall(DMLabelGetDefaultValue(label, &val));
643     PetscCall(DMLabelSetDefaultValue(newlabel, val));
644     PetscCall(DMLabelGetValueIS(label, &valueIS));
645     PetscCall(ISGetLocalSize(valueIS, &numValues));
646     PetscCall(ISGetIndices(valueIS, &values));
647     for (val = 0; val < numValues; ++val) {
648       IS              pointIS;
649       const PetscInt *points;
650       PetscInt        numPoints, p;
651 
652       PetscCall(DMLabelGetStratumIS(label, values[val], &pointIS));
653       PetscCall(ISGetLocalSize(pointIS, &numPoints));
654       PetscCall(ISGetIndices(pointIS, &points));
655       for (p = 0; p < numPoints; ++p) {
656         const PetscInt newpoint = DMPlexShiftPoint_Internal(points[p], depth, depthShift);
657 
658         PetscCall(DMLabelSetValue(newlabel, newpoint, values[val]));
659       }
660       PetscCall(ISRestoreIndices(pointIS, &points));
661       PetscCall(ISDestroy(&pointIS));
662     }
663     PetscCall(ISRestoreIndices(valueIS, &values));
664     PetscCall(ISDestroy(&valueIS));
665   }
666   PetscFunctionReturn(0);
667 }
668 
669 static PetscErrorCode DMPlexCreateVTKLabel_Internal(DM dm, PetscBool createGhostLabel, DM dmNew) {
670   PetscSF            sfPoint;
671   DMLabel            vtkLabel, ghostLabel = NULL;
672   const PetscSFNode *leafRemote;
673   const PetscInt    *leafLocal;
674   PetscInt           cellHeight, cStart, cEnd, c, fStart, fEnd, f, numLeaves, l;
675   PetscMPIInt        rank;
676 
677   PetscFunctionBegin;
678   /* Step 11: Make label for output (vtk) and to mark ghost points (ghost) */
679   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
680   PetscCall(DMGetPointSF(dm, &sfPoint));
681   PetscCall(DMPlexGetVTKCellHeight(dmNew, &cellHeight));
682   PetscCall(DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd));
683   PetscCall(PetscSFGetGraph(sfPoint, NULL, &numLeaves, &leafLocal, &leafRemote));
684   PetscCall(DMCreateLabel(dmNew, "vtk"));
685   PetscCall(DMGetLabel(dmNew, "vtk", &vtkLabel));
686   if (createGhostLabel) {
687     PetscCall(DMCreateLabel(dmNew, "ghost"));
688     PetscCall(DMGetLabel(dmNew, "ghost", &ghostLabel));
689   }
690   for (l = 0, c = cStart; l < numLeaves && c < cEnd; ++l, ++c) {
691     for (; c < leafLocal[l] && c < cEnd; ++c) PetscCall(DMLabelSetValue(vtkLabel, c, 1));
692     if (leafLocal[l] >= cEnd) break;
693     if (leafRemote[l].rank == rank) {
694       PetscCall(DMLabelSetValue(vtkLabel, c, 1));
695     } else if (ghostLabel) PetscCall(DMLabelSetValue(ghostLabel, c, 2));
696   }
697   for (; c < cEnd; ++c) PetscCall(DMLabelSetValue(vtkLabel, c, 1));
698   if (ghostLabel) {
699     PetscCall(DMPlexGetHeightStratum(dmNew, 1, &fStart, &fEnd));
700     for (f = fStart; f < fEnd; ++f) {
701       PetscInt numCells;
702 
703       PetscCall(DMPlexGetSupportSize(dmNew, f, &numCells));
704       if (numCells < 2) {
705         PetscCall(DMLabelSetValue(ghostLabel, f, 1));
706       } else {
707         const PetscInt *cells = NULL;
708         PetscInt        vA, vB;
709 
710         PetscCall(DMPlexGetSupport(dmNew, f, &cells));
711         PetscCall(DMLabelGetValue(vtkLabel, cells[0], &vA));
712         PetscCall(DMLabelGetValue(vtkLabel, cells[1], &vB));
713         if (vA != 1 && vB != 1) PetscCall(DMLabelSetValue(ghostLabel, f, 1));
714       }
715     }
716   }
717   PetscFunctionReturn(0);
718 }
719 
720 static PetscErrorCode DMPlexShiftTree_Internal(DM dm, PetscInt depthShift[], DM dmNew) {
721   DM           refTree;
722   PetscSection pSec;
723   PetscInt    *parents, *childIDs;
724 
725   PetscFunctionBegin;
726   PetscCall(DMPlexGetReferenceTree(dm, &refTree));
727   PetscCall(DMPlexSetReferenceTree(dmNew, refTree));
728   PetscCall(DMPlexGetTree(dm, &pSec, &parents, &childIDs, NULL, NULL));
729   if (pSec) {
730     PetscInt     p, pStart, pEnd, *parentsShifted, pStartShifted, pEndShifted, depth;
731     PetscInt    *childIDsShifted;
732     PetscSection pSecShifted;
733 
734     PetscCall(PetscSectionGetChart(pSec, &pStart, &pEnd));
735     PetscCall(DMPlexGetDepth(dm, &depth));
736     pStartShifted = DMPlexShiftPoint_Internal(pStart, depth, depthShift);
737     pEndShifted   = DMPlexShiftPoint_Internal(pEnd, depth, depthShift);
738     PetscCall(PetscMalloc2(pEndShifted - pStartShifted, &parentsShifted, pEndShifted - pStartShifted, &childIDsShifted));
739     PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dmNew), &pSecShifted));
740     PetscCall(PetscSectionSetChart(pSecShifted, pStartShifted, pEndShifted));
741     for (p = pStartShifted; p < pEndShifted; p++) {
742       /* start off assuming no children */
743       PetscCall(PetscSectionSetDof(pSecShifted, p, 0));
744     }
745     for (p = pStart; p < pEnd; p++) {
746       PetscInt dof;
747       PetscInt pNew = DMPlexShiftPoint_Internal(p, depth, depthShift);
748 
749       PetscCall(PetscSectionGetDof(pSec, p, &dof));
750       PetscCall(PetscSectionSetDof(pSecShifted, pNew, dof));
751     }
752     PetscCall(PetscSectionSetUp(pSecShifted));
753     for (p = pStart; p < pEnd; p++) {
754       PetscInt dof;
755       PetscInt pNew = DMPlexShiftPoint_Internal(p, depth, depthShift);
756 
757       PetscCall(PetscSectionGetDof(pSec, p, &dof));
758       if (dof) {
759         PetscInt off, offNew;
760 
761         PetscCall(PetscSectionGetOffset(pSec, p, &off));
762         PetscCall(PetscSectionGetOffset(pSecShifted, pNew, &offNew));
763         parentsShifted[offNew]  = DMPlexShiftPoint_Internal(parents[off], depth, depthShift);
764         childIDsShifted[offNew] = childIDs[off];
765       }
766     }
767     PetscCall(DMPlexSetTree(dmNew, pSecShifted, parentsShifted, childIDsShifted));
768     PetscCall(PetscFree2(parentsShifted, childIDsShifted));
769     PetscCall(PetscSectionDestroy(&pSecShifted));
770   }
771   PetscFunctionReturn(0);
772 }
773 
774 static PetscErrorCode DMPlexConstructGhostCells_Internal(DM dm, DMLabel label, PetscInt *numGhostCells, DM gdm) {
775   PetscSF          sf;
776   IS               valueIS;
777   const PetscInt  *values, *leaves;
778   PetscInt        *depthShift;
779   PetscInt         d, depth = 0, nleaves, loc, Ng, numFS, fs, fStart, fEnd, ghostCell, cEnd, c;
780   const PetscReal *maxCell, *Lstart, *L;
781 
782   PetscFunctionBegin;
783   PetscCall(DMGetPointSF(dm, &sf));
784   PetscCall(PetscSFGetGraph(sf, NULL, &nleaves, &leaves, NULL));
785   nleaves = PetscMax(0, nleaves);
786   PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
787   /* Count ghost cells */
788   PetscCall(DMLabelGetValueIS(label, &valueIS));
789   PetscCall(ISGetLocalSize(valueIS, &numFS));
790   PetscCall(ISGetIndices(valueIS, &values));
791   Ng = 0;
792   for (fs = 0; fs < numFS; ++fs) {
793     IS              faceIS;
794     const PetscInt *faces;
795     PetscInt        numFaces, f, numBdFaces = 0;
796 
797     PetscCall(DMLabelGetStratumIS(label, values[fs], &faceIS));
798     PetscCall(ISGetLocalSize(faceIS, &numFaces));
799     PetscCall(ISGetIndices(faceIS, &faces));
800     for (f = 0; f < numFaces; ++f) {
801       PetscInt numChildren;
802 
803       PetscCall(PetscFindInt(faces[f], nleaves, leaves, &loc));
804       PetscCall(DMPlexGetTreeChildren(dm, faces[f], &numChildren, NULL));
805       /* non-local and ancestors points don't get to register ghosts */
806       if (loc >= 0 || numChildren) continue;
807       if ((faces[f] >= fStart) && (faces[f] < fEnd)) ++numBdFaces;
808     }
809     Ng += numBdFaces;
810     PetscCall(ISRestoreIndices(faceIS, &faces));
811     PetscCall(ISDestroy(&faceIS));
812   }
813   PetscCall(DMPlexGetDepth(dm, &depth));
814   PetscCall(PetscMalloc1(2 * (depth + 1), &depthShift));
815   for (d = 0; d <= depth; d++) {
816     PetscInt dEnd;
817 
818     PetscCall(DMPlexGetDepthStratum(dm, d, NULL, &dEnd));
819     depthShift[2 * d]     = dEnd;
820     depthShift[2 * d + 1] = 0;
821   }
822   if (depth >= 0) depthShift[2 * depth + 1] = Ng;
823   PetscCall(DMPlexShiftPointSetUp_Internal(depth, depthShift));
824   PetscCall(DMPlexShiftSizes_Internal(dm, depthShift, gdm));
825   /* Step 3: Set cone/support sizes for new points */
826   PetscCall(DMPlexGetHeightStratum(dm, 0, NULL, &cEnd));
827   for (c = cEnd; c < cEnd + Ng; ++c) PetscCall(DMPlexSetConeSize(gdm, c, 1));
828   for (fs = 0; fs < numFS; ++fs) {
829     IS              faceIS;
830     const PetscInt *faces;
831     PetscInt        numFaces, f;
832 
833     PetscCall(DMLabelGetStratumIS(label, values[fs], &faceIS));
834     PetscCall(ISGetLocalSize(faceIS, &numFaces));
835     PetscCall(ISGetIndices(faceIS, &faces));
836     for (f = 0; f < numFaces; ++f) {
837       PetscInt size, numChildren;
838 
839       PetscCall(PetscFindInt(faces[f], nleaves, leaves, &loc));
840       PetscCall(DMPlexGetTreeChildren(dm, faces[f], &numChildren, NULL));
841       if (loc >= 0 || numChildren) continue;
842       if ((faces[f] < fStart) || (faces[f] >= fEnd)) continue;
843       PetscCall(DMPlexGetSupportSize(dm, faces[f], &size));
844       PetscCheck(size == 1, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM has boundary face %" PetscInt_FMT " with %" PetscInt_FMT " support cells", faces[f], size);
845       PetscCall(DMPlexSetSupportSize(gdm, faces[f] + Ng, 2));
846     }
847     PetscCall(ISRestoreIndices(faceIS, &faces));
848     PetscCall(ISDestroy(&faceIS));
849   }
850   /* Step 4: Setup ghosted DM */
851   PetscCall(DMSetUp(gdm));
852   PetscCall(DMPlexShiftPoints_Internal(dm, depthShift, gdm));
853   /* Step 6: Set cones and supports for new points */
854   ghostCell = cEnd;
855   for (fs = 0; fs < numFS; ++fs) {
856     IS              faceIS;
857     const PetscInt *faces;
858     PetscInt        numFaces, f;
859 
860     PetscCall(DMLabelGetStratumIS(label, values[fs], &faceIS));
861     PetscCall(ISGetLocalSize(faceIS, &numFaces));
862     PetscCall(ISGetIndices(faceIS, &faces));
863     for (f = 0; f < numFaces; ++f) {
864       PetscInt newFace = faces[f] + Ng, numChildren;
865 
866       PetscCall(PetscFindInt(faces[f], nleaves, leaves, &loc));
867       PetscCall(DMPlexGetTreeChildren(dm, faces[f], &numChildren, NULL));
868       if (loc >= 0 || numChildren) continue;
869       if ((faces[f] < fStart) || (faces[f] >= fEnd)) continue;
870       PetscCall(DMPlexSetCone(gdm, ghostCell, &newFace));
871       PetscCall(DMPlexInsertSupport(gdm, newFace, 1, ghostCell));
872       ++ghostCell;
873     }
874     PetscCall(ISRestoreIndices(faceIS, &faces));
875     PetscCall(ISDestroy(&faceIS));
876   }
877   PetscCall(ISRestoreIndices(valueIS, &values));
878   PetscCall(ISDestroy(&valueIS));
879   PetscCall(DMPlexShiftCoordinates_Internal(dm, depthShift, gdm));
880   PetscCall(DMPlexShiftSF_Internal(dm, depthShift, gdm));
881   PetscCall(DMPlexShiftLabels_Internal(dm, depthShift, gdm));
882   PetscCall(DMPlexCreateVTKLabel_Internal(dm, PETSC_TRUE, gdm));
883   PetscCall(DMPlexShiftTree_Internal(dm, depthShift, gdm));
884   PetscCall(PetscFree(depthShift));
885   for (c = cEnd; c < cEnd + Ng; ++c) PetscCall(DMPlexSetCellType(gdm, c, DM_POLYTOPE_FV_GHOST));
886   /* Step 7: Periodicity */
887   PetscCall(DMGetPeriodicity(dm, &maxCell, &Lstart, &L));
888   PetscCall(DMSetPeriodicity(gdm, maxCell, Lstart, L));
889   if (numGhostCells) *numGhostCells = Ng;
890   PetscFunctionReturn(0);
891 }
892 
893 /*@C
894   DMPlexConstructGhostCells - Construct ghost cells which connect to every boundary face
895 
896   Collective on dm
897 
898   Input Parameters:
899 + dm - The original DM
900 - labelName - The label specifying the boundary faces, or "Face Sets" if this is NULL
901 
902   Output Parameters:
903 + numGhostCells - The number of ghost cells added to the DM
904 - dmGhosted - The new DM
905 
906   Note: If no label exists of that name, one will be created marking all boundary faces
907 
908   Level: developer
909 
910 .seealso: `DMCreate()`
911 @*/
912 PetscErrorCode DMPlexConstructGhostCells(DM dm, const char labelName[], PetscInt *numGhostCells, DM *dmGhosted) {
913   DM          gdm;
914   DMLabel     label;
915   const char *name = labelName ? labelName : "Face Sets";
916   PetscInt    dim, Ng = 0;
917   PetscBool   useCone, useClosure;
918 
919   PetscFunctionBegin;
920   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
921   if (numGhostCells) PetscValidIntPointer(numGhostCells, 3);
922   PetscValidPointer(dmGhosted, 4);
923   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &gdm));
924   PetscCall(DMSetType(gdm, DMPLEX));
925   PetscCall(DMGetDimension(dm, &dim));
926   PetscCall(DMSetDimension(gdm, dim));
927   PetscCall(DMGetBasicAdjacency(dm, &useCone, &useClosure));
928   PetscCall(DMSetBasicAdjacency(gdm, useCone, useClosure));
929   PetscCall(DMGetLabel(dm, name, &label));
930   if (!label) {
931     /* Get label for boundary faces */
932     PetscCall(DMCreateLabel(dm, name));
933     PetscCall(DMGetLabel(dm, name, &label));
934     PetscCall(DMPlexMarkBoundaryFaces(dm, 1, label));
935   }
936   PetscCall(DMPlexConstructGhostCells_Internal(dm, label, &Ng, gdm));
937   PetscCall(DMCopyDisc(dm, gdm));
938   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_TRUE, gdm));
939   gdm->setfromoptionscalled = dm->setfromoptionscalled;
940   if (numGhostCells) *numGhostCells = Ng;
941   *dmGhosted = gdm;
942   PetscFunctionReturn(0);
943 }
944 
945 static PetscErrorCode DivideCells_Private(DM dm, DMLabel label, DMPlexPointQueue queue) {
946   PetscInt dim, d, shift = 100, *pStart, *pEnd;
947 
948   PetscFunctionBegin;
949   PetscCall(DMGetDimension(dm, &dim));
950   PetscCall(PetscMalloc2(dim, &pStart, dim, &pEnd));
951   for (d = 0; d < dim; ++d) PetscCall(DMPlexGetDepthStratum(dm, d, &pStart[d], &pEnd[d]));
952   while (!DMPlexPointQueueEmpty(queue)) {
953     PetscInt  cell    = -1;
954     PetscInt *closure = NULL;
955     PetscInt  closureSize, cl, cval;
956 
957     PetscCall(DMPlexPointQueueDequeue(queue, &cell));
958     PetscCall(DMLabelGetValue(label, cell, &cval));
959     PetscCall(DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
960     /* Mark points in the cell closure that touch the fault */
961     for (d = 0; d < dim; ++d) {
962       for (cl = 0; cl < closureSize * 2; cl += 2) {
963         const PetscInt clp = closure[cl];
964         PetscInt       clval;
965 
966         if ((clp < pStart[d]) || (clp >= pEnd[d])) continue;
967         PetscCall(DMLabelGetValue(label, clp, &clval));
968         if (clval == -1) {
969           const PetscInt *cone;
970           PetscInt        coneSize, c;
971 
972           /* If a cone point touches the fault, then this point touches the fault */
973           PetscCall(DMPlexGetCone(dm, clp, &cone));
974           PetscCall(DMPlexGetConeSize(dm, clp, &coneSize));
975           for (c = 0; c < coneSize; ++c) {
976             PetscInt cpval;
977 
978             PetscCall(DMLabelGetValue(label, cone[c], &cpval));
979             if (cpval != -1) {
980               PetscInt dep;
981 
982               PetscCall(DMPlexGetPointDepth(dm, clp, &dep));
983               clval = cval < 0 ? -(shift + dep) : shift + dep;
984               PetscCall(DMLabelSetValue(label, clp, clval));
985               break;
986             }
987           }
988         }
989         /* Mark neighbor cells through marked faces (these cells must also touch the fault) */
990         if (d == dim - 1 && clval != -1) {
991           const PetscInt *support;
992           PetscInt        supportSize, s, nval;
993 
994           PetscCall(DMPlexGetSupport(dm, clp, &support));
995           PetscCall(DMPlexGetSupportSize(dm, clp, &supportSize));
996           for (s = 0; s < supportSize; ++s) {
997             PetscCall(DMLabelGetValue(label, support[s], &nval));
998             if (nval == -1) {
999               PetscCall(DMLabelSetValue(label, support[s], clval < 0 ? clval - 1 : clval + 1));
1000               PetscCall(DMPlexPointQueueEnqueue(queue, support[s]));
1001             }
1002           }
1003         }
1004       }
1005     }
1006     PetscCall(DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
1007   }
1008   PetscCall(PetscFree2(pStart, pEnd));
1009   PetscFunctionReturn(0);
1010 }
1011 
1012 typedef struct {
1013   DM               dm;
1014   DMPlexPointQueue queue;
1015 } PointDivision;
1016 
1017 static PetscErrorCode divideCell(DMLabel label, PetscInt p, PetscInt val, void *ctx) {
1018   PointDivision  *div  = (PointDivision *)ctx;
1019   PetscInt        cval = val < 0 ? val - 1 : val + 1;
1020   const PetscInt *support;
1021   PetscInt        supportSize, s;
1022 
1023   PetscFunctionBegin;
1024   PetscCall(DMPlexGetSupport(div->dm, p, &support));
1025   PetscCall(DMPlexGetSupportSize(div->dm, p, &supportSize));
1026   for (s = 0; s < supportSize; ++s) {
1027     PetscCall(DMLabelSetValue(label, support[s], cval));
1028     PetscCall(DMPlexPointQueueEnqueue(div->queue, support[s]));
1029   }
1030   PetscFunctionReturn(0);
1031 }
1032 
1033 /* Mark cells by label propagation */
1034 static PetscErrorCode DMPlexLabelFaultHalo(DM dm, DMLabel faultLabel) {
1035   DMPlexPointQueue queue = NULL;
1036   PointDivision    div;
1037   PetscSF          pointSF;
1038   IS               pointIS;
1039   const PetscInt  *points;
1040   PetscBool        empty;
1041   PetscInt         dim, shift = 100, n, i;
1042 
1043   PetscFunctionBegin;
1044   PetscCall(DMGetDimension(dm, &dim));
1045   PetscCall(DMPlexPointQueueCreate(1024, &queue));
1046   div.dm    = dm;
1047   div.queue = queue;
1048   /* Enqueue cells on fault */
1049   PetscCall(DMLabelGetStratumIS(faultLabel, shift + dim, &pointIS));
1050   if (pointIS) {
1051     PetscCall(ISGetLocalSize(pointIS, &n));
1052     PetscCall(ISGetIndices(pointIS, &points));
1053     for (i = 0; i < n; ++i) PetscCall(DMPlexPointQueueEnqueue(queue, points[i]));
1054     PetscCall(ISRestoreIndices(pointIS, &points));
1055     PetscCall(ISDestroy(&pointIS));
1056   }
1057   PetscCall(DMLabelGetStratumIS(faultLabel, -(shift + dim), &pointIS));
1058   if (pointIS) {
1059     PetscCall(ISGetLocalSize(pointIS, &n));
1060     PetscCall(ISGetIndices(pointIS, &points));
1061     for (i = 0; i < n; ++i) PetscCall(DMPlexPointQueueEnqueue(queue, points[i]));
1062     PetscCall(ISRestoreIndices(pointIS, &points));
1063     PetscCall(ISDestroy(&pointIS));
1064   }
1065 
1066   PetscCall(DMGetPointSF(dm, &pointSF));
1067   PetscCall(DMLabelPropagateBegin(faultLabel, pointSF));
1068   /* While edge queue is not empty: */
1069   PetscCall(DMPlexPointQueueEmptyCollective((PetscObject)dm, queue, &empty));
1070   while (!empty) {
1071     PetscCall(DivideCells_Private(dm, faultLabel, queue));
1072     PetscCall(DMLabelPropagatePush(faultLabel, pointSF, divideCell, &div));
1073     PetscCall(DMPlexPointQueueEmptyCollective((PetscObject)dm, queue, &empty));
1074   }
1075   PetscCall(DMLabelPropagateEnd(faultLabel, pointSF));
1076   PetscCall(DMPlexPointQueueDestroy(&queue));
1077   PetscFunctionReturn(0);
1078 }
1079 
1080 /*
1081   We are adding three kinds of points here:
1082     Replicated:     Copies of points which exist in the mesh, such as vertices identified across a fault
1083     Non-replicated: Points which exist on the fault, but are not replicated
1084     Ghost:          These are shared fault faces which are not owned by this process. These do not produce hybrid cells and do not replicate
1085     Hybrid:         Entirely new points, such as cohesive cells
1086 
1087   When creating subsequent cohesive cells, we shift the old hybrid cells to the end of the numbering at
1088   each depth so that the new split/hybrid points can be inserted as a block.
1089 */
1090 static PetscErrorCode DMPlexConstructCohesiveCells_Internal(DM dm, DMLabel label, DMLabel splitLabel, DM sdm) {
1091   MPI_Comm         comm;
1092   IS               valueIS;
1093   PetscInt         numSP = 0; /* The number of depths for which we have replicated points */
1094   const PetscInt  *values;    /* List of depths for which we have replicated points */
1095   IS              *splitIS;
1096   IS              *unsplitIS;
1097   IS               ghostIS;
1098   PetscInt        *numSplitPoints;     /* The number of replicated points at each depth */
1099   PetscInt        *numUnsplitPoints;   /* The number of non-replicated points at each depth which still give rise to hybrid points */
1100   PetscInt        *numHybridPoints;    /* The number of new hybrid points at each depth */
1101   PetscInt        *numHybridPointsOld; /* The number of existing hybrid points at each depth */
1102   PetscInt         numGhostPoints;     /* The number of unowned, shared fault faces */
1103   const PetscInt **splitPoints;        /* Replicated points for each depth */
1104   const PetscInt **unsplitPoints;      /* Non-replicated points for each depth */
1105   const PetscInt  *ghostPoints;        /* Ghost fault faces */
1106   PetscSection     coordSection;
1107   Vec              coordinates;
1108   PetscScalar     *coords;
1109   PetscInt        *depthMax;   /* The first hybrid point at each depth in the original mesh */
1110   PetscInt        *depthEnd;   /* The point limit at each depth in the original mesh */
1111   PetscInt        *depthShift; /* Number of replicated+hybrid points at each depth */
1112   PetscInt        *pMaxNew;    /* The first replicated point at each depth in the new mesh, hybrids come after this */
1113   PetscInt        *coneNew, *coneONew, *supportNew;
1114   PetscInt         shift = 100, shift2 = 200, depth = 0, dep, dim, d, sp, maxConeSize, maxSupportSize, maxConeSizeNew, maxSupportSizeNew, numLabels, vStart, vEnd, pEnd, p, v;
1115 
1116   PetscFunctionBegin;
1117   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
1118   PetscCall(DMGetDimension(dm, &dim));
1119   PetscCall(DMPlexGetDepth(dm, &depth));
1120   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
1121   /* We do not want this label automatically computed, instead we compute it here */
1122   PetscCall(DMCreateLabel(sdm, "celltype"));
1123   /* Count split points and add cohesive cells */
1124   PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize));
1125   PetscCall(PetscMalloc5(depth + 1, &depthMax, depth + 1, &depthEnd, 2 * (depth + 1), &depthShift, depth + 1, &pMaxNew, depth + 1, &numHybridPointsOld));
1126   PetscCall(PetscMalloc7(depth + 1, &splitIS, depth + 1, &unsplitIS, depth + 1, &numSplitPoints, depth + 1, &numUnsplitPoints, depth + 1, &numHybridPoints, depth + 1, &splitPoints, depth + 1, &unsplitPoints));
1127   for (d = 0; d <= depth; ++d) {
1128     PetscCall(DMPlexGetDepthStratum(dm, d, NULL, &pMaxNew[d]));
1129     PetscCall(DMPlexGetTensorPrismBounds_Internal(dm, d, &depthMax[d], NULL));
1130     depthEnd[d]           = pMaxNew[d];
1131     depthMax[d]           = depthMax[d] < 0 ? depthEnd[d] : depthMax[d];
1132     numSplitPoints[d]     = 0;
1133     numUnsplitPoints[d]   = 0;
1134     numHybridPoints[d]    = 0;
1135     numHybridPointsOld[d] = depthMax[d] < 0 ? 0 : depthEnd[d] - depthMax[d];
1136     splitPoints[d]        = NULL;
1137     unsplitPoints[d]      = NULL;
1138     splitIS[d]            = NULL;
1139     unsplitIS[d]          = NULL;
1140     /* we are shifting the existing hybrid points with the stratum behind them, so
1141      * the split comes at the end of the normal points, i.e., at depthMax[d] */
1142     depthShift[2 * d]     = depthMax[d];
1143     depthShift[2 * d + 1] = 0;
1144   }
1145   numGhostPoints = 0;
1146   ghostPoints    = NULL;
1147   if (label) {
1148     PetscCall(DMLabelGetValueIS(label, &valueIS));
1149     PetscCall(ISGetLocalSize(valueIS, &numSP));
1150     PetscCall(ISGetIndices(valueIS, &values));
1151   }
1152   for (sp = 0; sp < numSP; ++sp) {
1153     const PetscInt dep = values[sp];
1154 
1155     if ((dep < 0) || (dep > depth)) continue;
1156     PetscCall(DMLabelGetStratumIS(label, dep, &splitIS[dep]));
1157     if (splitIS[dep]) {
1158       PetscCall(ISGetLocalSize(splitIS[dep], &numSplitPoints[dep]));
1159       PetscCall(ISGetIndices(splitIS[dep], &splitPoints[dep]));
1160     }
1161     PetscCall(DMLabelGetStratumIS(label, shift2 + dep, &unsplitIS[dep]));
1162     if (unsplitIS[dep]) {
1163       PetscCall(ISGetLocalSize(unsplitIS[dep], &numUnsplitPoints[dep]));
1164       PetscCall(ISGetIndices(unsplitIS[dep], &unsplitPoints[dep]));
1165     }
1166   }
1167   PetscCall(DMLabelGetStratumIS(label, shift2 + dim - 1, &ghostIS));
1168   if (ghostIS) {
1169     PetscCall(ISGetLocalSize(ghostIS, &numGhostPoints));
1170     PetscCall(ISGetIndices(ghostIS, &ghostPoints));
1171   }
1172   /* Calculate number of hybrid points */
1173   for (d = 1; d <= depth; ++d) numHybridPoints[d] = numSplitPoints[d - 1] + numUnsplitPoints[d - 1]; /* There is a hybrid cell/face/edge for every split face/edge/vertex   */
1174   for (d = 0; d <= depth; ++d) depthShift[2 * d + 1] = numSplitPoints[d] + numHybridPoints[d];
1175   PetscCall(DMPlexShiftPointSetUp_Internal(depth, depthShift));
1176   /* the end of the points in this stratum that come before the new points:
1177    * shifting pMaxNew[d] gets the new start of the next stratum, then count back the old hybrid points and the newly
1178    * added points */
1179   for (d = 0; d <= depth; ++d) pMaxNew[d] = DMPlexShiftPoint_Internal(pMaxNew[d], depth, depthShift) - (numHybridPointsOld[d] + numSplitPoints[d] + numHybridPoints[d]);
1180   PetscCall(DMPlexShiftSizes_Internal(dm, depthShift, sdm));
1181   /* Step 3: Set cone/support sizes for new points */
1182   for (dep = 0; dep <= depth; ++dep) {
1183     for (p = 0; p < numSplitPoints[dep]; ++p) {
1184       const PetscInt  oldp   = splitPoints[dep][p];
1185       const PetscInt  newp   = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*oldp + depthOffset[dep]*/;
1186       const PetscInt  splitp = p + pMaxNew[dep];
1187       const PetscInt *support;
1188       DMPolytopeType  ct;
1189       PetscInt        coneSize, supportSize, qf, qn, qp, e;
1190 
1191       PetscCall(DMPlexGetConeSize(dm, oldp, &coneSize));
1192       PetscCall(DMPlexSetConeSize(sdm, splitp, coneSize));
1193       PetscCall(DMPlexGetSupportSize(dm, oldp, &supportSize));
1194       PetscCall(DMPlexSetSupportSize(sdm, splitp, supportSize));
1195       PetscCall(DMPlexGetCellType(dm, oldp, &ct));
1196       PetscCall(DMPlexSetCellType(sdm, splitp, ct));
1197       if (dep == depth - 1) {
1198         const PetscInt hybcell = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1];
1199 
1200         /* Add cohesive cells, they are prisms */
1201         PetscCall(DMPlexSetConeSize(sdm, hybcell, 2 + coneSize));
1202         switch (coneSize) {
1203         case 2: PetscCall(DMPlexSetCellType(sdm, hybcell, DM_POLYTOPE_SEG_PRISM_TENSOR)); break;
1204         case 3: PetscCall(DMPlexSetCellType(sdm, hybcell, DM_POLYTOPE_TRI_PRISM_TENSOR)); break;
1205         case 4: PetscCall(DMPlexSetCellType(sdm, hybcell, DM_POLYTOPE_QUAD_PRISM_TENSOR)); break;
1206         }
1207         /* Shared fault faces with only one support cell now have two with the cohesive cell */
1208         /*   TODO Check thaat oldp has rootdegree == 1 */
1209         if (supportSize == 1) {
1210           const PetscInt *support;
1211           PetscInt        val;
1212 
1213           PetscCall(DMPlexGetSupport(dm, oldp, &support));
1214           PetscCall(DMLabelGetValue(label, support[0], &val));
1215           if (val < 0) PetscCall(DMPlexSetSupportSize(sdm, splitp, 2));
1216           else PetscCall(DMPlexSetSupportSize(sdm, newp, 2));
1217         }
1218       } else if (dep == 0) {
1219         const PetscInt hybedge = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1];
1220 
1221         PetscCall(DMPlexGetSupport(dm, oldp, &support));
1222         for (e = 0, qn = 0, qp = 0, qf = 0; e < supportSize; ++e) {
1223           PetscInt val;
1224 
1225           PetscCall(DMLabelGetValue(label, support[e], &val));
1226           if (val == 1) ++qf;
1227           if ((val == 1) || (val == (shift + 1))) ++qn;
1228           if ((val == 1) || (val == -(shift + 1))) ++qp;
1229         }
1230         /* Split old vertex: Edges into original vertex and new cohesive edge */
1231         PetscCall(DMPlexSetSupportSize(sdm, newp, qn + 1));
1232         /* Split new vertex: Edges into split vertex and new cohesive edge */
1233         PetscCall(DMPlexSetSupportSize(sdm, splitp, qp + 1));
1234         /* Add hybrid edge */
1235         PetscCall(DMPlexSetConeSize(sdm, hybedge, 2));
1236         PetscCall(DMPlexSetSupportSize(sdm, hybedge, qf));
1237         PetscCall(DMPlexSetCellType(sdm, hybedge, DM_POLYTOPE_POINT_PRISM_TENSOR));
1238       } else if (dep == dim - 2) {
1239         const PetscInt hybface = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1];
1240 
1241         PetscCall(DMPlexGetSupport(dm, oldp, &support));
1242         for (e = 0, qn = 0, qp = 0, qf = 0; e < supportSize; ++e) {
1243           PetscInt val;
1244 
1245           PetscCall(DMLabelGetValue(label, support[e], &val));
1246           if (val == dim - 1) ++qf;
1247           if ((val == dim - 1) || (val == (shift + dim - 1))) ++qn;
1248           if ((val == dim - 1) || (val == -(shift + dim - 1))) ++qp;
1249         }
1250         /* Split old edge: Faces into original edge and cohesive face (positive side?) */
1251         PetscCall(DMPlexSetSupportSize(sdm, newp, qn + 1));
1252         /* Split new edge: Faces into split edge and cohesive face (negative side?) */
1253         PetscCall(DMPlexSetSupportSize(sdm, splitp, qp + 1));
1254         /* Add hybrid face */
1255         PetscCall(DMPlexSetConeSize(sdm, hybface, 4));
1256         PetscCall(DMPlexSetSupportSize(sdm, hybface, qf));
1257         PetscCall(DMPlexSetCellType(sdm, hybface, DM_POLYTOPE_SEG_PRISM_TENSOR));
1258       }
1259     }
1260   }
1261   for (dep = 0; dep <= depth; ++dep) {
1262     for (p = 0; p < numUnsplitPoints[dep]; ++p) {
1263       const PetscInt  oldp = unsplitPoints[dep][p];
1264       const PetscInt  newp = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*oldp + depthOffset[dep]*/;
1265       const PetscInt *support;
1266       PetscInt        coneSize, supportSize, qf, e, s;
1267 
1268       PetscCall(DMPlexGetConeSize(dm, oldp, &coneSize));
1269       PetscCall(DMPlexGetSupportSize(dm, oldp, &supportSize));
1270       PetscCall(DMPlexGetSupport(dm, oldp, &support));
1271       if (dep == 0) {
1272         const PetscInt hybedge = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1] + numSplitPoints[dep];
1273 
1274         /* Unsplit vertex: Edges into original vertex, split edges, and new cohesive edge twice */
1275         for (s = 0, qf = 0; s < supportSize; ++s, ++qf) {
1276           PetscCall(PetscFindInt(support[s], numSplitPoints[dep + 1], splitPoints[dep + 1], &e));
1277           if (e >= 0) ++qf;
1278         }
1279         PetscCall(DMPlexSetSupportSize(sdm, newp, qf + 2));
1280         /* Add hybrid edge */
1281         PetscCall(DMPlexSetConeSize(sdm, hybedge, 2));
1282         for (e = 0, qf = 0; e < supportSize; ++e) {
1283           PetscInt val;
1284 
1285           PetscCall(DMLabelGetValue(label, support[e], &val));
1286           /* Split and unsplit edges produce hybrid faces */
1287           if (val == 1) ++qf;
1288           if (val == (shift2 + 1)) ++qf;
1289         }
1290         PetscCall(DMPlexSetSupportSize(sdm, hybedge, qf));
1291         PetscCall(DMPlexSetCellType(sdm, hybedge, DM_POLYTOPE_POINT_PRISM_TENSOR));
1292       } else if (dep == dim - 2) {
1293         const PetscInt hybface = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1] + numSplitPoints[dep];
1294         PetscInt       val;
1295 
1296         for (e = 0, qf = 0; e < supportSize; ++e) {
1297           PetscCall(DMLabelGetValue(label, support[e], &val));
1298           if (val == dim - 1) qf += 2;
1299           else ++qf;
1300         }
1301         /* Unsplit edge: Faces into original edge, split face, and cohesive face twice */
1302         PetscCall(DMPlexSetSupportSize(sdm, newp, qf + 2));
1303         /* Add hybrid face */
1304         for (e = 0, qf = 0; e < supportSize; ++e) {
1305           PetscCall(DMLabelGetValue(label, support[e], &val));
1306           if (val == dim - 1) ++qf;
1307         }
1308         PetscCall(DMPlexSetConeSize(sdm, hybface, 4));
1309         PetscCall(DMPlexSetSupportSize(sdm, hybface, qf));
1310         PetscCall(DMPlexSetCellType(sdm, hybface, DM_POLYTOPE_SEG_PRISM_TENSOR));
1311       }
1312     }
1313   }
1314   /* Step 4: Setup split DM */
1315   PetscCall(DMSetUp(sdm));
1316   PetscCall(DMPlexShiftPoints_Internal(dm, depthShift, sdm));
1317   PetscCall(DMPlexGetMaxSizes(sdm, &maxConeSizeNew, &maxSupportSizeNew));
1318   PetscCall(PetscMalloc3(PetscMax(maxConeSize, maxConeSizeNew) * 3, &coneNew, PetscMax(maxConeSize, maxConeSizeNew) * 3, &coneONew, PetscMax(maxSupportSize, maxSupportSizeNew), &supportNew));
1319   /* Step 6: Set cones and supports for new points */
1320   for (dep = 0; dep <= depth; ++dep) {
1321     for (p = 0; p < numSplitPoints[dep]; ++p) {
1322       const PetscInt  oldp   = splitPoints[dep][p];
1323       const PetscInt  newp   = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*oldp + depthOffset[dep]*/;
1324       const PetscInt  splitp = p + pMaxNew[dep];
1325       const PetscInt *cone, *support, *ornt;
1326       DMPolytopeType  ct;
1327       PetscInt        coneSize, supportSize, q, qf, qn, qp, v, e, s;
1328 
1329       PetscCall(DMPlexGetCellType(dm, oldp, &ct));
1330       PetscCall(DMPlexGetConeSize(dm, oldp, &coneSize));
1331       PetscCall(DMPlexGetCone(dm, oldp, &cone));
1332       PetscCall(DMPlexGetConeOrientation(dm, oldp, &ornt));
1333       PetscCall(DMPlexGetSupportSize(dm, oldp, &supportSize));
1334       PetscCall(DMPlexGetSupport(dm, oldp, &support));
1335       if (dep == depth - 1) {
1336         PetscBool       hasUnsplit = PETSC_FALSE;
1337         const PetscInt  hybcell    = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1];
1338         const PetscInt *supportF;
1339 
1340         coneONew[0] = coneONew[1] = -1000;
1341         /* Split face:       copy in old face to new face to start */
1342         PetscCall(DMPlexGetSupport(sdm, newp, &supportF));
1343         PetscCall(DMPlexSetSupport(sdm, splitp, supportF));
1344         /* Split old face:   old vertices/edges in cone so no change */
1345         /* Split new face:   new vertices/edges in cone */
1346         for (q = 0; q < coneSize; ++q) {
1347           PetscCall(PetscFindInt(cone[q], numSplitPoints[dep - 1], splitPoints[dep - 1], &v));
1348           if (v < 0) {
1349             PetscCall(PetscFindInt(cone[q], numUnsplitPoints[dep - 1], unsplitPoints[dep - 1], &v));
1350             PetscCheck(v >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %" PetscInt_FMT " in split or unsplit points of depth %" PetscInt_FMT, cone[q], dep - 1);
1351             coneNew[2 + q] = DMPlexShiftPoint_Internal(cone[q], depth, depthShift) /*cone[q] + depthOffset[dep-1]*/;
1352             hasUnsplit     = PETSC_TRUE;
1353           } else {
1354             coneNew[2 + q] = v + pMaxNew[dep - 1];
1355             if (dep > 1) {
1356               const PetscInt *econe;
1357               PetscInt        econeSize, r, vs, vu;
1358 
1359               PetscCall(DMPlexGetConeSize(dm, cone[q], &econeSize));
1360               PetscCall(DMPlexGetCone(dm, cone[q], &econe));
1361               for (r = 0; r < econeSize; ++r) {
1362                 PetscCall(PetscFindInt(econe[r], numSplitPoints[dep - 2], splitPoints[dep - 2], &vs));
1363                 PetscCall(PetscFindInt(econe[r], numUnsplitPoints[dep - 2], unsplitPoints[dep - 2], &vu));
1364                 if (vs >= 0) continue;
1365                 PetscCheck(vu >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %" PetscInt_FMT " in split or unsplit points of depth %" PetscInt_FMT, econe[r], dep - 2);
1366                 hasUnsplit = PETSC_TRUE;
1367               }
1368             }
1369           }
1370         }
1371         PetscCall(DMPlexSetCone(sdm, splitp, &coneNew[2]));
1372         PetscCall(DMPlexSetConeOrientation(sdm, splitp, ornt));
1373         /* Face support */
1374         PetscInt vals[2];
1375 
1376         PetscCall(DMLabelGetValue(label, support[0], &vals[0]));
1377         if (supportSize > 1) PetscCall(DMLabelGetValue(label, support[1], &vals[1]));
1378         else vals[1] = -vals[0];
1379         PetscCheck(vals[0] * vals[1] < 0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid support labels %" PetscInt_FMT " %" PetscInt_FMT, vals[0], vals[1]);
1380 
1381         for (s = 0; s < 2; ++s) {
1382           if (s >= supportSize) {
1383             if (vals[s] < 0) {
1384               /* Ghost old face:   Replace negative side cell with cohesive cell */
1385               PetscCall(DMPlexInsertSupport(sdm, newp, s, hybcell));
1386             } else {
1387               /* Ghost new face:   Replace positive side cell with cohesive cell */
1388               PetscCall(DMPlexInsertSupport(sdm, splitp, s, hybcell));
1389             }
1390           } else {
1391             if (vals[s] < 0) {
1392               /* Split old face:   Replace negative side cell with cohesive cell */
1393               PetscCall(DMPlexInsertSupport(sdm, newp, s, hybcell));
1394             } else {
1395               /* Split new face:   Replace positive side cell with cohesive cell */
1396               PetscCall(DMPlexInsertSupport(sdm, splitp, s, hybcell));
1397             }
1398           }
1399         }
1400         /* Get orientation for cohesive face using the positive side cell */
1401         {
1402           const PetscInt *ncone, *nconeO;
1403           PetscInt        nconeSize, nc, ocell;
1404           PetscBool       flip = PETSC_FALSE;
1405 
1406           if (supportSize > 1) {
1407             ocell = vals[0] < 0 ? support[1] : support[0];
1408           } else {
1409             ocell = support[0];
1410             flip  = vals[0] < 0 ? PETSC_TRUE : PETSC_FALSE;
1411           }
1412           PetscCall(DMPlexGetConeSize(dm, ocell, &nconeSize));
1413           PetscCall(DMPlexGetCone(dm, ocell, &ncone));
1414           PetscCall(DMPlexGetConeOrientation(dm, ocell, &nconeO));
1415           for (nc = 0; nc < nconeSize; ++nc) {
1416             if (ncone[nc] == oldp) {
1417               coneONew[0] = flip ? -(nconeO[nc] + 1) : nconeO[nc];
1418               break;
1419             }
1420           }
1421           PetscCheck(nc < nconeSize, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate face %" PetscInt_FMT " in neighboring cell %" PetscInt_FMT, oldp, ocell);
1422         }
1423         /* Cohesive cell:    Old and new split face, then new cohesive faces */
1424         {
1425           const PetscInt No = DMPolytopeTypeGetNumArrangments(ct) / 2;
1426           PetscCheck((coneONew[0] >= -No) && (coneONew[0] < No), PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid %s orientation %" PetscInt_FMT, DMPolytopeTypes[ct], coneONew[0]);
1427         }
1428         const PetscInt *arr = DMPolytopeTypeGetArrangment(ct, coneONew[0]);
1429 
1430         coneNew[0]  = newp; /* Extracted negative side orientation above */
1431         coneNew[1]  = splitp;
1432         coneONew[1] = coneONew[0];
1433         for (q = 0; q < coneSize; ++q) {
1434           /* Hybrid faces must follow order from oriented end face */
1435           const PetscInt qa = arr[q * 2 + 0];
1436           const PetscInt qo = arr[q * 2 + 1];
1437           DMPolytopeType ft = dep == 2 ? DM_POLYTOPE_SEGMENT : DM_POLYTOPE_POINT;
1438 
1439           PetscCall(PetscFindInt(cone[qa], numSplitPoints[dep - 1], splitPoints[dep - 1], &v));
1440           if (v < 0) {
1441             PetscCall(PetscFindInt(cone[qa], numUnsplitPoints[dep - 1], unsplitPoints[dep - 1], &v));
1442             coneNew[2 + q] = v + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep - 1];
1443           } else {
1444             coneNew[2 + q] = v + pMaxNew[dep] + numSplitPoints[dep];
1445           }
1446           coneONew[2 + q] = DMPolytopeTypeComposeOrientation(ft, qo, ornt[qa]);
1447         }
1448         PetscCall(DMPlexSetCone(sdm, hybcell, coneNew));
1449         PetscCall(DMPlexSetConeOrientation(sdm, hybcell, coneONew));
1450         /* Label the hybrid cells on the boundary of the split */
1451         if (hasUnsplit) PetscCall(DMLabelSetValue(label, -hybcell, dim));
1452       } else if (dep == 0) {
1453         const PetscInt hybedge = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1];
1454 
1455         /* Split old vertex: Edges in old split faces and new cohesive edge */
1456         for (e = 0, qn = 0; e < supportSize; ++e) {
1457           PetscInt val;
1458 
1459           PetscCall(DMLabelGetValue(label, support[e], &val));
1460           if ((val == 1) || (val == (shift + 1))) supportNew[qn++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1461         }
1462         supportNew[qn] = hybedge;
1463         PetscCall(DMPlexSetSupport(sdm, newp, supportNew));
1464         /* Split new vertex: Edges in new split faces and new cohesive edge */
1465         for (e = 0, qp = 0; e < supportSize; ++e) {
1466           PetscInt val, edge;
1467 
1468           PetscCall(DMLabelGetValue(label, support[e], &val));
1469           if (val == 1) {
1470             PetscCall(PetscFindInt(support[e], numSplitPoints[dep + 1], splitPoints[dep + 1], &edge));
1471             PetscCheck(edge >= 0, comm, PETSC_ERR_ARG_WRONG, "Edge %" PetscInt_FMT " is not a split edge", support[e]);
1472             supportNew[qp++] = edge + pMaxNew[dep + 1];
1473           } else if (val == -(shift + 1)) {
1474             supportNew[qp++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1475           }
1476         }
1477         supportNew[qp] = hybedge;
1478         PetscCall(DMPlexSetSupport(sdm, splitp, supportNew));
1479         /* Hybrid edge:    Old and new split vertex */
1480         coneNew[0] = newp;
1481         coneNew[1] = splitp;
1482         PetscCall(DMPlexSetCone(sdm, hybedge, coneNew));
1483         for (e = 0, qf = 0; e < supportSize; ++e) {
1484           PetscInt val, edge;
1485 
1486           PetscCall(DMLabelGetValue(label, support[e], &val));
1487           if (val == 1) {
1488             PetscCall(PetscFindInt(support[e], numSplitPoints[dep + 1], splitPoints[dep + 1], &edge));
1489             PetscCheck(edge >= 0, comm, PETSC_ERR_ARG_WRONG, "Edge %" PetscInt_FMT " is not a split edge", support[e]);
1490             supportNew[qf++] = edge + pMaxNew[dep + 2] + numSplitPoints[dep + 2];
1491           }
1492         }
1493         PetscCall(DMPlexSetSupport(sdm, hybedge, supportNew));
1494       } else if (dep == dim - 2) {
1495         const PetscInt hybface = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1];
1496 
1497         /* Split old edge:   old vertices in cone so no change */
1498         /* Split new edge:   new vertices in cone */
1499         for (q = 0; q < coneSize; ++q) {
1500           PetscCall(PetscFindInt(cone[q], numSplitPoints[dep - 1], splitPoints[dep - 1], &v));
1501           if (v < 0) {
1502             PetscCall(PetscFindInt(cone[q], numUnsplitPoints[dep - 1], unsplitPoints[dep - 1], &v));
1503             PetscCheck(v >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %" PetscInt_FMT " in split or unsplit points of depth %" PetscInt_FMT, cone[q], dep - 1);
1504             coneNew[q] = DMPlexShiftPoint_Internal(cone[q], depth, depthShift) /*cone[q] + depthOffset[dep-1]*/;
1505           } else {
1506             coneNew[q] = v + pMaxNew[dep - 1];
1507           }
1508         }
1509         PetscCall(DMPlexSetCone(sdm, splitp, coneNew));
1510         /* Split old edge: Faces in positive side cells and old split faces */
1511         for (e = 0, q = 0; e < supportSize; ++e) {
1512           PetscInt val;
1513 
1514           PetscCall(DMLabelGetValue(label, support[e], &val));
1515           if (val == dim - 1) {
1516             supportNew[q++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1517           } else if (val == (shift + dim - 1)) {
1518             supportNew[q++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1519           }
1520         }
1521         supportNew[q++] = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1];
1522         PetscCall(DMPlexSetSupport(sdm, newp, supportNew));
1523         /* Split new edge: Faces in negative side cells and new split faces */
1524         for (e = 0, q = 0; e < supportSize; ++e) {
1525           PetscInt val, face;
1526 
1527           PetscCall(DMLabelGetValue(label, support[e], &val));
1528           if (val == dim - 1) {
1529             PetscCall(PetscFindInt(support[e], numSplitPoints[dep + 1], splitPoints[dep + 1], &face));
1530             PetscCheck(face >= 0, comm, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " is not a split face", support[e]);
1531             supportNew[q++] = face + pMaxNew[dep + 1];
1532           } else if (val == -(shift + dim - 1)) {
1533             supportNew[q++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1534           }
1535         }
1536         supportNew[q++] = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1];
1537         PetscCall(DMPlexSetSupport(sdm, splitp, supportNew));
1538         /* Hybrid face */
1539         coneNew[0] = newp;
1540         coneNew[1] = splitp;
1541         for (v = 0; v < coneSize; ++v) {
1542           PetscInt vertex;
1543           PetscCall(PetscFindInt(cone[v], numSplitPoints[dep - 1], splitPoints[dep - 1], &vertex));
1544           if (vertex < 0) {
1545             PetscCall(PetscFindInt(cone[v], numUnsplitPoints[dep - 1], unsplitPoints[dep - 1], &vertex));
1546             PetscCheck(vertex >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %" PetscInt_FMT " in split or unsplit points of depth %" PetscInt_FMT, cone[v], dep - 1);
1547             coneNew[2 + v] = vertex + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep - 1];
1548           } else {
1549             coneNew[2 + v] = vertex + pMaxNew[dep] + numSplitPoints[dep];
1550           }
1551         }
1552         PetscCall(DMPlexSetCone(sdm, hybface, coneNew));
1553         for (e = 0, qf = 0; e < supportSize; ++e) {
1554           PetscInt val, face;
1555 
1556           PetscCall(DMLabelGetValue(label, support[e], &val));
1557           if (val == dim - 1) {
1558             PetscCall(PetscFindInt(support[e], numSplitPoints[dep + 1], splitPoints[dep + 1], &face));
1559             PetscCheck(face >= 0, comm, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " is not a split face", support[e]);
1560             supportNew[qf++] = face + pMaxNew[dep + 2] + numSplitPoints[dep + 2];
1561           }
1562         }
1563         PetscCall(DMPlexSetSupport(sdm, hybface, supportNew));
1564       }
1565     }
1566   }
1567   for (dep = 0; dep <= depth; ++dep) {
1568     for (p = 0; p < numUnsplitPoints[dep]; ++p) {
1569       const PetscInt  oldp = unsplitPoints[dep][p];
1570       const PetscInt  newp = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*oldp + depthOffset[dep]*/;
1571       const PetscInt *cone, *support;
1572       PetscInt        coneSize, supportSize, supportSizeNew, q, qf, e, f, s;
1573 
1574       PetscCall(DMPlexGetConeSize(dm, oldp, &coneSize));
1575       PetscCall(DMPlexGetCone(dm, oldp, &cone));
1576       PetscCall(DMPlexGetSupportSize(dm, oldp, &supportSize));
1577       PetscCall(DMPlexGetSupport(dm, oldp, &support));
1578       if (dep == 0) {
1579         const PetscInt hybedge = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1] + numSplitPoints[dep];
1580 
1581         /* Unsplit vertex */
1582         PetscCall(DMPlexGetSupportSize(sdm, newp, &supportSizeNew));
1583         for (s = 0, q = 0; s < supportSize; ++s) {
1584           supportNew[q++] = DMPlexShiftPoint_Internal(support[s], depth, depthShift) /*support[s] + depthOffset[dep+1]*/;
1585           PetscCall(PetscFindInt(support[s], numSplitPoints[dep + 1], splitPoints[dep + 1], &e));
1586           if (e >= 0) supportNew[q++] = e + pMaxNew[dep + 1];
1587         }
1588         supportNew[q++] = hybedge;
1589         supportNew[q++] = hybedge;
1590         PetscCheck(q == supportSizeNew, comm, PETSC_ERR_ARG_WRONG, "Support size %" PetscInt_FMT " != %" PetscInt_FMT " for vertex %" PetscInt_FMT, q, supportSizeNew, newp);
1591         PetscCall(DMPlexSetSupport(sdm, newp, supportNew));
1592         /* Hybrid edge */
1593         coneNew[0] = newp;
1594         coneNew[1] = newp;
1595         PetscCall(DMPlexSetCone(sdm, hybedge, coneNew));
1596         for (e = 0, qf = 0; e < supportSize; ++e) {
1597           PetscInt val, edge;
1598 
1599           PetscCall(DMLabelGetValue(label, support[e], &val));
1600           if (val == 1) {
1601             PetscCall(PetscFindInt(support[e], numSplitPoints[dep + 1], splitPoints[dep + 1], &edge));
1602             PetscCheck(edge >= 0, comm, PETSC_ERR_ARG_WRONG, "Edge %" PetscInt_FMT " is not a split edge", support[e]);
1603             supportNew[qf++] = edge + pMaxNew[dep + 2] + numSplitPoints[dep + 2];
1604           } else if (val == (shift2 + 1)) {
1605             PetscCall(PetscFindInt(support[e], numUnsplitPoints[dep + 1], unsplitPoints[dep + 1], &edge));
1606             PetscCheck(edge >= 0, comm, PETSC_ERR_ARG_WRONG, "Edge %" PetscInt_FMT " is not a unsplit edge", support[e]);
1607             supportNew[qf++] = edge + pMaxNew[dep + 2] + numSplitPoints[dep + 2] + numSplitPoints[dep + 1];
1608           }
1609         }
1610         PetscCall(DMPlexSetSupport(sdm, hybedge, supportNew));
1611       } else if (dep == dim - 2) {
1612         const PetscInt hybface = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1] + numSplitPoints[dep];
1613 
1614         /* Unsplit edge: Faces into original edge, split face, and hybrid face twice */
1615         for (f = 0, qf = 0; f < supportSize; ++f) {
1616           PetscInt val, face;
1617 
1618           PetscCall(DMLabelGetValue(label, support[f], &val));
1619           if (val == dim - 1) {
1620             PetscCall(PetscFindInt(support[f], numSplitPoints[dep + 1], splitPoints[dep + 1], &face));
1621             PetscCheck(face >= 0, comm, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " is not a split face", support[f]);
1622             supportNew[qf++] = DMPlexShiftPoint_Internal(support[f], depth, depthShift) /*support[f] + depthOffset[dep+1]*/;
1623             supportNew[qf++] = face + pMaxNew[dep + 1];
1624           } else {
1625             supportNew[qf++] = DMPlexShiftPoint_Internal(support[f], depth, depthShift) /*support[f] + depthOffset[dep+1]*/;
1626           }
1627         }
1628         supportNew[qf++] = hybface;
1629         supportNew[qf++] = hybface;
1630         PetscCall(DMPlexGetSupportSize(sdm, newp, &supportSizeNew));
1631         PetscCheck(qf == supportSizeNew, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Support size for unsplit edge %" PetscInt_FMT " is %" PetscInt_FMT " != %" PetscInt_FMT, newp, qf, supportSizeNew);
1632         PetscCall(DMPlexSetSupport(sdm, newp, supportNew));
1633         /* Add hybrid face */
1634         coneNew[0] = newp;
1635         coneNew[1] = newp;
1636         PetscCall(PetscFindInt(cone[0], numUnsplitPoints[dep - 1], unsplitPoints[dep - 1], &v));
1637         PetscCheck(v >= 0, comm, PETSC_ERR_ARG_WRONG, "Vertex %" PetscInt_FMT " is not an unsplit vertex", cone[0]);
1638         coneNew[2] = v + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep - 1];
1639         PetscCall(PetscFindInt(cone[1], numUnsplitPoints[dep - 1], unsplitPoints[dep - 1], &v));
1640         PetscCheck(v >= 0, comm, PETSC_ERR_ARG_WRONG, "Vertex %" PetscInt_FMT " is not an unsplit vertex", cone[1]);
1641         coneNew[3] = v + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep - 1];
1642         PetscCall(DMPlexSetCone(sdm, hybface, coneNew));
1643         for (f = 0, qf = 0; f < supportSize; ++f) {
1644           PetscInt val, face;
1645 
1646           PetscCall(DMLabelGetValue(label, support[f], &val));
1647           if (val == dim - 1) {
1648             PetscCall(PetscFindInt(support[f], numSplitPoints[dep + 1], splitPoints[dep + 1], &face));
1649             supportNew[qf++] = face + pMaxNew[dep + 2] + numSplitPoints[dep + 2];
1650           }
1651         }
1652         PetscCall(DMPlexGetSupportSize(sdm, hybface, &supportSizeNew));
1653         PetscCheck(qf == supportSizeNew, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Support size for hybrid face %" PetscInt_FMT " is %" PetscInt_FMT " != %" PetscInt_FMT, hybface, qf, supportSizeNew);
1654         PetscCall(DMPlexSetSupport(sdm, hybface, supportNew));
1655       }
1656     }
1657   }
1658   /* Step 6b: Replace split points in negative side cones */
1659   for (sp = 0; sp < numSP; ++sp) {
1660     PetscInt        dep = values[sp];
1661     IS              pIS;
1662     PetscInt        numPoints;
1663     const PetscInt *points;
1664 
1665     if (dep >= 0) continue;
1666     PetscCall(DMLabelGetStratumIS(label, dep, &pIS));
1667     if (!pIS) continue;
1668     dep = -dep - shift;
1669     PetscCall(ISGetLocalSize(pIS, &numPoints));
1670     PetscCall(ISGetIndices(pIS, &points));
1671     for (p = 0; p < numPoints; ++p) {
1672       const PetscInt  oldp = points[p];
1673       const PetscInt  newp = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*depthOffset[dep] + oldp*/;
1674       const PetscInt *cone;
1675       PetscInt        coneSize, c;
1676       /* PetscBool       replaced = PETSC_FALSE; */
1677 
1678       /* Negative edge: replace split vertex */
1679       /* Negative cell: replace split face */
1680       PetscCall(DMPlexGetConeSize(sdm, newp, &coneSize));
1681       PetscCall(DMPlexGetCone(sdm, newp, &cone));
1682       for (c = 0; c < coneSize; ++c) {
1683         const PetscInt coldp = DMPlexShiftPointInverse_Internal(cone[c], depth, depthShift);
1684         PetscInt       csplitp, cp, val;
1685 
1686         PetscCall(DMLabelGetValue(label, coldp, &val));
1687         if (val == dep - 1) {
1688           PetscCall(PetscFindInt(coldp, numSplitPoints[dep - 1], splitPoints[dep - 1], &cp));
1689           PetscCheck(cp >= 0, comm, PETSC_ERR_ARG_WRONG, "Point %" PetscInt_FMT " is not a split point of dimension %" PetscInt_FMT, oldp, dep - 1);
1690           csplitp = pMaxNew[dep - 1] + cp;
1691           PetscCall(DMPlexInsertCone(sdm, newp, c, csplitp));
1692           /* replaced = PETSC_TRUE; */
1693         }
1694       }
1695       /* Cells with only a vertex or edge on the submesh have no replacement */
1696       /* PetscCheck(replaced,comm, PETSC_ERR_ARG_WRONG, "The cone of point %d does not contain split points", oldp); */
1697     }
1698     PetscCall(ISRestoreIndices(pIS, &points));
1699     PetscCall(ISDestroy(&pIS));
1700   }
1701   /* Step 7: Coordinates */
1702   PetscCall(DMPlexShiftCoordinates_Internal(dm, depthShift, sdm));
1703   PetscCall(DMGetCoordinateSection(sdm, &coordSection));
1704   PetscCall(DMGetCoordinatesLocal(sdm, &coordinates));
1705   PetscCall(VecGetArray(coordinates, &coords));
1706   for (v = 0; v < (numSplitPoints ? numSplitPoints[0] : 0); ++v) {
1707     const PetscInt newp   = DMPlexShiftPoint_Internal(splitPoints[0][v], depth, depthShift) /*depthOffset[0] + splitPoints[0][v]*/;
1708     const PetscInt splitp = pMaxNew[0] + v;
1709     PetscInt       dof, off, soff, d;
1710 
1711     PetscCall(PetscSectionGetDof(coordSection, newp, &dof));
1712     PetscCall(PetscSectionGetOffset(coordSection, newp, &off));
1713     PetscCall(PetscSectionGetOffset(coordSection, splitp, &soff));
1714     for (d = 0; d < dof; ++d) coords[soff + d] = coords[off + d];
1715   }
1716   PetscCall(VecRestoreArray(coordinates, &coords));
1717   /* Step 8: SF, if I can figure this out we can split the mesh in parallel */
1718   PetscCall(DMPlexShiftSF_Internal(dm, depthShift, sdm));
1719   /*   TODO We need to associate the ghost points with the correct replica */
1720   /* Step 9: Labels */
1721   PetscCall(DMPlexShiftLabels_Internal(dm, depthShift, sdm));
1722   PetscCall(DMPlexCreateVTKLabel_Internal(dm, PETSC_FALSE, sdm));
1723   PetscCall(DMGetNumLabels(sdm, &numLabels));
1724   for (dep = 0; dep <= depth; ++dep) {
1725     for (p = 0; p < numSplitPoints[dep]; ++p) {
1726       const PetscInt newp   = DMPlexShiftPoint_Internal(splitPoints[dep][p], depth, depthShift) /*depthOffset[dep] + splitPoints[dep][p]*/;
1727       const PetscInt splitp = pMaxNew[dep] + p;
1728       PetscInt       l;
1729 
1730       if (splitLabel) {
1731         const PetscInt val = 100 + dep;
1732 
1733         PetscCall(DMLabelSetValue(splitLabel, newp, val));
1734         PetscCall(DMLabelSetValue(splitLabel, splitp, -val));
1735       }
1736       for (l = 0; l < numLabels; ++l) {
1737         DMLabel     mlabel;
1738         const char *lname;
1739         PetscInt    val;
1740         PetscBool   isDepth;
1741 
1742         PetscCall(DMGetLabelName(sdm, l, &lname));
1743         PetscCall(PetscStrcmp(lname, "depth", &isDepth));
1744         if (isDepth) continue;
1745         PetscCall(DMGetLabel(sdm, lname, &mlabel));
1746         PetscCall(DMLabelGetValue(mlabel, newp, &val));
1747         if (val >= 0) PetscCall(DMLabelSetValue(mlabel, splitp, val));
1748       }
1749     }
1750   }
1751   for (sp = 0; sp < numSP; ++sp) {
1752     const PetscInt dep = values[sp];
1753 
1754     if ((dep < 0) || (dep > depth)) continue;
1755     if (splitIS[dep]) PetscCall(ISRestoreIndices(splitIS[dep], &splitPoints[dep]));
1756     PetscCall(ISDestroy(&splitIS[dep]));
1757     if (unsplitIS[dep]) PetscCall(ISRestoreIndices(unsplitIS[dep], &unsplitPoints[dep]));
1758     PetscCall(ISDestroy(&unsplitIS[dep]));
1759   }
1760   if (ghostIS) PetscCall(ISRestoreIndices(ghostIS, &ghostPoints));
1761   PetscCall(ISDestroy(&ghostIS));
1762   if (label) {
1763     PetscCall(ISRestoreIndices(valueIS, &values));
1764     PetscCall(ISDestroy(&valueIS));
1765   }
1766   for (d = 0; d <= depth; ++d) {
1767     PetscCall(DMPlexGetDepthStratum(sdm, d, NULL, &pEnd));
1768     pMaxNew[d] = pEnd - numHybridPoints[d] - numHybridPointsOld[d];
1769   }
1770   PetscCall(PetscFree3(coneNew, coneONew, supportNew));
1771   PetscCall(PetscFree5(depthMax, depthEnd, depthShift, pMaxNew, numHybridPointsOld));
1772   PetscCall(PetscFree7(splitIS, unsplitIS, numSplitPoints, numUnsplitPoints, numHybridPoints, splitPoints, unsplitPoints));
1773   PetscFunctionReturn(0);
1774 }
1775 
1776 /*@C
1777   DMPlexConstructCohesiveCells - Construct cohesive cells which split the face along an internal interface
1778 
1779   Collective on dm
1780 
1781   Input Parameters:
1782 + dm - The original DM
1783 - label - The label specifying the boundary faces (this could be auto-generated)
1784 
1785   Output Parameters:
1786 + splitLabel - The label containing the split points, or NULL if no output is desired
1787 - dmSplit - The new DM
1788 
1789   Level: developer
1790 
1791 .seealso: `DMCreate()`, `DMPlexLabelCohesiveComplete()`
1792 @*/
1793 PetscErrorCode DMPlexConstructCohesiveCells(DM dm, DMLabel label, DMLabel splitLabel, DM *dmSplit) {
1794   DM       sdm;
1795   PetscInt dim;
1796 
1797   PetscFunctionBegin;
1798   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1799   PetscValidPointer(dmSplit, 4);
1800   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &sdm));
1801   PetscCall(DMSetType(sdm, DMPLEX));
1802   PetscCall(DMGetDimension(dm, &dim));
1803   PetscCall(DMSetDimension(sdm, dim));
1804   switch (dim) {
1805   case 2:
1806   case 3: PetscCall(DMPlexConstructCohesiveCells_Internal(dm, label, splitLabel, sdm)); break;
1807   default: SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cannot construct cohesive cells for dimension %" PetscInt_FMT, dim);
1808   }
1809   *dmSplit = sdm;
1810   PetscFunctionReturn(0);
1811 }
1812 
1813 /* Returns the side of the surface for a given cell with a face on the surface */
1814 static PetscErrorCode GetSurfaceSide_Static(DM dm, DM subdm, PetscInt numSubpoints, const PetscInt *subpoints, PetscInt cell, PetscInt face, PetscBool *pos) {
1815   const PetscInt *cone, *ornt;
1816   PetscInt        dim, coneSize, c;
1817 
1818   PetscFunctionBegin;
1819   *pos = PETSC_TRUE;
1820   PetscCall(DMGetDimension(dm, &dim));
1821   PetscCall(DMPlexGetConeSize(dm, cell, &coneSize));
1822   PetscCall(DMPlexGetCone(dm, cell, &cone));
1823   PetscCall(DMPlexGetConeOrientation(dm, cell, &ornt));
1824   for (c = 0; c < coneSize; ++c) {
1825     if (cone[c] == face) {
1826       PetscInt o = ornt[c];
1827 
1828       if (subdm) {
1829         const PetscInt *subcone, *subornt;
1830         PetscInt        subpoint, subface, subconeSize, sc;
1831 
1832         PetscCall(PetscFindInt(cell, numSubpoints, subpoints, &subpoint));
1833         PetscCall(PetscFindInt(face, numSubpoints, subpoints, &subface));
1834         PetscCall(DMPlexGetConeSize(subdm, subpoint, &subconeSize));
1835         PetscCall(DMPlexGetCone(subdm, subpoint, &subcone));
1836         PetscCall(DMPlexGetConeOrientation(subdm, subpoint, &subornt));
1837         for (sc = 0; sc < subconeSize; ++sc) {
1838           if (subcone[sc] == subface) {
1839             o = subornt[0];
1840             break;
1841           }
1842         }
1843         PetscCheck(sc < subconeSize, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not find subpoint %" PetscInt_FMT " (%" PetscInt_FMT ") in cone for subpoint %" PetscInt_FMT " (%" PetscInt_FMT ")", subface, face, subpoint, cell);
1844       }
1845       if (o >= 0) *pos = PETSC_TRUE;
1846       else *pos = PETSC_FALSE;
1847       break;
1848     }
1849   }
1850   PetscCheck(c != coneSize, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Cell %" PetscInt_FMT " in split face %" PetscInt_FMT " support does not have it in the cone", cell, face);
1851   PetscFunctionReturn(0);
1852 }
1853 
1854 static PetscErrorCode CheckFaultEdge_Private(DM dm, DMLabel label) {
1855   IS              facePosIS, faceNegIS, dimIS;
1856   const PetscInt *points;
1857   PetscInt        dim, numPoints, p, shift = 100, shift2 = 200;
1858 
1859   PetscFunctionBegin;
1860   PetscCall(DMGetDimension(dm, &dim));
1861   /* If any faces touching the fault divide cells on either side, split them */
1862   PetscCall(DMLabelGetStratumIS(label, shift + dim - 1, &facePosIS));
1863   PetscCall(DMLabelGetStratumIS(label, -(shift + dim - 1), &faceNegIS));
1864   if (!facePosIS || !faceNegIS) {
1865     PetscCall(ISDestroy(&facePosIS));
1866     PetscCall(ISDestroy(&faceNegIS));
1867     PetscFunctionReturn(0);
1868   }
1869   PetscCall(ISExpand(facePosIS, faceNegIS, &dimIS));
1870   PetscCall(ISDestroy(&facePosIS));
1871   PetscCall(ISDestroy(&faceNegIS));
1872   PetscCall(ISGetLocalSize(dimIS, &numPoints));
1873   PetscCall(ISGetIndices(dimIS, &points));
1874   for (p = 0; p < numPoints; ++p) {
1875     const PetscInt  point = points[p];
1876     const PetscInt *support;
1877     PetscInt        supportSize, valA, valB;
1878 
1879     PetscCall(DMPlexGetSupportSize(dm, point, &supportSize));
1880     if (supportSize != 2) continue;
1881     PetscCall(DMPlexGetSupport(dm, point, &support));
1882     PetscCall(DMLabelGetValue(label, support[0], &valA));
1883     PetscCall(DMLabelGetValue(label, support[1], &valB));
1884     if ((valA == -1) || (valB == -1)) continue;
1885     if (valA * valB > 0) continue;
1886     /* Check that this face is not incident on only unsplit faces, meaning has at least one split face */
1887     {
1888       PetscInt *closure = NULL;
1889       PetscBool split   = PETSC_FALSE;
1890       PetscInt  closureSize, cl;
1891 
1892       PetscCall(DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
1893       for (cl = 0; cl < closureSize * 2; cl += 2) {
1894         PetscCall(DMLabelGetValue(label, closure[cl], &valA));
1895         if ((valA >= 0) && (valA <= dim)) {
1896           split = PETSC_TRUE;
1897           break;
1898         }
1899       }
1900       PetscCall(DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
1901       if (!split) continue;
1902     }
1903     /* Split the face */
1904     PetscCall(DMLabelGetValue(label, point, &valA));
1905     PetscCall(DMLabelClearValue(label, point, valA));
1906     PetscCall(DMLabelSetValue(label, point, dim - 1));
1907     /* Label its closure:
1908       unmarked: label as unsplit
1909       incident: relabel as split
1910       split:    do nothing
1911     */
1912     {
1913       PetscInt *closure = NULL;
1914       PetscInt  closureSize, cl, dep;
1915 
1916       PetscCall(DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
1917       for (cl = 0; cl < closureSize * 2; cl += 2) {
1918         PetscCall(DMLabelGetValue(label, closure[cl], &valA));
1919         if (valA == -1) { /* Mark as unsplit */
1920           PetscCall(DMPlexGetPointDepth(dm, closure[cl], &dep));
1921           PetscCall(DMLabelSetValue(label, closure[cl], shift2 + dep));
1922         } else if (((valA >= shift) && (valA < shift2)) || ((valA <= -shift) && (valA > -shift2))) {
1923           PetscCall(DMPlexGetPointDepth(dm, closure[cl], &dep));
1924           PetscCall(DMLabelClearValue(label, closure[cl], valA));
1925           PetscCall(DMLabelSetValue(label, closure[cl], dep));
1926         }
1927       }
1928       PetscCall(DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
1929     }
1930   }
1931   PetscCall(ISRestoreIndices(dimIS, &points));
1932   PetscCall(ISDestroy(&dimIS));
1933   PetscFunctionReturn(0);
1934 }
1935 
1936 /*@
1937   DMPlexLabelCohesiveComplete - Starting with a label marking points on an internal surface, we add all other mesh pieces
1938   to complete the surface
1939 
1940   Input Parameters:
1941 + dm     - The DM
1942 . label  - A DMLabel marking the surface
1943 . blabel - A DMLabel marking the vertices on the boundary which will not be duplicated, or NULL to find them automatically
1944 . bvalue - Value of DMLabel marking the vertices on the boundary
1945 . flip   - Flag to flip the submesh normal and replace points on the other side
1946 - subdm  - The subDM associated with the label, or NULL
1947 
1948   Output Parameter:
1949 . label - A DMLabel marking all surface points
1950 
1951   Note: The vertices in blabel are called "unsplit" in the terminology from hybrid cell creation.
1952 
1953   Level: developer
1954 
1955 .seealso: `DMPlexConstructCohesiveCells()`, `DMPlexLabelComplete()`
1956 @*/
1957 PetscErrorCode DMPlexLabelCohesiveComplete(DM dm, DMLabel label, DMLabel blabel, PetscInt bvalue, PetscBool flip, DM subdm) {
1958   DMLabel         depthLabel;
1959   IS              dimIS, subpointIS = NULL;
1960   const PetscInt *points, *subpoints;
1961   const PetscInt  rev   = flip ? -1 : 1;
1962   PetscInt        shift = 100, shift2 = 200, shift3 = 300, dim, depth, numPoints, numSubpoints, p, val;
1963 
1964   PetscFunctionBegin;
1965   PetscCall(DMPlexGetDepth(dm, &depth));
1966   PetscCall(DMGetDimension(dm, &dim));
1967   PetscCall(DMPlexGetDepthLabel(dm, &depthLabel));
1968   if (subdm) {
1969     PetscCall(DMPlexGetSubpointIS(subdm, &subpointIS));
1970     if (subpointIS) {
1971       PetscCall(ISGetLocalSize(subpointIS, &numSubpoints));
1972       PetscCall(ISGetIndices(subpointIS, &subpoints));
1973     }
1974   }
1975   /* Mark cell on the fault, and its faces which touch the fault: cell orientation for face gives the side of the fault */
1976   PetscCall(DMLabelGetStratumIS(label, dim - 1, &dimIS));
1977   if (!dimIS) goto divide;
1978   PetscCall(ISGetLocalSize(dimIS, &numPoints));
1979   PetscCall(ISGetIndices(dimIS, &points));
1980   for (p = 0; p < numPoints; ++p) { /* Loop over fault faces */
1981     const PetscInt *support;
1982     PetscInt        supportSize, s;
1983 
1984     PetscCall(DMPlexGetSupportSize(dm, points[p], &supportSize));
1985 #if 0
1986     if (supportSize != 2) {
1987       const PetscInt *lp;
1988       PetscInt        Nlp, pind;
1989 
1990       /* Check that for a cell with a single support face, that face is in the SF */
1991       /*   THis check only works for the remote side. We would need root side information */
1992       PetscCall(PetscSFGetGraph(dm->sf, NULL, &Nlp, &lp, NULL));
1993       PetscCall(PetscFindInt(points[p], Nlp, lp, &pind));
1994       PetscCheck(pind >= 0,PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Split face %" PetscInt_FMT " has %" PetscInt_FMT " != 2 supports, and the face is not shared with another process", points[p], supportSize);
1995     }
1996 #endif
1997     PetscCall(DMPlexGetSupport(dm, points[p], &support));
1998     for (s = 0; s < supportSize; ++s) {
1999       const PetscInt *cone;
2000       PetscInt        coneSize, c;
2001       PetscBool       pos;
2002 
2003       PetscCall(GetSurfaceSide_Static(dm, subdm, numSubpoints, subpoints, support[s], points[p], &pos));
2004       if (pos) PetscCall(DMLabelSetValue(label, support[s], rev * (shift + dim)));
2005       else PetscCall(DMLabelSetValue(label, support[s], -rev * (shift + dim)));
2006       if (rev < 0) pos = !pos ? PETSC_TRUE : PETSC_FALSE;
2007       /* Put faces touching the fault in the label */
2008       PetscCall(DMPlexGetConeSize(dm, support[s], &coneSize));
2009       PetscCall(DMPlexGetCone(dm, support[s], &cone));
2010       for (c = 0; c < coneSize; ++c) {
2011         const PetscInt point = cone[c];
2012 
2013         PetscCall(DMLabelGetValue(label, point, &val));
2014         if (val == -1) {
2015           PetscInt *closure = NULL;
2016           PetscInt  closureSize, cl;
2017 
2018           PetscCall(DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
2019           for (cl = 0; cl < closureSize * 2; cl += 2) {
2020             const PetscInt clp  = closure[cl];
2021             PetscInt       bval = -1;
2022 
2023             PetscCall(DMLabelGetValue(label, clp, &val));
2024             if (blabel) PetscCall(DMLabelGetValue(blabel, clp, &bval));
2025             if ((val >= 0) && (val < dim - 1) && (bval < 0)) {
2026               PetscCall(DMLabelSetValue(label, point, pos == PETSC_TRUE ? shift + dim - 1 : -(shift + dim - 1)));
2027               break;
2028             }
2029           }
2030           PetscCall(DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
2031         }
2032       }
2033     }
2034   }
2035   PetscCall(ISRestoreIndices(dimIS, &points));
2036   PetscCall(ISDestroy(&dimIS));
2037   /* Mark boundary points as unsplit */
2038   if (blabel) {
2039     IS bdIS;
2040 
2041     PetscCall(DMLabelGetStratumIS(blabel, bvalue, &bdIS));
2042     PetscCall(ISGetLocalSize(bdIS, &numPoints));
2043     PetscCall(ISGetIndices(bdIS, &points));
2044     for (p = 0; p < numPoints; ++p) {
2045       const PetscInt point = points[p];
2046       PetscInt       val, bval;
2047 
2048       PetscCall(DMLabelGetValue(blabel, point, &bval));
2049       if (bval >= 0) {
2050         PetscCall(DMLabelGetValue(label, point, &val));
2051         if ((val < 0) || (val > dim)) {
2052           /* This could be a point added from splitting a vertex on an adjacent fault, otherwise its just wrong */
2053           PetscCall(DMLabelClearValue(blabel, point, bval));
2054         }
2055       }
2056     }
2057     for (p = 0; p < numPoints; ++p) {
2058       const PetscInt point = points[p];
2059       PetscInt       val, bval;
2060 
2061       PetscCall(DMLabelGetValue(blabel, point, &bval));
2062       if (bval >= 0) {
2063         const PetscInt *cone, *support;
2064         PetscInt        coneSize, supportSize, s, valA, valB, valE;
2065 
2066         /* Mark as unsplit */
2067         PetscCall(DMLabelGetValue(label, point, &val));
2068         PetscCheck(val >= 0 && val <= dim, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %" PetscInt_FMT " has label value %" PetscInt_FMT ", should be part of the fault", point, val);
2069         PetscCall(DMLabelClearValue(label, point, val));
2070         PetscCall(DMLabelSetValue(label, point, shift2 + val));
2071         /* Check for cross-edge
2072              A cross-edge has endpoints which are both on the boundary of the surface, but the edge itself is not. */
2073         if (val != 0) continue;
2074         PetscCall(DMPlexGetSupport(dm, point, &support));
2075         PetscCall(DMPlexGetSupportSize(dm, point, &supportSize));
2076         for (s = 0; s < supportSize; ++s) {
2077           PetscCall(DMPlexGetCone(dm, support[s], &cone));
2078           PetscCall(DMPlexGetConeSize(dm, support[s], &coneSize));
2079           PetscCheck(coneSize == 2, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Edge %" PetscInt_FMT " has %" PetscInt_FMT " vertices != 2", support[s], coneSize);
2080           PetscCall(DMLabelGetValue(blabel, cone[0], &valA));
2081           PetscCall(DMLabelGetValue(blabel, cone[1], &valB));
2082           PetscCall(DMLabelGetValue(blabel, support[s], &valE));
2083           if ((valE < 0) && (valA >= 0) && (valB >= 0) && (cone[0] != cone[1])) PetscCall(DMLabelSetValue(blabel, support[s], 2));
2084         }
2085       }
2086     }
2087     PetscCall(ISRestoreIndices(bdIS, &points));
2088     PetscCall(ISDestroy(&bdIS));
2089   }
2090   /* Mark ghost fault cells */
2091   {
2092     PetscSF         sf;
2093     const PetscInt *leaves;
2094     PetscInt        Nl, l;
2095 
2096     PetscCall(DMGetPointSF(dm, &sf));
2097     PetscCall(PetscSFGetGraph(sf, NULL, &Nl, &leaves, NULL));
2098     PetscCall(DMLabelGetStratumIS(label, dim - 1, &dimIS));
2099     if (!dimIS) goto divide;
2100     PetscCall(ISGetLocalSize(dimIS, &numPoints));
2101     PetscCall(ISGetIndices(dimIS, &points));
2102     if (Nl > 0) {
2103       for (p = 0; p < numPoints; ++p) {
2104         const PetscInt point = points[p];
2105         PetscInt       val;
2106 
2107         PetscCall(PetscFindInt(point, Nl, leaves, &l));
2108         if (l >= 0) {
2109           PetscInt *closure = NULL;
2110           PetscInt  closureSize, cl;
2111 
2112           PetscCall(DMLabelGetValue(label, point, &val));
2113           PetscCheck((val == dim - 1) || (val == shift2 + dim - 1), PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %" PetscInt_FMT " has label value %" PetscInt_FMT ", should be a fault face", point, val);
2114           PetscCall(DMLabelClearValue(label, point, val));
2115           PetscCall(DMLabelSetValue(label, point, shift3 + val));
2116           PetscCall(DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
2117           for (cl = 2; cl < closureSize * 2; cl += 2) {
2118             const PetscInt clp = closure[cl];
2119 
2120             PetscCall(DMLabelGetValue(label, clp, &val));
2121             PetscCheck(val != -1, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %" PetscInt_FMT " is missing from label, but is in the closure of a fault face", point);
2122             PetscCall(DMLabelClearValue(label, clp, val));
2123             PetscCall(DMLabelSetValue(label, clp, shift3 + val));
2124           }
2125           PetscCall(DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
2126         }
2127       }
2128     }
2129     PetscCall(ISRestoreIndices(dimIS, &points));
2130     PetscCall(ISDestroy(&dimIS));
2131   }
2132 divide:
2133   if (subpointIS) PetscCall(ISRestoreIndices(subpointIS, &subpoints));
2134   PetscCall(DMPlexLabelFaultHalo(dm, label));
2135   PetscCall(CheckFaultEdge_Private(dm, label));
2136   PetscFunctionReturn(0);
2137 }
2138 
2139 /* Check that no cell have all vertices on the fault */
2140 PetscErrorCode DMPlexCheckValidSubmesh_Private(DM dm, DMLabel label, DM subdm) {
2141   IS              subpointIS;
2142   const PetscInt *dmpoints;
2143   PetscInt        defaultValue, cStart, cEnd, c, vStart, vEnd;
2144 
2145   PetscFunctionBegin;
2146   if (!label) PetscFunctionReturn(0);
2147   PetscCall(DMLabelGetDefaultValue(label, &defaultValue));
2148   PetscCall(DMPlexGetSubpointIS(subdm, &subpointIS));
2149   if (!subpointIS) PetscFunctionReturn(0);
2150   PetscCall(DMPlexGetHeightStratum(subdm, 0, &cStart, &cEnd));
2151   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
2152   PetscCall(ISGetIndices(subpointIS, &dmpoints));
2153   for (c = cStart; c < cEnd; ++c) {
2154     PetscBool invalidCell = PETSC_TRUE;
2155     PetscInt *closure     = NULL;
2156     PetscInt  closureSize, cl;
2157 
2158     PetscCall(DMPlexGetTransitiveClosure(dm, dmpoints[c], PETSC_TRUE, &closureSize, &closure));
2159     for (cl = 0; cl < closureSize * 2; cl += 2) {
2160       PetscInt value = 0;
2161 
2162       if ((closure[cl] < vStart) || (closure[cl] >= vEnd)) continue;
2163       PetscCall(DMLabelGetValue(label, closure[cl], &value));
2164       if (value == defaultValue) {
2165         invalidCell = PETSC_FALSE;
2166         break;
2167       }
2168     }
2169     PetscCall(DMPlexRestoreTransitiveClosure(dm, dmpoints[c], PETSC_TRUE, &closureSize, &closure));
2170     if (invalidCell) {
2171       PetscCall(ISRestoreIndices(subpointIS, &dmpoints));
2172       PetscCall(ISDestroy(&subpointIS));
2173       PetscCall(DMDestroy(&subdm));
2174       SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Ambiguous submesh. Cell %" PetscInt_FMT " has all of its vertices on the submesh.", dmpoints[c]);
2175     }
2176   }
2177   PetscCall(ISRestoreIndices(subpointIS, &dmpoints));
2178   PetscFunctionReturn(0);
2179 }
2180 
2181 /*@
2182   DMPlexCreateHybridMesh - Create a mesh with hybrid cells along an internal interface
2183 
2184   Collective on dm
2185 
2186   Input Parameters:
2187 + dm - The original DM
2188 . label - The label specifying the interface vertices
2189 . bdlabel - The optional label specifying the interface boundary vertices
2190 - bdvalue - Value of optional label specifying the interface boundary vertices
2191 
2192   Output Parameters:
2193 + hybridLabel - The label fully marking the interface, or NULL if no output is desired
2194 . splitLabel - The label containing the split points, or NULL if no output is desired
2195 . dmInterface - The new interface DM, or NULL
2196 - dmHybrid - The new DM with cohesive cells
2197 
2198   Note: The hybridLabel indicates what parts of the original mesh impinged on the on division surface. For points
2199   directly on the division surface, they are labeled with their dimension, so an edge 7 on the division surface would be
2200   7 (1) in hybridLabel. For points that impinge from the positive side, they are labeled with 100+dim, so an edge 6 with
2201   one vertex 3 on the surface would be 6 (101) and 3 (0) in hybridLabel. If an edge 9 from the negative side of the
2202   surface also hits vertex 3, it would be 9 (-101) in hybridLabel.
2203 
2204   The splitLabel indicates what points in the new hybrid mesh were the result of splitting points in the original
2205   mesh. The label value is +=100+dim for each point. For example, if two edges 10 and 14 in the hybrid resulting from
2206   splitting an edge in the original mesh, you would have 10 (101) and 14 (-101) in the splitLabel.
2207 
2208   The dmInterface is a DM built from the original division surface. It has a label which can be retrieved using
2209   DMPlexGetSubpointMap() which maps each point back to the point in the surface of the original mesh.
2210 
2211   Level: developer
2212 
2213 .seealso: `DMPlexConstructCohesiveCells()`, `DMPlexLabelCohesiveComplete()`, `DMPlexGetSubpointMap()`, `DMCreate()`
2214 @*/
2215 PetscErrorCode DMPlexCreateHybridMesh(DM dm, DMLabel label, DMLabel bdlabel, PetscInt bdvalue, DMLabel *hybridLabel, DMLabel *splitLabel, DM *dmInterface, DM *dmHybrid) {
2216   DM       idm;
2217   DMLabel  subpointMap, hlabel, slabel = NULL;
2218   PetscInt dim;
2219 
2220   PetscFunctionBegin;
2221   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2222   if (label) PetscValidPointer(label, 2);
2223   if (bdlabel) PetscValidPointer(bdlabel, 3);
2224   if (hybridLabel) PetscValidPointer(hybridLabel, 5);
2225   if (splitLabel) PetscValidPointer(splitLabel, 6);
2226   if (dmInterface) PetscValidPointer(dmInterface, 7);
2227   PetscValidPointer(dmHybrid, 8);
2228   PetscCall(DMGetDimension(dm, &dim));
2229   PetscCall(DMPlexCreateSubmesh(dm, label, 1, PETSC_FALSE, &idm));
2230   PetscCall(DMPlexCheckValidSubmesh_Private(dm, label, idm));
2231   PetscCall(DMPlexOrient(idm));
2232   PetscCall(DMPlexGetSubpointMap(idm, &subpointMap));
2233   PetscCall(DMLabelDuplicate(subpointMap, &hlabel));
2234   PetscCall(DMLabelClearStratum(hlabel, dim));
2235   if (splitLabel) {
2236     const char *name;
2237     char        sname[PETSC_MAX_PATH_LEN];
2238 
2239     PetscCall(PetscObjectGetName((PetscObject)hlabel, &name));
2240     PetscCall(PetscStrncpy(sname, name, PETSC_MAX_PATH_LEN));
2241     PetscCall(PetscStrcat(sname, " split"));
2242     PetscCall(DMLabelCreate(PETSC_COMM_SELF, sname, &slabel));
2243   }
2244   PetscCall(DMPlexLabelCohesiveComplete(dm, hlabel, bdlabel, bdvalue, PETSC_FALSE, idm));
2245   if (dmInterface) {
2246     *dmInterface = idm;
2247   } else PetscCall(DMDestroy(&idm));
2248   PetscCall(DMPlexConstructCohesiveCells(dm, hlabel, slabel, dmHybrid));
2249   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_TRUE, *dmHybrid));
2250   if (hybridLabel) *hybridLabel = hlabel;
2251   else PetscCall(DMLabelDestroy(&hlabel));
2252   if (splitLabel) *splitLabel = slabel;
2253   {
2254     DM      cdm;
2255     DMLabel ctLabel;
2256 
2257     /* We need to somehow share the celltype label with the coordinate dm */
2258     PetscCall(DMGetCoordinateDM(*dmHybrid, &cdm));
2259     PetscCall(DMPlexGetCellTypeLabel(*dmHybrid, &ctLabel));
2260     PetscCall(DMSetLabel(cdm, ctLabel));
2261   }
2262   PetscFunctionReturn(0);
2263 }
2264 
2265 /* Here we need the explicit assumption that:
2266 
2267      For any marked cell, the marked vertices constitute a single face
2268 */
2269 static PetscErrorCode DMPlexMarkSubmesh_Uninterpolated(DM dm, DMLabel vertexLabel, PetscInt value, DMLabel subpointMap, PetscInt *numFaces, PetscInt *nFV, DM subdm) {
2270   IS              subvertexIS = NULL;
2271   const PetscInt *subvertices;
2272   PetscInt       *pStart, *pEnd, pSize;
2273   PetscInt        depth, dim, d, numSubVerticesInitial = 0, v;
2274 
2275   PetscFunctionBegin;
2276   *numFaces = 0;
2277   *nFV      = 0;
2278   PetscCall(DMPlexGetDepth(dm, &depth));
2279   PetscCall(DMGetDimension(dm, &dim));
2280   pSize = PetscMax(depth, dim) + 1;
2281   PetscCall(PetscMalloc2(pSize, &pStart, pSize, &pEnd));
2282   for (d = 0; d <= depth; ++d) PetscCall(DMPlexGetSimplexOrBoxCells(dm, depth - d, &pStart[d], &pEnd[d]));
2283   /* Loop over initial vertices and mark all faces in the collective star() */
2284   if (vertexLabel) PetscCall(DMLabelGetStratumIS(vertexLabel, value, &subvertexIS));
2285   if (subvertexIS) {
2286     PetscCall(ISGetSize(subvertexIS, &numSubVerticesInitial));
2287     PetscCall(ISGetIndices(subvertexIS, &subvertices));
2288   }
2289   for (v = 0; v < numSubVerticesInitial; ++v) {
2290     const PetscInt vertex = subvertices[v];
2291     PetscInt      *star   = NULL;
2292     PetscInt       starSize, s, numCells = 0, c;
2293 
2294     PetscCall(DMPlexGetTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star));
2295     for (s = 0; s < starSize * 2; s += 2) {
2296       const PetscInt point = star[s];
2297       if ((point >= pStart[depth]) && (point < pEnd[depth])) star[numCells++] = point;
2298     }
2299     for (c = 0; c < numCells; ++c) {
2300       const PetscInt cell    = star[c];
2301       PetscInt      *closure = NULL;
2302       PetscInt       closureSize, cl;
2303       PetscInt       cellLoc, numCorners = 0, faceSize = 0;
2304 
2305       PetscCall(DMLabelGetValue(subpointMap, cell, &cellLoc));
2306       if (cellLoc == 2) continue;
2307       PetscCheck(cellLoc < 0, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Cell %" PetscInt_FMT " has dimension %" PetscInt_FMT " in the surface label", cell, cellLoc);
2308       PetscCall(DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
2309       for (cl = 0; cl < closureSize * 2; cl += 2) {
2310         const PetscInt point = closure[cl];
2311         PetscInt       vertexLoc;
2312 
2313         if ((point >= pStart[0]) && (point < pEnd[0])) {
2314           ++numCorners;
2315           PetscCall(DMLabelGetValue(vertexLabel, point, &vertexLoc));
2316           if (vertexLoc == value) closure[faceSize++] = point;
2317         }
2318       }
2319       if (!(*nFV)) PetscCall(DMPlexGetNumFaceVertices(dm, dim, numCorners, nFV));
2320       PetscCheck(faceSize <= *nFV, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Invalid submesh: Too many vertices %" PetscInt_FMT " of an element on the surface", faceSize);
2321       if (faceSize == *nFV) {
2322         const PetscInt *cells = NULL;
2323         PetscInt        numCells, nc;
2324 
2325         ++(*numFaces);
2326         for (cl = 0; cl < faceSize; ++cl) PetscCall(DMLabelSetValue(subpointMap, closure[cl], 0));
2327         PetscCall(DMPlexGetJoin(dm, faceSize, closure, &numCells, &cells));
2328         for (nc = 0; nc < numCells; ++nc) PetscCall(DMLabelSetValue(subpointMap, cells[nc], 2));
2329         PetscCall(DMPlexRestoreJoin(dm, faceSize, closure, &numCells, &cells));
2330       }
2331       PetscCall(DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
2332     }
2333     PetscCall(DMPlexRestoreTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star));
2334   }
2335   if (subvertexIS) PetscCall(ISRestoreIndices(subvertexIS, &subvertices));
2336   PetscCall(ISDestroy(&subvertexIS));
2337   PetscCall(PetscFree2(pStart, pEnd));
2338   PetscFunctionReturn(0);
2339 }
2340 
2341 static PetscErrorCode DMPlexMarkSubmesh_Interpolated(DM dm, DMLabel vertexLabel, PetscInt value, PetscBool markedFaces, DMLabel subpointMap, DM subdm) {
2342   IS              subvertexIS = NULL;
2343   const PetscInt *subvertices;
2344   PetscInt       *pStart, *pEnd;
2345   PetscInt        dim, d, numSubVerticesInitial = 0, v;
2346 
2347   PetscFunctionBegin;
2348   PetscCall(DMGetDimension(dm, &dim));
2349   PetscCall(PetscMalloc2(dim + 1, &pStart, dim + 1, &pEnd));
2350   for (d = 0; d <= dim; ++d) PetscCall(DMPlexGetSimplexOrBoxCells(dm, dim - d, &pStart[d], &pEnd[d]));
2351   /* Loop over initial vertices and mark all faces in the collective star() */
2352   if (vertexLabel) {
2353     PetscCall(DMLabelGetStratumIS(vertexLabel, value, &subvertexIS));
2354     if (subvertexIS) {
2355       PetscCall(ISGetSize(subvertexIS, &numSubVerticesInitial));
2356       PetscCall(ISGetIndices(subvertexIS, &subvertices));
2357     }
2358   }
2359   for (v = 0; v < numSubVerticesInitial; ++v) {
2360     const PetscInt vertex = subvertices[v];
2361     PetscInt      *star   = NULL;
2362     PetscInt       starSize, s, numFaces = 0, f;
2363 
2364     PetscCall(DMPlexGetTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star));
2365     for (s = 0; s < starSize * 2; s += 2) {
2366       const PetscInt point = star[s];
2367       PetscInt       faceLoc;
2368 
2369       if ((point >= pStart[dim - 1]) && (point < pEnd[dim - 1])) {
2370         if (markedFaces) {
2371           PetscCall(DMLabelGetValue(vertexLabel, point, &faceLoc));
2372           if (faceLoc < 0) continue;
2373         }
2374         star[numFaces++] = point;
2375       }
2376     }
2377     for (f = 0; f < numFaces; ++f) {
2378       const PetscInt face    = star[f];
2379       PetscInt      *closure = NULL;
2380       PetscInt       closureSize, c;
2381       PetscInt       faceLoc;
2382 
2383       PetscCall(DMLabelGetValue(subpointMap, face, &faceLoc));
2384       if (faceLoc == dim - 1) continue;
2385       PetscCheck(faceLoc < 0, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Face %" PetscInt_FMT " has dimension %" PetscInt_FMT " in the surface label", face, faceLoc);
2386       PetscCall(DMPlexGetTransitiveClosure(dm, face, PETSC_TRUE, &closureSize, &closure));
2387       for (c = 0; c < closureSize * 2; c += 2) {
2388         const PetscInt point = closure[c];
2389         PetscInt       vertexLoc;
2390 
2391         if ((point >= pStart[0]) && (point < pEnd[0])) {
2392           PetscCall(DMLabelGetValue(vertexLabel, point, &vertexLoc));
2393           if (vertexLoc != value) break;
2394         }
2395       }
2396       if (c == closureSize * 2) {
2397         const PetscInt *support;
2398         PetscInt        supportSize, s;
2399 
2400         for (c = 0; c < closureSize * 2; c += 2) {
2401           const PetscInt point = closure[c];
2402 
2403           for (d = 0; d < dim; ++d) {
2404             if ((point >= pStart[d]) && (point < pEnd[d])) {
2405               PetscCall(DMLabelSetValue(subpointMap, point, d));
2406               break;
2407             }
2408           }
2409         }
2410         PetscCall(DMPlexGetSupportSize(dm, face, &supportSize));
2411         PetscCall(DMPlexGetSupport(dm, face, &support));
2412         for (s = 0; s < supportSize; ++s) PetscCall(DMLabelSetValue(subpointMap, support[s], dim));
2413       }
2414       PetscCall(DMPlexRestoreTransitiveClosure(dm, face, PETSC_TRUE, &closureSize, &closure));
2415     }
2416     PetscCall(DMPlexRestoreTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star));
2417   }
2418   if (subvertexIS) PetscCall(ISRestoreIndices(subvertexIS, &subvertices));
2419   PetscCall(ISDestroy(&subvertexIS));
2420   PetscCall(PetscFree2(pStart, pEnd));
2421   PetscFunctionReturn(0);
2422 }
2423 
2424 static PetscErrorCode DMPlexMarkCohesiveSubmesh_Uninterpolated(DM dm, PetscBool hasLagrange, const char labelname[], PetscInt value, DMLabel subpointMap, PetscInt *numFaces, PetscInt *nFV, PetscInt *subCells[], DM subdm) {
2425   DMLabel         label = NULL;
2426   const PetscInt *cone;
2427   PetscInt        dim, cMax, cEnd, c, subc = 0, p, coneSize = -1;
2428 
2429   PetscFunctionBegin;
2430   *numFaces = 0;
2431   *nFV      = 0;
2432   if (labelname) PetscCall(DMGetLabel(dm, labelname, &label));
2433   *subCells = NULL;
2434   PetscCall(DMGetDimension(dm, &dim));
2435   PetscCall(DMPlexGetTensorPrismBounds_Internal(dm, dim, &cMax, &cEnd));
2436   if (cMax < 0) PetscFunctionReturn(0);
2437   if (label) {
2438     for (c = cMax; c < cEnd; ++c) {
2439       PetscInt val;
2440 
2441       PetscCall(DMLabelGetValue(label, c, &val));
2442       if (val == value) {
2443         ++(*numFaces);
2444         PetscCall(DMPlexGetConeSize(dm, c, &coneSize));
2445       }
2446     }
2447   } else {
2448     *numFaces = cEnd - cMax;
2449     PetscCall(DMPlexGetConeSize(dm, cMax, &coneSize));
2450   }
2451   PetscCall(PetscMalloc1(*numFaces * 2, subCells));
2452   if (!(*numFaces)) PetscFunctionReturn(0);
2453   *nFV = hasLagrange ? coneSize / 3 : coneSize / 2;
2454   for (c = cMax; c < cEnd; ++c) {
2455     const PetscInt *cells;
2456     PetscInt        numCells;
2457 
2458     if (label) {
2459       PetscInt val;
2460 
2461       PetscCall(DMLabelGetValue(label, c, &val));
2462       if (val != value) continue;
2463     }
2464     PetscCall(DMPlexGetCone(dm, c, &cone));
2465     for (p = 0; p < *nFV; ++p) PetscCall(DMLabelSetValue(subpointMap, cone[p], 0));
2466     /* Negative face */
2467     PetscCall(DMPlexGetJoin(dm, *nFV, cone, &numCells, &cells));
2468     /* Not true in parallel
2469     PetscCheck(numCells == 2,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive cells should separate two cells"); */
2470     for (p = 0; p < numCells; ++p) {
2471       PetscCall(DMLabelSetValue(subpointMap, cells[p], 2));
2472       (*subCells)[subc++] = cells[p];
2473     }
2474     PetscCall(DMPlexRestoreJoin(dm, *nFV, cone, &numCells, &cells));
2475     /* Positive face is not included */
2476   }
2477   PetscFunctionReturn(0);
2478 }
2479 
2480 static PetscErrorCode DMPlexMarkCohesiveSubmesh_Interpolated(DM dm, DMLabel label, PetscInt value, DMLabel subpointMap, DM subdm) {
2481   PetscInt *pStart, *pEnd;
2482   PetscInt  dim, cMax, cEnd, c, d;
2483 
2484   PetscFunctionBegin;
2485   PetscCall(DMGetDimension(dm, &dim));
2486   PetscCall(DMPlexGetTensorPrismBounds_Internal(dm, dim, &cMax, &cEnd));
2487   if (cMax < 0) PetscFunctionReturn(0);
2488   PetscCall(PetscMalloc2(dim + 1, &pStart, dim + 1, &pEnd));
2489   for (d = 0; d <= dim; ++d) PetscCall(DMPlexGetDepthStratum(dm, d, &pStart[d], &pEnd[d]));
2490   for (c = cMax; c < cEnd; ++c) {
2491     const PetscInt *cone;
2492     PetscInt       *closure = NULL;
2493     PetscInt        fconeSize, coneSize, closureSize, cl, val;
2494 
2495     if (label) {
2496       PetscCall(DMLabelGetValue(label, c, &val));
2497       if (val != value) continue;
2498     }
2499     PetscCall(DMPlexGetConeSize(dm, c, &coneSize));
2500     PetscCall(DMPlexGetCone(dm, c, &cone));
2501     PetscCall(DMPlexGetConeSize(dm, cone[0], &fconeSize));
2502     PetscCheck(coneSize == (fconeSize ? fconeSize : 1) + 2, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive cells should separate two cells");
2503     /* Negative face */
2504     PetscCall(DMPlexGetTransitiveClosure(dm, cone[0], PETSC_TRUE, &closureSize, &closure));
2505     for (cl = 0; cl < closureSize * 2; cl += 2) {
2506       const PetscInt point = closure[cl];
2507 
2508       for (d = 0; d <= dim; ++d) {
2509         if ((point >= pStart[d]) && (point < pEnd[d])) {
2510           PetscCall(DMLabelSetValue(subpointMap, point, d));
2511           break;
2512         }
2513       }
2514     }
2515     PetscCall(DMPlexRestoreTransitiveClosure(dm, cone[0], PETSC_TRUE, &closureSize, &closure));
2516     /* Cells -- positive face is not included */
2517     for (cl = 0; cl < 1; ++cl) {
2518       const PetscInt *support;
2519       PetscInt        supportSize, s;
2520 
2521       PetscCall(DMPlexGetSupportSize(dm, cone[cl], &supportSize));
2522       /* PetscCheck(supportSize == 2,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive faces should separate two cells"); */
2523       PetscCall(DMPlexGetSupport(dm, cone[cl], &support));
2524       for (s = 0; s < supportSize; ++s) PetscCall(DMLabelSetValue(subpointMap, support[s], dim));
2525     }
2526   }
2527   PetscCall(PetscFree2(pStart, pEnd));
2528   PetscFunctionReturn(0);
2529 }
2530 
2531 static PetscErrorCode DMPlexGetFaceOrientation(DM dm, PetscInt cell, PetscInt numCorners, PetscInt indices[], PetscInt oppositeVertex, PetscInt origVertices[], PetscInt faceVertices[], PetscBool *posOriented) {
2532   MPI_Comm       comm;
2533   PetscBool      posOrient = PETSC_FALSE;
2534   const PetscInt debug     = 0;
2535   PetscInt       cellDim, faceSize, f;
2536 
2537   PetscFunctionBegin;
2538   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
2539   PetscCall(DMGetDimension(dm, &cellDim));
2540   if (debug) PetscCall(PetscPrintf(comm, "cellDim: %" PetscInt_FMT " numCorners: %" PetscInt_FMT "\n", cellDim, numCorners));
2541 
2542   if (cellDim == 1 && numCorners == 2) {
2543     /* Triangle */
2544     faceSize  = numCorners - 1;
2545     posOrient = !(oppositeVertex % 2) ? PETSC_TRUE : PETSC_FALSE;
2546   } else if (cellDim == 2 && numCorners == 3) {
2547     /* Triangle */
2548     faceSize  = numCorners - 1;
2549     posOrient = !(oppositeVertex % 2) ? PETSC_TRUE : PETSC_FALSE;
2550   } else if (cellDim == 3 && numCorners == 4) {
2551     /* Tetrahedron */
2552     faceSize  = numCorners - 1;
2553     posOrient = (oppositeVertex % 2) ? PETSC_TRUE : PETSC_FALSE;
2554   } else if (cellDim == 1 && numCorners == 3) {
2555     /* Quadratic line */
2556     faceSize  = 1;
2557     posOrient = PETSC_TRUE;
2558   } else if (cellDim == 2 && numCorners == 4) {
2559     /* Quads */
2560     faceSize = 2;
2561     if ((indices[1] > indices[0]) && (indices[1] - indices[0] == 1)) {
2562       posOrient = PETSC_TRUE;
2563     } else if ((indices[0] == 3) && (indices[1] == 0)) {
2564       posOrient = PETSC_TRUE;
2565     } else {
2566       if (((indices[0] > indices[1]) && (indices[0] - indices[1] == 1)) || ((indices[0] == 0) && (indices[1] == 3))) {
2567         posOrient = PETSC_FALSE;
2568       } else SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid quad crossedge");
2569     }
2570   } else if (cellDim == 2 && numCorners == 6) {
2571     /* Quadratic triangle (I hate this) */
2572     /* Edges are determined by the first 2 vertices (corners of edges) */
2573     const PetscInt faceSizeTri = 3;
2574     PetscInt       sortedIndices[3], i, iFace;
2575     PetscBool      found                    = PETSC_FALSE;
2576     PetscInt       faceVerticesTriSorted[9] = {
2577             0, 3, 4, /* bottom */
2578             1, 4, 5, /* right */
2579             2, 3, 5, /* left */
2580     };
2581     PetscInt faceVerticesTri[9] = {
2582       0, 3, 4, /* bottom */
2583       1, 4, 5, /* right */
2584       2, 5, 3, /* left */
2585     };
2586 
2587     for (i = 0; i < faceSizeTri; ++i) sortedIndices[i] = indices[i];
2588     PetscCall(PetscSortInt(faceSizeTri, sortedIndices));
2589     for (iFace = 0; iFace < 3; ++iFace) {
2590       const PetscInt ii = iFace * faceSizeTri;
2591       PetscInt       fVertex, cVertex;
2592 
2593       if ((sortedIndices[0] == faceVerticesTriSorted[ii + 0]) && (sortedIndices[1] == faceVerticesTriSorted[ii + 1])) {
2594         for (fVertex = 0; fVertex < faceSizeTri; ++fVertex) {
2595           for (cVertex = 0; cVertex < faceSizeTri; ++cVertex) {
2596             if (indices[cVertex] == faceVerticesTri[ii + fVertex]) {
2597               faceVertices[fVertex] = origVertices[cVertex];
2598               break;
2599             }
2600           }
2601         }
2602         found = PETSC_TRUE;
2603         break;
2604       }
2605     }
2606     PetscCheck(found, comm, PETSC_ERR_ARG_WRONG, "Invalid tri crossface");
2607     if (posOriented) *posOriented = PETSC_TRUE;
2608     PetscFunctionReturn(0);
2609   } else if (cellDim == 2 && numCorners == 9) {
2610     /* Quadratic quad (I hate this) */
2611     /* Edges are determined by the first 2 vertices (corners of edges) */
2612     const PetscInt faceSizeQuad = 3;
2613     PetscInt       sortedIndices[3], i, iFace;
2614     PetscBool      found                      = PETSC_FALSE;
2615     PetscInt       faceVerticesQuadSorted[12] = {
2616             0, 1, 4, /* bottom */
2617             1, 2, 5, /* right */
2618             2, 3, 6, /* top */
2619             0, 3, 7, /* left */
2620     };
2621     PetscInt faceVerticesQuad[12] = {
2622       0, 1, 4, /* bottom */
2623       1, 2, 5, /* right */
2624       2, 3, 6, /* top */
2625       3, 0, 7, /* left */
2626     };
2627 
2628     for (i = 0; i < faceSizeQuad; ++i) sortedIndices[i] = indices[i];
2629     PetscCall(PetscSortInt(faceSizeQuad, sortedIndices));
2630     for (iFace = 0; iFace < 4; ++iFace) {
2631       const PetscInt ii = iFace * faceSizeQuad;
2632       PetscInt       fVertex, cVertex;
2633 
2634       if ((sortedIndices[0] == faceVerticesQuadSorted[ii + 0]) && (sortedIndices[1] == faceVerticesQuadSorted[ii + 1])) {
2635         for (fVertex = 0; fVertex < faceSizeQuad; ++fVertex) {
2636           for (cVertex = 0; cVertex < faceSizeQuad; ++cVertex) {
2637             if (indices[cVertex] == faceVerticesQuad[ii + fVertex]) {
2638               faceVertices[fVertex] = origVertices[cVertex];
2639               break;
2640             }
2641           }
2642         }
2643         found = PETSC_TRUE;
2644         break;
2645       }
2646     }
2647     PetscCheck(found, comm, PETSC_ERR_ARG_WRONG, "Invalid quad crossface");
2648     if (posOriented) *posOriented = PETSC_TRUE;
2649     PetscFunctionReturn(0);
2650   } else if (cellDim == 3 && numCorners == 8) {
2651     /* Hexes
2652        A hex is two oriented quads with the normal of the first
2653        pointing up at the second.
2654 
2655           7---6
2656          /|  /|
2657         4---5 |
2658         | 1-|-2
2659         |/  |/
2660         0---3
2661 
2662         Faces are determined by the first 4 vertices (corners of faces) */
2663     const PetscInt faceSizeHex = 4;
2664     PetscInt       sortedIndices[4], i, iFace;
2665     PetscBool      found                     = PETSC_FALSE;
2666     PetscInt       faceVerticesHexSorted[24] = {
2667             0, 1, 2, 3, /* bottom */
2668             4, 5, 6, 7, /* top */
2669             0, 3, 4, 5, /* front */
2670             2, 3, 5, 6, /* right */
2671             1, 2, 6, 7, /* back */
2672             0, 1, 4, 7, /* left */
2673     };
2674     PetscInt faceVerticesHex[24] = {
2675       1, 2, 3, 0, /* bottom */
2676       4, 5, 6, 7, /* top */
2677       0, 3, 5, 4, /* front */
2678       3, 2, 6, 5, /* right */
2679       2, 1, 7, 6, /* back */
2680       1, 0, 4, 7, /* left */
2681     };
2682 
2683     for (i = 0; i < faceSizeHex; ++i) sortedIndices[i] = indices[i];
2684     PetscCall(PetscSortInt(faceSizeHex, sortedIndices));
2685     for (iFace = 0; iFace < 6; ++iFace) {
2686       const PetscInt ii = iFace * faceSizeHex;
2687       PetscInt       fVertex, cVertex;
2688 
2689       if ((sortedIndices[0] == faceVerticesHexSorted[ii + 0]) && (sortedIndices[1] == faceVerticesHexSorted[ii + 1]) && (sortedIndices[2] == faceVerticesHexSorted[ii + 2]) && (sortedIndices[3] == faceVerticesHexSorted[ii + 3])) {
2690         for (fVertex = 0; fVertex < faceSizeHex; ++fVertex) {
2691           for (cVertex = 0; cVertex < faceSizeHex; ++cVertex) {
2692             if (indices[cVertex] == faceVerticesHex[ii + fVertex]) {
2693               faceVertices[fVertex] = origVertices[cVertex];
2694               break;
2695             }
2696           }
2697         }
2698         found = PETSC_TRUE;
2699         break;
2700       }
2701     }
2702     PetscCheck(found, comm, PETSC_ERR_ARG_WRONG, "Invalid hex crossface");
2703     if (posOriented) *posOriented = PETSC_TRUE;
2704     PetscFunctionReturn(0);
2705   } else if (cellDim == 3 && numCorners == 10) {
2706     /* Quadratic tet */
2707     /* Faces are determined by the first 3 vertices (corners of faces) */
2708     const PetscInt faceSizeTet = 6;
2709     PetscInt       sortedIndices[6], i, iFace;
2710     PetscBool      found                     = PETSC_FALSE;
2711     PetscInt       faceVerticesTetSorted[24] = {
2712             0, 1, 2, 6, 7, 8, /* bottom */
2713             0, 3, 4, 6, 7, 9, /* front */
2714             1, 4, 5, 7, 8, 9, /* right */
2715             2, 3, 5, 6, 8, 9, /* left */
2716     };
2717     PetscInt faceVerticesTet[24] = {
2718       0, 1, 2, 6, 7, 8, /* bottom */
2719       0, 4, 3, 6, 7, 9, /* front */
2720       1, 5, 4, 7, 8, 9, /* right */
2721       2, 3, 5, 8, 6, 9, /* left */
2722     };
2723 
2724     for (i = 0; i < faceSizeTet; ++i) sortedIndices[i] = indices[i];
2725     PetscCall(PetscSortInt(faceSizeTet, sortedIndices));
2726     for (iFace = 0; iFace < 4; ++iFace) {
2727       const PetscInt ii = iFace * faceSizeTet;
2728       PetscInt       fVertex, cVertex;
2729 
2730       if ((sortedIndices[0] == faceVerticesTetSorted[ii + 0]) && (sortedIndices[1] == faceVerticesTetSorted[ii + 1]) && (sortedIndices[2] == faceVerticesTetSorted[ii + 2]) && (sortedIndices[3] == faceVerticesTetSorted[ii + 3])) {
2731         for (fVertex = 0; fVertex < faceSizeTet; ++fVertex) {
2732           for (cVertex = 0; cVertex < faceSizeTet; ++cVertex) {
2733             if (indices[cVertex] == faceVerticesTet[ii + fVertex]) {
2734               faceVertices[fVertex] = origVertices[cVertex];
2735               break;
2736             }
2737           }
2738         }
2739         found = PETSC_TRUE;
2740         break;
2741       }
2742     }
2743     PetscCheck(found, comm, PETSC_ERR_ARG_WRONG, "Invalid tet crossface");
2744     if (posOriented) *posOriented = PETSC_TRUE;
2745     PetscFunctionReturn(0);
2746   } else if (cellDim == 3 && numCorners == 27) {
2747     /* Quadratic hexes (I hate this)
2748        A hex is two oriented quads with the normal of the first
2749        pointing up at the second.
2750 
2751          7---6
2752         /|  /|
2753        4---5 |
2754        | 3-|-2
2755        |/  |/
2756        0---1
2757 
2758        Faces are determined by the first 4 vertices (corners of faces) */
2759     const PetscInt faceSizeQuadHex = 9;
2760     PetscInt       sortedIndices[9], i, iFace;
2761     PetscBool      found                         = PETSC_FALSE;
2762     PetscInt       faceVerticesQuadHexSorted[54] = {
2763             0, 1, 2, 3, 8,  9,  10, 11, 24, /* bottom */
2764             4, 5, 6, 7, 12, 13, 14, 15, 25, /* top */
2765             0, 1, 4, 5, 8,  12, 16, 17, 22, /* front */
2766             1, 2, 5, 6, 9,  13, 17, 18, 21, /* right */
2767             2, 3, 6, 7, 10, 14, 18, 19, 23, /* back */
2768             0, 3, 4, 7, 11, 15, 16, 19, 20, /* left */
2769     };
2770     PetscInt faceVerticesQuadHex[54] = {
2771       3, 2, 1, 0, 10, 9,  8,  11, 24, /* bottom */
2772       4, 5, 6, 7, 12, 13, 14, 15, 25, /* top */
2773       0, 1, 5, 4, 8,  17, 12, 16, 22, /* front */
2774       1, 2, 6, 5, 9,  18, 13, 17, 21, /* right */
2775       2, 3, 7, 6, 10, 19, 14, 18, 23, /* back */
2776       3, 0, 4, 7, 11, 16, 15, 19, 20  /* left */
2777     };
2778 
2779     for (i = 0; i < faceSizeQuadHex; ++i) sortedIndices[i] = indices[i];
2780     PetscCall(PetscSortInt(faceSizeQuadHex, sortedIndices));
2781     for (iFace = 0; iFace < 6; ++iFace) {
2782       const PetscInt ii = iFace * faceSizeQuadHex;
2783       PetscInt       fVertex, cVertex;
2784 
2785       if ((sortedIndices[0] == faceVerticesQuadHexSorted[ii + 0]) && (sortedIndices[1] == faceVerticesQuadHexSorted[ii + 1]) && (sortedIndices[2] == faceVerticesQuadHexSorted[ii + 2]) && (sortedIndices[3] == faceVerticesQuadHexSorted[ii + 3])) {
2786         for (fVertex = 0; fVertex < faceSizeQuadHex; ++fVertex) {
2787           for (cVertex = 0; cVertex < faceSizeQuadHex; ++cVertex) {
2788             if (indices[cVertex] == faceVerticesQuadHex[ii + fVertex]) {
2789               faceVertices[fVertex] = origVertices[cVertex];
2790               break;
2791             }
2792           }
2793         }
2794         found = PETSC_TRUE;
2795         break;
2796       }
2797     }
2798     PetscCheck(found, comm, PETSC_ERR_ARG_WRONG, "Invalid hex crossface");
2799     if (posOriented) *posOriented = PETSC_TRUE;
2800     PetscFunctionReturn(0);
2801   } else SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Unknown cell type for faceOrientation().");
2802   if (!posOrient) {
2803     if (debug) PetscCall(PetscPrintf(comm, "  Reversing initial face orientation\n"));
2804     for (f = 0; f < faceSize; ++f) faceVertices[f] = origVertices[faceSize - 1 - f];
2805   } else {
2806     if (debug) PetscCall(PetscPrintf(comm, "  Keeping initial face orientation\n"));
2807     for (f = 0; f < faceSize; ++f) faceVertices[f] = origVertices[f];
2808   }
2809   if (posOriented) *posOriented = posOrient;
2810   PetscFunctionReturn(0);
2811 }
2812 
2813 /*@
2814   DMPlexGetOrientedFace - Given a cell and a face, as a set of vertices, return the oriented face, as a set of vertices,
2815   in faceVertices. The orientation is such that the face normal points out of the cell
2816 
2817   Not collective
2818 
2819   Input Parameters:
2820 + dm           - The original mesh
2821 . cell         - The cell mesh point
2822 . faceSize     - The number of vertices on the face
2823 . face         - The face vertices
2824 . numCorners   - The number of vertices on the cell
2825 . indices      - Local numbering of face vertices in cell cone
2826 - origVertices - Original face vertices
2827 
2828   Output Parameters:
2829 + faceVertices - The face vertices properly oriented
2830 - posOriented  - PETSC_TRUE if the face was oriented with outward normal
2831 
2832   Level: developer
2833 
2834 .seealso: `DMPlexGetCone()`
2835 @*/
2836 PetscErrorCode DMPlexGetOrientedFace(DM dm, PetscInt cell, PetscInt faceSize, const PetscInt face[], PetscInt numCorners, PetscInt indices[], PetscInt origVertices[], PetscInt faceVertices[], PetscBool *posOriented) {
2837   const PetscInt *cone = NULL;
2838   PetscInt        coneSize, v, f, v2;
2839   PetscInt        oppositeVertex = -1;
2840 
2841   PetscFunctionBegin;
2842   PetscCall(DMPlexGetConeSize(dm, cell, &coneSize));
2843   PetscCall(DMPlexGetCone(dm, cell, &cone));
2844   for (v = 0, v2 = 0; v < coneSize; ++v) {
2845     PetscBool found = PETSC_FALSE;
2846 
2847     for (f = 0; f < faceSize; ++f) {
2848       if (face[f] == cone[v]) {
2849         found = PETSC_TRUE;
2850         break;
2851       }
2852     }
2853     if (found) {
2854       indices[v2]      = v;
2855       origVertices[v2] = cone[v];
2856       ++v2;
2857     } else {
2858       oppositeVertex = v;
2859     }
2860   }
2861   PetscCall(DMPlexGetFaceOrientation(dm, cell, numCorners, indices, oppositeVertex, origVertices, faceVertices, posOriented));
2862   PetscFunctionReturn(0);
2863 }
2864 
2865 /*
2866   DMPlexInsertFace_Internal - Puts a face into the mesh
2867 
2868   Not collective
2869 
2870   Input Parameters:
2871   + dm              - The DMPlex
2872   . numFaceVertex   - The number of vertices in the face
2873   . faceVertices    - The vertices in the face for dm
2874   . subfaceVertices - The vertices in the face for subdm
2875   . numCorners      - The number of vertices in the cell
2876   . cell            - A cell in dm containing the face
2877   . subcell         - A cell in subdm containing the face
2878   . firstFace       - First face in the mesh
2879   - newFacePoint    - Next face in the mesh
2880 
2881   Output Parameters:
2882   . newFacePoint - Contains next face point number on input, updated on output
2883 
2884   Level: developer
2885 */
2886 static PetscErrorCode DMPlexInsertFace_Internal(DM dm, DM subdm, PetscInt numFaceVertices, const PetscInt faceVertices[], const PetscInt subfaceVertices[], PetscInt numCorners, PetscInt cell, PetscInt subcell, PetscInt firstFace, PetscInt *newFacePoint) {
2887   MPI_Comm        comm;
2888   DM_Plex        *submesh = (DM_Plex *)subdm->data;
2889   const PetscInt *faces;
2890   PetscInt        numFaces, coneSize;
2891 
2892   PetscFunctionBegin;
2893   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
2894   PetscCall(DMPlexGetConeSize(subdm, subcell, &coneSize));
2895   PetscCheck(coneSize == 1, comm, PETSC_ERR_ARG_OUTOFRANGE, "Cone size of cell %" PetscInt_FMT " is %" PetscInt_FMT " != 1", cell, coneSize);
2896 #if 0
2897   /* Cannot use this because support() has not been constructed yet */
2898   PetscCall(DMPlexGetJoin(subdm, numFaceVertices, subfaceVertices, &numFaces, &faces));
2899 #else
2900   {
2901     PetscInt f;
2902 
2903     numFaces = 0;
2904     PetscCall(DMGetWorkArray(subdm, 1, MPIU_INT, (void **)&faces));
2905     for (f = firstFace; f < *newFacePoint; ++f) {
2906       PetscInt dof, off, d;
2907 
2908       PetscCall(PetscSectionGetDof(submesh->coneSection, f, &dof));
2909       PetscCall(PetscSectionGetOffset(submesh->coneSection, f, &off));
2910       /* Yes, I know this is quadratic, but I expect the sizes to be <5 */
2911       for (d = 0; d < dof; ++d) {
2912         const PetscInt p = submesh->cones[off + d];
2913         PetscInt       v;
2914 
2915         for (v = 0; v < numFaceVertices; ++v) {
2916           if (subfaceVertices[v] == p) break;
2917         }
2918         if (v == numFaceVertices) break;
2919       }
2920       if (d == dof) {
2921         numFaces               = 1;
2922         ((PetscInt *)faces)[0] = f;
2923       }
2924     }
2925   }
2926 #endif
2927   PetscCheck(numFaces <= 1, comm, PETSC_ERR_ARG_WRONG, "Vertex set had %" PetscInt_FMT " faces, not one", numFaces);
2928   if (numFaces == 1) {
2929     /* Add the other cell neighbor for this face */
2930     PetscCall(DMPlexSetCone(subdm, subcell, faces));
2931   } else {
2932     PetscInt *indices, *origVertices, *orientedVertices, *orientedSubVertices, v, ov;
2933     PetscBool posOriented;
2934 
2935     PetscCall(DMGetWorkArray(subdm, 4 * numFaceVertices * sizeof(PetscInt), MPIU_INT, &orientedVertices));
2936     origVertices        = &orientedVertices[numFaceVertices];
2937     indices             = &orientedVertices[numFaceVertices * 2];
2938     orientedSubVertices = &orientedVertices[numFaceVertices * 3];
2939     PetscCall(DMPlexGetOrientedFace(dm, cell, numFaceVertices, faceVertices, numCorners, indices, origVertices, orientedVertices, &posOriented));
2940     /* TODO: I know that routine should return a permutation, not the indices */
2941     for (v = 0; v < numFaceVertices; ++v) {
2942       const PetscInt vertex = faceVertices[v], subvertex = subfaceVertices[v];
2943       for (ov = 0; ov < numFaceVertices; ++ov) {
2944         if (orientedVertices[ov] == vertex) {
2945           orientedSubVertices[ov] = subvertex;
2946           break;
2947         }
2948       }
2949       PetscCheck(ov != numFaceVertices, comm, PETSC_ERR_PLIB, "Could not find face vertex %" PetscInt_FMT " in orientated set", vertex);
2950     }
2951     PetscCall(DMPlexSetCone(subdm, *newFacePoint, orientedSubVertices));
2952     PetscCall(DMPlexSetCone(subdm, subcell, newFacePoint));
2953     PetscCall(DMRestoreWorkArray(subdm, 4 * numFaceVertices * sizeof(PetscInt), MPIU_INT, &orientedVertices));
2954     ++(*newFacePoint);
2955   }
2956 #if 0
2957   PetscCall(DMPlexRestoreJoin(subdm, numFaceVertices, subfaceVertices, &numFaces, &faces));
2958 #else
2959   PetscCall(DMRestoreWorkArray(subdm, 1, MPIU_INT, (void **)&faces));
2960 #endif
2961   PetscFunctionReturn(0);
2962 }
2963 
2964 static PetscErrorCode DMPlexCreateSubmesh_Uninterpolated(DM dm, DMLabel vertexLabel, PetscInt value, DM subdm) {
2965   MPI_Comm        comm;
2966   DMLabel         subpointMap;
2967   IS              subvertexIS, subcellIS;
2968   const PetscInt *subVertices, *subCells;
2969   PetscInt        numSubVertices, firstSubVertex, numSubCells;
2970   PetscInt       *subface, maxConeSize, numSubFaces = 0, firstSubFace, newFacePoint, nFV = 0;
2971   PetscInt        vStart, vEnd, c, f;
2972 
2973   PetscFunctionBegin;
2974   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
2975   /* Create subpointMap which marks the submesh */
2976   PetscCall(DMLabelCreate(PETSC_COMM_SELF, "subpoint_map", &subpointMap));
2977   PetscCall(DMPlexSetSubpointMap(subdm, subpointMap));
2978   PetscCall(DMLabelDestroy(&subpointMap));
2979   if (vertexLabel) PetscCall(DMPlexMarkSubmesh_Uninterpolated(dm, vertexLabel, value, subpointMap, &numSubFaces, &nFV, subdm));
2980   /* Setup chart */
2981   PetscCall(DMLabelGetStratumSize(subpointMap, 0, &numSubVertices));
2982   PetscCall(DMLabelGetStratumSize(subpointMap, 2, &numSubCells));
2983   PetscCall(DMPlexSetChart(subdm, 0, numSubCells + numSubFaces + numSubVertices));
2984   PetscCall(DMPlexSetVTKCellHeight(subdm, 1));
2985   /* Set cone sizes */
2986   firstSubVertex = numSubCells;
2987   firstSubFace   = numSubCells + numSubVertices;
2988   newFacePoint   = firstSubFace;
2989   PetscCall(DMLabelGetStratumIS(subpointMap, 0, &subvertexIS));
2990   if (subvertexIS) PetscCall(ISGetIndices(subvertexIS, &subVertices));
2991   PetscCall(DMLabelGetStratumIS(subpointMap, 2, &subcellIS));
2992   if (subcellIS) PetscCall(ISGetIndices(subcellIS, &subCells));
2993   for (c = 0; c < numSubCells; ++c) PetscCall(DMPlexSetConeSize(subdm, c, 1));
2994   for (f = firstSubFace; f < firstSubFace + numSubFaces; ++f) PetscCall(DMPlexSetConeSize(subdm, f, nFV));
2995   PetscCall(DMSetUp(subdm));
2996   /* Create face cones */
2997   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
2998   PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, NULL));
2999   PetscCall(DMGetWorkArray(subdm, maxConeSize, MPIU_INT, (void **)&subface));
3000   for (c = 0; c < numSubCells; ++c) {
3001     const PetscInt cell    = subCells[c];
3002     const PetscInt subcell = c;
3003     PetscInt      *closure = NULL;
3004     PetscInt       closureSize, cl, numCorners = 0, faceSize = 0;
3005 
3006     PetscCall(DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
3007     for (cl = 0; cl < closureSize * 2; cl += 2) {
3008       const PetscInt point = closure[cl];
3009       PetscInt       subVertex;
3010 
3011       if ((point >= vStart) && (point < vEnd)) {
3012         ++numCorners;
3013         PetscCall(PetscFindInt(point, numSubVertices, subVertices, &subVertex));
3014         if (subVertex >= 0) {
3015           closure[faceSize] = point;
3016           subface[faceSize] = firstSubVertex + subVertex;
3017           ++faceSize;
3018         }
3019       }
3020     }
3021     PetscCheck(faceSize <= nFV, comm, PETSC_ERR_ARG_WRONG, "Invalid submesh: Too many vertices %" PetscInt_FMT " of an element on the surface", faceSize);
3022     if (faceSize == nFV) PetscCall(DMPlexInsertFace_Internal(dm, subdm, faceSize, closure, subface, numCorners, cell, subcell, firstSubFace, &newFacePoint));
3023     PetscCall(DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
3024   }
3025   PetscCall(DMRestoreWorkArray(subdm, maxConeSize, MPIU_INT, (void **)&subface));
3026   PetscCall(DMPlexSymmetrize(subdm));
3027   PetscCall(DMPlexStratify(subdm));
3028   /* Build coordinates */
3029   {
3030     PetscSection coordSection, subCoordSection;
3031     Vec          coordinates, subCoordinates;
3032     PetscScalar *coords, *subCoords;
3033     PetscInt     numComp, coordSize, v;
3034     const char  *name;
3035 
3036     PetscCall(DMGetCoordinateSection(dm, &coordSection));
3037     PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
3038     PetscCall(DMGetCoordinateSection(subdm, &subCoordSection));
3039     PetscCall(PetscSectionSetNumFields(subCoordSection, 1));
3040     PetscCall(PetscSectionGetFieldComponents(coordSection, 0, &numComp));
3041     PetscCall(PetscSectionSetFieldComponents(subCoordSection, 0, numComp));
3042     PetscCall(PetscSectionSetChart(subCoordSection, firstSubVertex, firstSubVertex + numSubVertices));
3043     for (v = 0; v < numSubVertices; ++v) {
3044       const PetscInt vertex    = subVertices[v];
3045       const PetscInt subvertex = firstSubVertex + v;
3046       PetscInt       dof;
3047 
3048       PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3049       PetscCall(PetscSectionSetDof(subCoordSection, subvertex, dof));
3050       PetscCall(PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof));
3051     }
3052     PetscCall(PetscSectionSetUp(subCoordSection));
3053     PetscCall(PetscSectionGetStorageSize(subCoordSection, &coordSize));
3054     PetscCall(VecCreate(PETSC_COMM_SELF, &subCoordinates));
3055     PetscCall(PetscObjectGetName((PetscObject)coordinates, &name));
3056     PetscCall(PetscObjectSetName((PetscObject)subCoordinates, name));
3057     PetscCall(VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE));
3058     PetscCall(VecSetType(subCoordinates, VECSTANDARD));
3059     if (coordSize) {
3060       PetscCall(VecGetArray(coordinates, &coords));
3061       PetscCall(VecGetArray(subCoordinates, &subCoords));
3062       for (v = 0; v < numSubVertices; ++v) {
3063         const PetscInt vertex    = subVertices[v];
3064         const PetscInt subvertex = firstSubVertex + v;
3065         PetscInt       dof, off, sdof, soff, d;
3066 
3067         PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3068         PetscCall(PetscSectionGetOffset(coordSection, vertex, &off));
3069         PetscCall(PetscSectionGetDof(subCoordSection, subvertex, &sdof));
3070         PetscCall(PetscSectionGetOffset(subCoordSection, subvertex, &soff));
3071         PetscCheck(dof == sdof, comm, PETSC_ERR_PLIB, "Coordinate dimension %" PetscInt_FMT " on subvertex %" PetscInt_FMT ", vertex %" PetscInt_FMT " should be %" PetscInt_FMT, sdof, subvertex, vertex, dof);
3072         for (d = 0; d < dof; ++d) subCoords[soff + d] = coords[off + d];
3073       }
3074       PetscCall(VecRestoreArray(coordinates, &coords));
3075       PetscCall(VecRestoreArray(subCoordinates, &subCoords));
3076     }
3077     PetscCall(DMSetCoordinatesLocal(subdm, subCoordinates));
3078     PetscCall(VecDestroy(&subCoordinates));
3079   }
3080   /* Cleanup */
3081   if (subvertexIS) PetscCall(ISRestoreIndices(subvertexIS, &subVertices));
3082   PetscCall(ISDestroy(&subvertexIS));
3083   if (subcellIS) PetscCall(ISRestoreIndices(subcellIS, &subCells));
3084   PetscCall(ISDestroy(&subcellIS));
3085   PetscFunctionReturn(0);
3086 }
3087 
3088 /* TODO: Fix this to properly propogate up error conditions it may find */
3089 static inline PetscInt DMPlexFilterPoint_Internal(PetscInt point, PetscInt firstSubPoint, PetscInt numSubPoints, const PetscInt subPoints[]) {
3090   PetscInt       subPoint;
3091   PetscErrorCode ierr;
3092 
3093   ierr = PetscFindInt(point, numSubPoints, subPoints, &subPoint);
3094   if (ierr) return -1;
3095   return subPoint < 0 ? subPoint : firstSubPoint + subPoint;
3096 }
3097 
3098 /* TODO: Fix this to properly propogate up error conditions it may find */
3099 static inline PetscInt DMPlexFilterPointPerm_Internal(PetscInt point, PetscInt firstSubPoint, PetscInt numSubPoints, const PetscInt subPoints[], const PetscInt subIndices[]) {
3100   PetscInt       subPoint;
3101   PetscErrorCode ierr;
3102 
3103   ierr = PetscFindInt(point, numSubPoints, subPoints, &subPoint);
3104   if (ierr) return -1;
3105   return subPoint < 0 ? subPoint : firstSubPoint + (subIndices ? subIndices[subPoint] : subPoint);
3106 }
3107 
3108 static PetscErrorCode DMPlexFilterLabels_Internal(DM dm, const PetscInt numSubPoints[], const PetscInt *subpoints[], const PetscInt firstSubPoint[], DM subdm) {
3109   DMLabel  depthLabel;
3110   PetscInt Nl, l, d;
3111 
3112   PetscFunctionBegin;
3113   // Reset depth label for fast lookup
3114   PetscCall(DMPlexGetDepthLabel(dm, &depthLabel));
3115   PetscCall(DMLabelMakeAllInvalid_Internal(depthLabel));
3116   PetscCall(DMGetNumLabels(dm, &Nl));
3117   for (l = 0; l < Nl; ++l) {
3118     DMLabel         label, newlabel;
3119     const char     *lname;
3120     PetscBool       isDepth, isDim, isCelltype, isVTK;
3121     IS              valueIS;
3122     const PetscInt *values;
3123     PetscInt        Nv, v;
3124 
3125     PetscCall(DMGetLabelName(dm, l, &lname));
3126     PetscCall(PetscStrcmp(lname, "depth", &isDepth));
3127     PetscCall(PetscStrcmp(lname, "dim", &isDim));
3128     PetscCall(PetscStrcmp(lname, "celltype", &isCelltype));
3129     PetscCall(PetscStrcmp(lname, "vtk", &isVTK));
3130     if (isDepth || isDim || isCelltype || isVTK) continue;
3131     PetscCall(DMCreateLabel(subdm, lname));
3132     PetscCall(DMGetLabel(dm, lname, &label));
3133     PetscCall(DMGetLabel(subdm, lname, &newlabel));
3134     PetscCall(DMLabelGetDefaultValue(label, &v));
3135     PetscCall(DMLabelSetDefaultValue(newlabel, v));
3136     PetscCall(DMLabelGetValueIS(label, &valueIS));
3137     PetscCall(ISGetLocalSize(valueIS, &Nv));
3138     PetscCall(ISGetIndices(valueIS, &values));
3139     for (v = 0; v < Nv; ++v) {
3140       IS              pointIS;
3141       const PetscInt *points;
3142       PetscInt        Np, p;
3143 
3144       PetscCall(DMLabelGetStratumIS(label, values[v], &pointIS));
3145       PetscCall(ISGetLocalSize(pointIS, &Np));
3146       PetscCall(ISGetIndices(pointIS, &points));
3147       for (p = 0; p < Np; ++p) {
3148         const PetscInt point = points[p];
3149         PetscInt       subp;
3150 
3151         PetscCall(DMPlexGetPointDepth(dm, point, &d));
3152         subp = DMPlexFilterPoint_Internal(point, firstSubPoint[d], numSubPoints[d], subpoints[d]);
3153         if (subp >= 0) PetscCall(DMLabelSetValue(newlabel, subp, values[v]));
3154       }
3155       PetscCall(ISRestoreIndices(pointIS, &points));
3156       PetscCall(ISDestroy(&pointIS));
3157     }
3158     PetscCall(ISRestoreIndices(valueIS, &values));
3159     PetscCall(ISDestroy(&valueIS));
3160   }
3161   PetscFunctionReturn(0);
3162 }
3163 
3164 static PetscErrorCode DMPlexCreateSubmeshGeneric_Interpolated(DM dm, DMLabel label, PetscInt value, PetscBool markedFaces, PetscBool isCohesive, PetscInt cellHeight, DM subdm) {
3165   MPI_Comm         comm;
3166   DMLabel          subpointMap;
3167   IS              *subpointIS;
3168   const PetscInt **subpoints;
3169   PetscInt        *numSubPoints, *firstSubPoint, *coneNew, *orntNew;
3170   PetscInt         totSubPoints = 0, maxConeSize, dim, sdim, cdim, p, d, v;
3171   PetscMPIInt      rank;
3172 
3173   PetscFunctionBegin;
3174   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
3175   PetscCallMPI(MPI_Comm_rank(comm, &rank));
3176   /* Create subpointMap which marks the submesh */
3177   PetscCall(DMLabelCreate(PETSC_COMM_SELF, "subpoint_map", &subpointMap));
3178   PetscCall(DMPlexSetSubpointMap(subdm, subpointMap));
3179   if (cellHeight) {
3180     if (isCohesive) PetscCall(DMPlexMarkCohesiveSubmesh_Interpolated(dm, label, value, subpointMap, subdm));
3181     else PetscCall(DMPlexMarkSubmesh_Interpolated(dm, label, value, markedFaces, subpointMap, subdm));
3182   } else {
3183     DMLabel         depth;
3184     IS              pointIS;
3185     const PetscInt *points;
3186     PetscInt        numPoints = 0;
3187 
3188     PetscCall(DMPlexGetDepthLabel(dm, &depth));
3189     PetscCall(DMLabelGetStratumIS(label, value, &pointIS));
3190     if (pointIS) {
3191       PetscCall(ISGetIndices(pointIS, &points));
3192       PetscCall(ISGetLocalSize(pointIS, &numPoints));
3193     }
3194     for (p = 0; p < numPoints; ++p) {
3195       PetscInt *closure = NULL;
3196       PetscInt  closureSize, c, pdim;
3197 
3198       PetscCall(DMPlexGetTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closure));
3199       for (c = 0; c < closureSize * 2; c += 2) {
3200         PetscCall(DMLabelGetValue(depth, closure[c], &pdim));
3201         PetscCall(DMLabelSetValue(subpointMap, closure[c], pdim));
3202       }
3203       PetscCall(DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closure));
3204     }
3205     if (pointIS) PetscCall(ISRestoreIndices(pointIS, &points));
3206     PetscCall(ISDestroy(&pointIS));
3207   }
3208   /* Setup chart */
3209   PetscCall(DMGetDimension(dm, &dim));
3210   PetscCall(DMGetCoordinateDim(dm, &cdim));
3211   PetscCall(PetscMalloc4(dim + 1, &numSubPoints, dim + 1, &firstSubPoint, dim + 1, &subpointIS, dim + 1, &subpoints));
3212   for (d = 0; d <= dim; ++d) {
3213     PetscCall(DMLabelGetStratumSize(subpointMap, d, &numSubPoints[d]));
3214     totSubPoints += numSubPoints[d];
3215   }
3216   // Determine submesh dimension
3217   PetscCall(DMGetDimension(subdm, &sdim));
3218   if (sdim > 0) {
3219     // Calling function knows what dimension to use, and we include neighboring cells as well
3220     sdim = dim;
3221   } else {
3222     // We reset the subdimension based on what is being selected
3223     PetscInt lsdim;
3224     for (lsdim = dim; lsdim >= 0; --lsdim)
3225       if (numSubPoints[lsdim]) break;
3226     PetscCall(MPI_Allreduce(&lsdim, &sdim, 1, MPIU_INT, MPIU_MAX, comm));
3227     PetscCall(DMSetDimension(subdm, sdim));
3228     PetscCall(DMSetCoordinateDim(subdm, cdim));
3229   }
3230   PetscCall(DMPlexSetChart(subdm, 0, totSubPoints));
3231   PetscCall(DMPlexSetVTKCellHeight(subdm, cellHeight));
3232   /* Set cone sizes */
3233   firstSubPoint[sdim] = 0;
3234   firstSubPoint[0]    = firstSubPoint[sdim] + numSubPoints[sdim];
3235   if (sdim > 1) firstSubPoint[sdim - 1] = firstSubPoint[0] + numSubPoints[0];
3236   if (sdim > 2) firstSubPoint[sdim - 2] = firstSubPoint[sdim - 1] + numSubPoints[sdim - 1];
3237   for (d = 0; d <= sdim; ++d) {
3238     PetscCall(DMLabelGetStratumIS(subpointMap, d, &subpointIS[d]));
3239     if (subpointIS[d]) PetscCall(ISGetIndices(subpointIS[d], &subpoints[d]));
3240   }
3241   /* We do not want this label automatically computed, instead we compute it here */
3242   PetscCall(DMCreateLabel(subdm, "celltype"));
3243   for (d = 0; d <= sdim; ++d) {
3244     for (p = 0; p < numSubPoints[d]; ++p) {
3245       const PetscInt  point    = subpoints[d][p];
3246       const PetscInt  subpoint = firstSubPoint[d] + p;
3247       const PetscInt *cone;
3248       PetscInt        coneSize;
3249 
3250       PetscCall(DMPlexGetConeSize(dm, point, &coneSize));
3251       if (cellHeight && (d == sdim)) {
3252         PetscInt coneSizeNew, c, val;
3253 
3254         PetscCall(DMPlexGetCone(dm, point, &cone));
3255         for (c = 0, coneSizeNew = 0; c < coneSize; ++c) {
3256           PetscCall(DMLabelGetValue(subpointMap, cone[c], &val));
3257           if (val >= 0) coneSizeNew++;
3258         }
3259         PetscCall(DMPlexSetConeSize(subdm, subpoint, coneSizeNew));
3260         PetscCall(DMPlexSetCellType(subdm, subpoint, DM_POLYTOPE_FV_GHOST));
3261       } else {
3262         DMPolytopeType ct;
3263 
3264         PetscCall(DMPlexSetConeSize(subdm, subpoint, coneSize));
3265         PetscCall(DMPlexGetCellType(dm, point, &ct));
3266         PetscCall(DMPlexSetCellType(subdm, subpoint, ct));
3267       }
3268     }
3269   }
3270   PetscCall(DMLabelDestroy(&subpointMap));
3271   PetscCall(DMSetUp(subdm));
3272   /* Set cones */
3273   PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, NULL));
3274   PetscCall(PetscMalloc2(maxConeSize, &coneNew, maxConeSize, &orntNew));
3275   for (d = 0; d <= sdim; ++d) {
3276     for (p = 0; p < numSubPoints[d]; ++p) {
3277       const PetscInt  point    = subpoints[d][p];
3278       const PetscInt  subpoint = firstSubPoint[d] + p;
3279       const PetscInt *cone, *ornt;
3280       PetscInt        coneSize, subconeSize, coneSizeNew, c, subc, fornt = 0;
3281 
3282       if (d == sdim - 1) {
3283         const PetscInt *support, *cone, *ornt;
3284         PetscInt        supportSize, coneSize, s, subc;
3285 
3286         PetscCall(DMPlexGetSupport(dm, point, &support));
3287         PetscCall(DMPlexGetSupportSize(dm, point, &supportSize));
3288         for (s = 0; s < supportSize; ++s) {
3289           PetscBool isHybrid;
3290 
3291           PetscCall(DMPlexCellIsHybrid_Internal(dm, support[s], &isHybrid));
3292           if (!isHybrid) continue;
3293           PetscCall(PetscFindInt(support[s], numSubPoints[d + 1], subpoints[d + 1], &subc));
3294           if (subc >= 0) {
3295             const PetscInt ccell = subpoints[d + 1][subc];
3296 
3297             PetscCall(DMPlexGetCone(dm, ccell, &cone));
3298             PetscCall(DMPlexGetConeSize(dm, ccell, &coneSize));
3299             PetscCall(DMPlexGetConeOrientation(dm, ccell, &ornt));
3300             for (c = 0; c < coneSize; ++c) {
3301               if (cone[c] == point) {
3302                 fornt = ornt[c];
3303                 break;
3304               }
3305             }
3306             break;
3307           }
3308         }
3309       }
3310       PetscCall(DMPlexGetConeSize(dm, point, &coneSize));
3311       PetscCall(DMPlexGetConeSize(subdm, subpoint, &subconeSize));
3312       PetscCall(DMPlexGetCone(dm, point, &cone));
3313       PetscCall(DMPlexGetConeOrientation(dm, point, &ornt));
3314       for (c = 0, coneSizeNew = 0; c < coneSize; ++c) {
3315         PetscCall(PetscFindInt(cone[c], numSubPoints[d - 1], subpoints[d - 1], &subc));
3316         if (subc >= 0) {
3317           coneNew[coneSizeNew] = firstSubPoint[d - 1] + subc;
3318           orntNew[coneSizeNew] = ornt[c];
3319           ++coneSizeNew;
3320         }
3321       }
3322       PetscCheck(coneSizeNew == subconeSize, comm, PETSC_ERR_PLIB, "Number of cone points located %" PetscInt_FMT " does not match subcone size %" PetscInt_FMT, coneSizeNew, subconeSize);
3323       PetscCall(DMPlexSetCone(subdm, subpoint, coneNew));
3324       PetscCall(DMPlexSetConeOrientation(subdm, subpoint, orntNew));
3325       if (fornt < 0) PetscCall(DMPlexOrientPoint(subdm, subpoint, fornt));
3326     }
3327   }
3328   PetscCall(PetscFree2(coneNew, orntNew));
3329   PetscCall(DMPlexSymmetrize(subdm));
3330   PetscCall(DMPlexStratify(subdm));
3331   /* Build coordinates */
3332   {
3333     PetscSection coordSection, subCoordSection;
3334     Vec          coordinates, subCoordinates;
3335     PetscScalar *coords, *subCoords;
3336     PetscInt     cdim, numComp, coordSize;
3337     const char  *name;
3338 
3339     PetscCall(DMGetCoordinateDim(dm, &cdim));
3340     PetscCall(DMGetCoordinateSection(dm, &coordSection));
3341     PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
3342     PetscCall(DMGetCoordinateSection(subdm, &subCoordSection));
3343     PetscCall(PetscSectionSetNumFields(subCoordSection, 1));
3344     PetscCall(PetscSectionGetFieldComponents(coordSection, 0, &numComp));
3345     PetscCall(PetscSectionSetFieldComponents(subCoordSection, 0, numComp));
3346     PetscCall(PetscSectionSetChart(subCoordSection, firstSubPoint[0], firstSubPoint[0] + numSubPoints[0]));
3347     for (v = 0; v < numSubPoints[0]; ++v) {
3348       const PetscInt vertex    = subpoints[0][v];
3349       const PetscInt subvertex = firstSubPoint[0] + v;
3350       PetscInt       dof;
3351 
3352       PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3353       PetscCall(PetscSectionSetDof(subCoordSection, subvertex, dof));
3354       PetscCall(PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof));
3355     }
3356     PetscCall(PetscSectionSetUp(subCoordSection));
3357     PetscCall(PetscSectionGetStorageSize(subCoordSection, &coordSize));
3358     PetscCall(VecCreate(PETSC_COMM_SELF, &subCoordinates));
3359     PetscCall(PetscObjectGetName((PetscObject)coordinates, &name));
3360     PetscCall(PetscObjectSetName((PetscObject)subCoordinates, name));
3361     PetscCall(VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE));
3362     PetscCall(VecSetBlockSize(subCoordinates, cdim));
3363     PetscCall(VecSetType(subCoordinates, VECSTANDARD));
3364     PetscCall(VecGetArray(coordinates, &coords));
3365     PetscCall(VecGetArray(subCoordinates, &subCoords));
3366     for (v = 0; v < numSubPoints[0]; ++v) {
3367       const PetscInt vertex    = subpoints[0][v];
3368       const PetscInt subvertex = firstSubPoint[0] + v;
3369       PetscInt       dof, off, sdof, soff, d;
3370 
3371       PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3372       PetscCall(PetscSectionGetOffset(coordSection, vertex, &off));
3373       PetscCall(PetscSectionGetDof(subCoordSection, subvertex, &sdof));
3374       PetscCall(PetscSectionGetOffset(subCoordSection, subvertex, &soff));
3375       PetscCheck(dof == sdof, comm, PETSC_ERR_PLIB, "Coordinate dimension %" PetscInt_FMT " on subvertex %" PetscInt_FMT ", vertex %" PetscInt_FMT " should be %" PetscInt_FMT, sdof, subvertex, vertex, dof);
3376       for (d = 0; d < dof; ++d) subCoords[soff + d] = coords[off + d];
3377     }
3378     PetscCall(VecRestoreArray(coordinates, &coords));
3379     PetscCall(VecRestoreArray(subCoordinates, &subCoords));
3380     PetscCall(DMSetCoordinatesLocal(subdm, subCoordinates));
3381     PetscCall(VecDestroy(&subCoordinates));
3382   }
3383   /* Build SF: We need this complexity because subpoints might not be selected on the owning process */
3384   {
3385     PetscSF            sfPoint, sfPointSub;
3386     IS                 subpIS;
3387     const PetscSFNode *remotePoints;
3388     PetscSFNode       *sremotePoints = NULL, *newLocalPoints = NULL, *newOwners = NULL;
3389     const PetscInt    *localPoints, *subpoints, *rootdegree;
3390     PetscInt          *slocalPoints = NULL, *sortedPoints = NULL, *sortedIndices = NULL;
3391     PetscInt           numRoots, numLeaves, numSubpoints = 0, numSubroots, numSubleaves = 0, l, sl = 0, ll = 0, pStart, pEnd, p;
3392     PetscMPIInt        rank, size;
3393 
3394     PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
3395     PetscCallMPI(MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size));
3396     PetscCall(DMGetPointSF(dm, &sfPoint));
3397     PetscCall(DMGetPointSF(subdm, &sfPointSub));
3398     PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
3399     PetscCall(DMPlexGetChart(subdm, NULL, &numSubroots));
3400     PetscCall(DMPlexGetSubpointIS(subdm, &subpIS));
3401     if (subpIS) {
3402       PetscBool sorted = PETSC_TRUE;
3403 
3404       PetscCall(ISGetIndices(subpIS, &subpoints));
3405       PetscCall(ISGetLocalSize(subpIS, &numSubpoints));
3406       for (p = 1; p < numSubpoints; ++p) sorted = sorted && (subpoints[p] >= subpoints[p - 1]) ? PETSC_TRUE : PETSC_FALSE;
3407       if (!sorted) {
3408         PetscCall(PetscMalloc2(numSubpoints, &sortedPoints, numSubpoints, &sortedIndices));
3409         for (p = 0; p < numSubpoints; ++p) sortedIndices[p] = p;
3410         PetscCall(PetscArraycpy(sortedPoints, subpoints, numSubpoints));
3411         PetscCall(PetscSortIntWithArray(numSubpoints, sortedPoints, sortedIndices));
3412       }
3413     }
3414     PetscCall(PetscSFGetGraph(sfPoint, &numRoots, &numLeaves, &localPoints, &remotePoints));
3415     if (numRoots >= 0) {
3416       PetscCall(PetscSFComputeDegreeBegin(sfPoint, &rootdegree));
3417       PetscCall(PetscSFComputeDegreeEnd(sfPoint, &rootdegree));
3418       PetscCall(PetscMalloc2(pEnd - pStart, &newLocalPoints, numRoots, &newOwners));
3419       for (p = 0; p < pEnd - pStart; ++p) {
3420         newLocalPoints[p].rank  = -2;
3421         newLocalPoints[p].index = -2;
3422       }
3423       /* Set subleaves */
3424       for (l = 0; l < numLeaves; ++l) {
3425         const PetscInt point    = localPoints[l];
3426         const PetscInt subpoint = DMPlexFilterPointPerm_Internal(point, 0, numSubpoints, sortedPoints ? sortedPoints : subpoints, sortedIndices);
3427 
3428         if (subpoint < 0) continue;
3429         newLocalPoints[point - pStart].rank  = rank;
3430         newLocalPoints[point - pStart].index = subpoint;
3431         ++numSubleaves;
3432       }
3433       /* Must put in owned subpoints */
3434       for (p = pStart; p < pEnd; ++p) {
3435         newOwners[p - pStart].rank  = -3;
3436         newOwners[p - pStart].index = -3;
3437       }
3438       for (p = 0; p < numSubpoints; ++p) {
3439         /* Hold on to currently owned points */
3440         if (rootdegree[subpoints[p] - pStart]) newOwners[subpoints[p] - pStart].rank = rank + size;
3441         else newOwners[subpoints[p] - pStart].rank = rank;
3442         newOwners[subpoints[p] - pStart].index = p;
3443       }
3444       PetscCall(PetscSFReduceBegin(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC));
3445       PetscCall(PetscSFReduceEnd(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC));
3446       for (p = pStart; p < pEnd; ++p)
3447         if (newOwners[p - pStart].rank >= size) newOwners[p - pStart].rank -= size;
3448       PetscCall(PetscSFBcastBegin(sfPoint, MPIU_2INT, newOwners, newLocalPoints, MPI_REPLACE));
3449       PetscCall(PetscSFBcastEnd(sfPoint, MPIU_2INT, newOwners, newLocalPoints, MPI_REPLACE));
3450       PetscCall(PetscMalloc1(numSubleaves, &slocalPoints));
3451       PetscCall(PetscMalloc1(numSubleaves, &sremotePoints));
3452       for (l = 0; l < numLeaves; ++l) {
3453         const PetscInt point    = localPoints[l];
3454         const PetscInt subpoint = DMPlexFilterPointPerm_Internal(point, 0, numSubpoints, sortedPoints ? sortedPoints : subpoints, sortedIndices);
3455 
3456         if (subpoint < 0) continue;
3457         if (newLocalPoints[point].rank == rank) {
3458           ++ll;
3459           continue;
3460         }
3461         slocalPoints[sl]        = subpoint;
3462         sremotePoints[sl].rank  = newLocalPoints[point].rank;
3463         sremotePoints[sl].index = newLocalPoints[point].index;
3464         PetscCheck(sremotePoints[sl].rank >= 0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote rank for local point %" PetscInt_FMT, point);
3465         PetscCheck(sremotePoints[sl].index >= 0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote subpoint for local point %" PetscInt_FMT, point);
3466         ++sl;
3467       }
3468       PetscCheck(sl + ll == numSubleaves, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatch in number of subleaves %" PetscInt_FMT " + %" PetscInt_FMT " != %" PetscInt_FMT, sl, ll, numSubleaves);
3469       PetscCall(PetscFree2(newLocalPoints, newOwners));
3470       PetscCall(PetscSFSetGraph(sfPointSub, numSubroots, sl, slocalPoints, PETSC_OWN_POINTER, sremotePoints, PETSC_OWN_POINTER));
3471     }
3472     if (subpIS) PetscCall(ISRestoreIndices(subpIS, &subpoints));
3473     PetscCall(PetscFree2(sortedPoints, sortedIndices));
3474   }
3475   /* Filter labels */
3476   PetscCall(DMPlexFilterLabels_Internal(dm, numSubPoints, subpoints, firstSubPoint, subdm));
3477   /* Cleanup */
3478   for (d = 0; d <= sdim; ++d) {
3479     if (subpointIS[d]) PetscCall(ISRestoreIndices(subpointIS[d], &subpoints[d]));
3480     PetscCall(ISDestroy(&subpointIS[d]));
3481   }
3482   PetscCall(PetscFree4(numSubPoints, firstSubPoint, subpointIS, subpoints));
3483   PetscFunctionReturn(0);
3484 }
3485 
3486 static PetscErrorCode DMPlexCreateSubmesh_Interpolated(DM dm, DMLabel vertexLabel, PetscInt value, PetscBool markedFaces, DM subdm) {
3487   PetscFunctionBegin;
3488   PetscCall(DMPlexCreateSubmeshGeneric_Interpolated(dm, vertexLabel, value, markedFaces, PETSC_FALSE, 1, subdm));
3489   PetscFunctionReturn(0);
3490 }
3491 
3492 /*@
3493   DMPlexCreateSubmesh - Extract a hypersurface from the mesh using vertices defined by a label
3494 
3495   Input Parameters:
3496 + dm           - The original mesh
3497 . vertexLabel  - The DMLabel marking points contained in the surface
3498 . value        - The label value to use
3499 - markedFaces  - PETSC_TRUE if surface faces are marked in addition to vertices, PETSC_FALSE if only vertices are marked
3500 
3501   Output Parameter:
3502 . subdm - The surface mesh
3503 
3504   Note: This function produces a DMLabel mapping original points in the submesh to their depth. This can be obtained using DMPlexGetSubpointMap().
3505 
3506   Level: developer
3507 
3508 .seealso: `DMPlexGetSubpointMap()`, `DMGetLabel()`, `DMLabelSetValue()`
3509 @*/
3510 PetscErrorCode DMPlexCreateSubmesh(DM dm, DMLabel vertexLabel, PetscInt value, PetscBool markedFaces, DM *subdm) {
3511   DMPlexInterpolatedFlag interpolated;
3512   PetscInt               dim, cdim;
3513 
3514   PetscFunctionBegin;
3515   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3516   PetscValidPointer(subdm, 5);
3517   PetscCall(DMGetDimension(dm, &dim));
3518   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), subdm));
3519   PetscCall(DMSetType(*subdm, DMPLEX));
3520   PetscCall(DMSetDimension(*subdm, dim - 1));
3521   PetscCall(DMGetCoordinateDim(dm, &cdim));
3522   PetscCall(DMSetCoordinateDim(*subdm, cdim));
3523   PetscCall(DMPlexIsInterpolated(dm, &interpolated));
3524   PetscCheck(interpolated != DMPLEX_INTERPOLATED_PARTIAL, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Not for partially interpolated meshes");
3525   if (interpolated) {
3526     PetscCall(DMPlexCreateSubmesh_Interpolated(dm, vertexLabel, value, markedFaces, *subdm));
3527   } else {
3528     PetscCall(DMPlexCreateSubmesh_Uninterpolated(dm, vertexLabel, value, *subdm));
3529   }
3530   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_TRUE, *subdm));
3531   PetscFunctionReturn(0);
3532 }
3533 
3534 static PetscErrorCode DMPlexCreateCohesiveSubmesh_Uninterpolated(DM dm, PetscBool hasLagrange, const char label[], PetscInt value, DM subdm) {
3535   MPI_Comm        comm;
3536   DMLabel         subpointMap;
3537   IS              subvertexIS;
3538   const PetscInt *subVertices;
3539   PetscInt        numSubVertices, firstSubVertex, numSubCells, *subCells = NULL;
3540   PetscInt       *subface, maxConeSize, numSubFaces, firstSubFace, newFacePoint, nFV;
3541   PetscInt        c, f;
3542 
3543   PetscFunctionBegin;
3544   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
3545   /* Create subpointMap which marks the submesh */
3546   PetscCall(DMLabelCreate(PETSC_COMM_SELF, "subpoint_map", &subpointMap));
3547   PetscCall(DMPlexSetSubpointMap(subdm, subpointMap));
3548   PetscCall(DMLabelDestroy(&subpointMap));
3549   PetscCall(DMPlexMarkCohesiveSubmesh_Uninterpolated(dm, hasLagrange, label, value, subpointMap, &numSubFaces, &nFV, &subCells, subdm));
3550   /* Setup chart */
3551   PetscCall(DMLabelGetStratumSize(subpointMap, 0, &numSubVertices));
3552   PetscCall(DMLabelGetStratumSize(subpointMap, 2, &numSubCells));
3553   PetscCall(DMPlexSetChart(subdm, 0, numSubCells + numSubFaces + numSubVertices));
3554   PetscCall(DMPlexSetVTKCellHeight(subdm, 1));
3555   /* Set cone sizes */
3556   firstSubVertex = numSubCells;
3557   firstSubFace   = numSubCells + numSubVertices;
3558   newFacePoint   = firstSubFace;
3559   PetscCall(DMLabelGetStratumIS(subpointMap, 0, &subvertexIS));
3560   if (subvertexIS) PetscCall(ISGetIndices(subvertexIS, &subVertices));
3561   for (c = 0; c < numSubCells; ++c) PetscCall(DMPlexSetConeSize(subdm, c, 1));
3562   for (f = firstSubFace; f < firstSubFace + numSubFaces; ++f) PetscCall(DMPlexSetConeSize(subdm, f, nFV));
3563   PetscCall(DMSetUp(subdm));
3564   /* Create face cones */
3565   PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, NULL));
3566   PetscCall(DMGetWorkArray(subdm, maxConeSize, MPIU_INT, (void **)&subface));
3567   for (c = 0; c < numSubCells; ++c) {
3568     const PetscInt  cell    = subCells[c];
3569     const PetscInt  subcell = c;
3570     const PetscInt *cone, *cells;
3571     PetscBool       isHybrid;
3572     PetscInt        numCells, subVertex, p, v;
3573 
3574     PetscCall(DMPlexCellIsHybrid_Internal(dm, cell, &isHybrid));
3575     if (!isHybrid) continue;
3576     PetscCall(DMPlexGetCone(dm, cell, &cone));
3577     for (v = 0; v < nFV; ++v) {
3578       PetscCall(PetscFindInt(cone[v], numSubVertices, subVertices, &subVertex));
3579       subface[v] = firstSubVertex + subVertex;
3580     }
3581     PetscCall(DMPlexSetCone(subdm, newFacePoint, subface));
3582     PetscCall(DMPlexSetCone(subdm, subcell, &newFacePoint));
3583     PetscCall(DMPlexGetJoin(dm, nFV, cone, &numCells, &cells));
3584     /* Not true in parallel
3585     PetscCheck(numCells == 2,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive cells should separate two cells"); */
3586     for (p = 0; p < numCells; ++p) {
3587       PetscInt  negsubcell;
3588       PetscBool isHybrid;
3589 
3590       PetscCall(DMPlexCellIsHybrid_Internal(dm, cells[p], &isHybrid));
3591       if (isHybrid) continue;
3592       /* I know this is a crap search */
3593       for (negsubcell = 0; negsubcell < numSubCells; ++negsubcell) {
3594         if (subCells[negsubcell] == cells[p]) break;
3595       }
3596       PetscCheck(negsubcell != numSubCells, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not find negative face neighbor for cohesive cell %" PetscInt_FMT, cell);
3597       PetscCall(DMPlexSetCone(subdm, negsubcell, &newFacePoint));
3598     }
3599     PetscCall(DMPlexRestoreJoin(dm, nFV, cone, &numCells, &cells));
3600     ++newFacePoint;
3601   }
3602   PetscCall(DMRestoreWorkArray(subdm, maxConeSize, MPIU_INT, (void **)&subface));
3603   PetscCall(DMPlexSymmetrize(subdm));
3604   PetscCall(DMPlexStratify(subdm));
3605   /* Build coordinates */
3606   {
3607     PetscSection coordSection, subCoordSection;
3608     Vec          coordinates, subCoordinates;
3609     PetscScalar *coords, *subCoords;
3610     PetscInt     cdim, numComp, coordSize, v;
3611     const char  *name;
3612 
3613     PetscCall(DMGetCoordinateDim(dm, &cdim));
3614     PetscCall(DMGetCoordinateSection(dm, &coordSection));
3615     PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
3616     PetscCall(DMGetCoordinateSection(subdm, &subCoordSection));
3617     PetscCall(PetscSectionSetNumFields(subCoordSection, 1));
3618     PetscCall(PetscSectionGetFieldComponents(coordSection, 0, &numComp));
3619     PetscCall(PetscSectionSetFieldComponents(subCoordSection, 0, numComp));
3620     PetscCall(PetscSectionSetChart(subCoordSection, firstSubVertex, firstSubVertex + numSubVertices));
3621     for (v = 0; v < numSubVertices; ++v) {
3622       const PetscInt vertex    = subVertices[v];
3623       const PetscInt subvertex = firstSubVertex + v;
3624       PetscInt       dof;
3625 
3626       PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3627       PetscCall(PetscSectionSetDof(subCoordSection, subvertex, dof));
3628       PetscCall(PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof));
3629     }
3630     PetscCall(PetscSectionSetUp(subCoordSection));
3631     PetscCall(PetscSectionGetStorageSize(subCoordSection, &coordSize));
3632     PetscCall(VecCreate(PETSC_COMM_SELF, &subCoordinates));
3633     PetscCall(PetscObjectGetName((PetscObject)coordinates, &name));
3634     PetscCall(PetscObjectSetName((PetscObject)subCoordinates, name));
3635     PetscCall(VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE));
3636     PetscCall(VecSetBlockSize(subCoordinates, cdim));
3637     PetscCall(VecSetType(subCoordinates, VECSTANDARD));
3638     PetscCall(VecGetArray(coordinates, &coords));
3639     PetscCall(VecGetArray(subCoordinates, &subCoords));
3640     for (v = 0; v < numSubVertices; ++v) {
3641       const PetscInt vertex    = subVertices[v];
3642       const PetscInt subvertex = firstSubVertex + v;
3643       PetscInt       dof, off, sdof, soff, d;
3644 
3645       PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3646       PetscCall(PetscSectionGetOffset(coordSection, vertex, &off));
3647       PetscCall(PetscSectionGetDof(subCoordSection, subvertex, &sdof));
3648       PetscCall(PetscSectionGetOffset(subCoordSection, subvertex, &soff));
3649       PetscCheck(dof == sdof, comm, PETSC_ERR_PLIB, "Coordinate dimension %" PetscInt_FMT " on subvertex %" PetscInt_FMT ", vertex %" PetscInt_FMT " should be %" PetscInt_FMT, sdof, subvertex, vertex, dof);
3650       for (d = 0; d < dof; ++d) subCoords[soff + d] = coords[off + d];
3651     }
3652     PetscCall(VecRestoreArray(coordinates, &coords));
3653     PetscCall(VecRestoreArray(subCoordinates, &subCoords));
3654     PetscCall(DMSetCoordinatesLocal(subdm, subCoordinates));
3655     PetscCall(VecDestroy(&subCoordinates));
3656   }
3657   /* Build SF */
3658   CHKMEMQ;
3659   {
3660     PetscSF            sfPoint, sfPointSub;
3661     const PetscSFNode *remotePoints;
3662     PetscSFNode       *sremotePoints, *newLocalPoints, *newOwners;
3663     const PetscInt    *localPoints;
3664     PetscInt          *slocalPoints;
3665     PetscInt           numRoots, numLeaves, numSubRoots = numSubCells + numSubFaces + numSubVertices, numSubLeaves = 0, l, sl, ll, pStart, pEnd, p, vStart, vEnd;
3666     PetscMPIInt        rank;
3667 
3668     PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
3669     PetscCall(DMGetPointSF(dm, &sfPoint));
3670     PetscCall(DMGetPointSF(subdm, &sfPointSub));
3671     PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
3672     PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
3673     PetscCall(PetscSFGetGraph(sfPoint, &numRoots, &numLeaves, &localPoints, &remotePoints));
3674     if (numRoots >= 0) {
3675       /* Only vertices should be shared */
3676       PetscCall(PetscMalloc2(pEnd - pStart, &newLocalPoints, numRoots, &newOwners));
3677       for (p = 0; p < pEnd - pStart; ++p) {
3678         newLocalPoints[p].rank  = -2;
3679         newLocalPoints[p].index = -2;
3680       }
3681       /* Set subleaves */
3682       for (l = 0; l < numLeaves; ++l) {
3683         const PetscInt point    = localPoints[l];
3684         const PetscInt subPoint = DMPlexFilterPoint_Internal(point, firstSubVertex, numSubVertices, subVertices);
3685 
3686         PetscCheck(!(point < vStart) || !(point >= vEnd), PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Should not be mapping anything but vertices, %" PetscInt_FMT, point);
3687         if (subPoint < 0) continue;
3688         newLocalPoints[point - pStart].rank  = rank;
3689         newLocalPoints[point - pStart].index = subPoint;
3690         ++numSubLeaves;
3691       }
3692       /* Must put in owned subpoints */
3693       for (p = pStart; p < pEnd; ++p) {
3694         const PetscInt subPoint = DMPlexFilterPoint_Internal(p, firstSubVertex, numSubVertices, subVertices);
3695 
3696         if (subPoint < 0) {
3697           newOwners[p - pStart].rank  = -3;
3698           newOwners[p - pStart].index = -3;
3699         } else {
3700           newOwners[p - pStart].rank  = rank;
3701           newOwners[p - pStart].index = subPoint;
3702         }
3703       }
3704       PetscCall(PetscSFReduceBegin(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC));
3705       PetscCall(PetscSFReduceEnd(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC));
3706       PetscCall(PetscSFBcastBegin(sfPoint, MPIU_2INT, newOwners, newLocalPoints, MPI_REPLACE));
3707       PetscCall(PetscSFBcastEnd(sfPoint, MPIU_2INT, newOwners, newLocalPoints, MPI_REPLACE));
3708       PetscCall(PetscMalloc1(numSubLeaves, &slocalPoints));
3709       PetscCall(PetscMalloc1(numSubLeaves, &sremotePoints));
3710       for (l = 0, sl = 0, ll = 0; l < numLeaves; ++l) {
3711         const PetscInt point    = localPoints[l];
3712         const PetscInt subPoint = DMPlexFilterPoint_Internal(point, firstSubVertex, numSubVertices, subVertices);
3713 
3714         if (subPoint < 0) continue;
3715         if (newLocalPoints[point].rank == rank) {
3716           ++ll;
3717           continue;
3718         }
3719         slocalPoints[sl]        = subPoint;
3720         sremotePoints[sl].rank  = newLocalPoints[point].rank;
3721         sremotePoints[sl].index = newLocalPoints[point].index;
3722         PetscCheck(sremotePoints[sl].rank >= 0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote rank for local point %" PetscInt_FMT, point);
3723         PetscCheck(sremotePoints[sl].index >= 0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote subpoint for local point %" PetscInt_FMT, point);
3724         ++sl;
3725       }
3726       PetscCall(PetscFree2(newLocalPoints, newOwners));
3727       PetscCheck(sl + ll == numSubLeaves, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatch in number of subleaves %" PetscInt_FMT " + %" PetscInt_FMT " != %" PetscInt_FMT, sl, ll, numSubLeaves);
3728       PetscCall(PetscSFSetGraph(sfPointSub, numSubRoots, sl, slocalPoints, PETSC_OWN_POINTER, sremotePoints, PETSC_OWN_POINTER));
3729     }
3730   }
3731   CHKMEMQ;
3732   /* Cleanup */
3733   if (subvertexIS) PetscCall(ISRestoreIndices(subvertexIS, &subVertices));
3734   PetscCall(ISDestroy(&subvertexIS));
3735   PetscCall(PetscFree(subCells));
3736   PetscFunctionReturn(0);
3737 }
3738 
3739 static PetscErrorCode DMPlexCreateCohesiveSubmesh_Interpolated(DM dm, const char labelname[], PetscInt value, DM subdm) {
3740   DMLabel label = NULL;
3741 
3742   PetscFunctionBegin;
3743   if (labelname) PetscCall(DMGetLabel(dm, labelname, &label));
3744   PetscCall(DMPlexCreateSubmeshGeneric_Interpolated(dm, label, value, PETSC_FALSE, PETSC_TRUE, 1, subdm));
3745   PetscFunctionReturn(0);
3746 }
3747 
3748 /*@C
3749   DMPlexCreateCohesiveSubmesh - Extract from a mesh with cohesive cells the hypersurface defined by one face of the cells. Optionally, a Label can be given to restrict the cells.
3750 
3751   Input Parameters:
3752 + dm          - The original mesh
3753 . hasLagrange - The mesh has Lagrange unknowns in the cohesive cells
3754 . label       - A label name, or NULL
3755 - value  - A label value
3756 
3757   Output Parameter:
3758 . subdm - The surface mesh
3759 
3760   Note: This function produces a DMLabel mapping original points in the submesh to their depth. This can be obtained using DMPlexGetSubpointMap().
3761 
3762   Level: developer
3763 
3764 .seealso: `DMPlexGetSubpointMap()`, `DMPlexCreateSubmesh()`
3765 @*/
3766 PetscErrorCode DMPlexCreateCohesiveSubmesh(DM dm, PetscBool hasLagrange, const char label[], PetscInt value, DM *subdm) {
3767   PetscInt dim, cdim, depth;
3768 
3769   PetscFunctionBegin;
3770   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3771   PetscValidPointer(subdm, 5);
3772   PetscCall(DMGetDimension(dm, &dim));
3773   PetscCall(DMPlexGetDepth(dm, &depth));
3774   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), subdm));
3775   PetscCall(DMSetType(*subdm, DMPLEX));
3776   PetscCall(DMSetDimension(*subdm, dim - 1));
3777   PetscCall(DMGetCoordinateDim(dm, &cdim));
3778   PetscCall(DMSetCoordinateDim(*subdm, cdim));
3779   if (depth == dim) {
3780     PetscCall(DMPlexCreateCohesiveSubmesh_Interpolated(dm, label, value, *subdm));
3781   } else {
3782     PetscCall(DMPlexCreateCohesiveSubmesh_Uninterpolated(dm, hasLagrange, label, value, *subdm));
3783   }
3784   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_TRUE, *subdm));
3785   PetscFunctionReturn(0);
3786 }
3787 
3788 /*@
3789   DMPlexFilter - Extract a subset of mesh cells defined by a label as a separate mesh
3790 
3791   Input Parameters:
3792 + dm        - The original mesh
3793 . cellLabel - The DMLabel marking cells contained in the new mesh
3794 - value     - The label value to use
3795 
3796   Output Parameter:
3797 . subdm - The new mesh
3798 
3799   Notes:
3800   This function produces a `DMLabel` mapping original points in the submesh to their depth. This can be obtained using `DMPlexGetSubpointMap()`.
3801 
3802   Level: developer
3803 
3804 .seealso: `DMPlexGetSubpointMap()`, `DMGetLabel()`, `DMLabelSetValue()`, `DMPlexCreateSubmesh()`
3805 @*/
3806 PetscErrorCode DMPlexFilter(DM dm, DMLabel cellLabel, PetscInt value, DM *subdm) {
3807   PetscInt dim;
3808 
3809   PetscFunctionBegin;
3810   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3811   PetscValidPointer(subdm, 4);
3812   PetscCall(DMGetDimension(dm, &dim));
3813   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), subdm));
3814   PetscCall(DMSetType(*subdm, DMPLEX));
3815   /* Extract submesh in place, could be empty on some procs, could have inconsistency if procs do not both extract a shared cell */
3816   PetscCall(DMPlexCreateSubmeshGeneric_Interpolated(dm, cellLabel, value, PETSC_FALSE, PETSC_FALSE, 0, *subdm));
3817   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_TRUE, *subdm));
3818   PetscFunctionReturn(0);
3819 }
3820 
3821 /*@
3822   DMPlexGetSubpointMap - Returns a DMLabel with point dimension as values
3823 
3824   Input Parameter:
3825 . dm - The submesh DM
3826 
3827   Output Parameter:
3828 . subpointMap - The DMLabel of all the points from the original mesh in this submesh, or NULL if this is not a submesh
3829 
3830   Level: developer
3831 
3832 .seealso: `DMPlexCreateSubmesh()`, `DMPlexGetSubpointIS()`
3833 @*/
3834 PetscErrorCode DMPlexGetSubpointMap(DM dm, DMLabel *subpointMap) {
3835   PetscFunctionBegin;
3836   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3837   PetscValidPointer(subpointMap, 2);
3838   *subpointMap = ((DM_Plex *)dm->data)->subpointMap;
3839   PetscFunctionReturn(0);
3840 }
3841 
3842 /*@
3843   DMPlexSetSubpointMap - Sets the DMLabel with point dimension as values
3844 
3845   Input Parameters:
3846 + dm - The submesh DM
3847 - subpointMap - The DMLabel of all the points from the original mesh in this submesh
3848 
3849   Note: Should normally not be called by the user, since it is set in DMPlexCreateSubmesh()
3850 
3851   Level: developer
3852 
3853 .seealso: `DMPlexCreateSubmesh()`, `DMPlexGetSubpointIS()`
3854 @*/
3855 PetscErrorCode DMPlexSetSubpointMap(DM dm, DMLabel subpointMap) {
3856   DM_Plex *mesh = (DM_Plex *)dm->data;
3857   DMLabel  tmp;
3858 
3859   PetscFunctionBegin;
3860   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3861   tmp               = mesh->subpointMap;
3862   mesh->subpointMap = subpointMap;
3863   PetscCall(PetscObjectReference((PetscObject)mesh->subpointMap));
3864   PetscCall(DMLabelDestroy(&tmp));
3865   PetscFunctionReturn(0);
3866 }
3867 
3868 static PetscErrorCode DMPlexCreateSubpointIS_Internal(DM dm, IS *subpointIS) {
3869   DMLabel  spmap;
3870   PetscInt depth, d;
3871 
3872   PetscFunctionBegin;
3873   PetscCall(DMPlexGetSubpointMap(dm, &spmap));
3874   PetscCall(DMPlexGetDepth(dm, &depth));
3875   if (spmap && depth >= 0) {
3876     DM_Plex  *mesh = (DM_Plex *)dm->data;
3877     PetscInt *points, *depths;
3878     PetscInt  pStart, pEnd, p, off;
3879 
3880     PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
3881     PetscCheck(!pStart, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Submeshes must start the point numbering at 0, not %" PetscInt_FMT, pStart);
3882     PetscCall(PetscMalloc1(pEnd, &points));
3883     PetscCall(DMGetWorkArray(dm, depth + 1, MPIU_INT, &depths));
3884     depths[0] = depth;
3885     depths[1] = 0;
3886     for (d = 2; d <= depth; ++d) depths[d] = depth + 1 - d;
3887     for (d = 0, off = 0; d <= depth; ++d) {
3888       const PetscInt dep = depths[d];
3889       PetscInt       depStart, depEnd, n;
3890 
3891       PetscCall(DMPlexGetDepthStratum(dm, dep, &depStart, &depEnd));
3892       PetscCall(DMLabelGetStratumSize(spmap, dep, &n));
3893       if (((d < 2) && (depth > 1)) || (d == 1)) { /* Only check vertices and cells for now since the map is broken for others */
3894         PetscCheck(n == depEnd - depStart, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "The number of mapped submesh points %" PetscInt_FMT " at depth %" PetscInt_FMT " should be %" PetscInt_FMT, n, dep, depEnd - depStart);
3895       } else {
3896         if (!n) {
3897           if (d == 0) {
3898             /* Missing cells */
3899             for (p = 0; p < depEnd - depStart; ++p, ++off) points[off] = -1;
3900           } else {
3901             /* Missing faces */
3902             for (p = 0; p < depEnd - depStart; ++p, ++off) points[off] = PETSC_MAX_INT;
3903           }
3904         }
3905       }
3906       if (n) {
3907         IS              is;
3908         const PetscInt *opoints;
3909 
3910         PetscCall(DMLabelGetStratumIS(spmap, dep, &is));
3911         PetscCall(ISGetIndices(is, &opoints));
3912         for (p = 0; p < n; ++p, ++off) points[off] = opoints[p];
3913         PetscCall(ISRestoreIndices(is, &opoints));
3914         PetscCall(ISDestroy(&is));
3915       }
3916     }
3917     PetscCall(DMRestoreWorkArray(dm, depth + 1, MPIU_INT, &depths));
3918     PetscCheck(off == pEnd, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "The number of mapped submesh points %" PetscInt_FMT " should be %" PetscInt_FMT, off, pEnd);
3919     PetscCall(ISCreateGeneral(PETSC_COMM_SELF, pEnd, points, PETSC_OWN_POINTER, subpointIS));
3920     PetscCall(PetscObjectStateGet((PetscObject)spmap, &mesh->subpointState));
3921   }
3922   PetscFunctionReturn(0);
3923 }
3924 
3925 /*@
3926   DMPlexGetSubpointIS - Returns an IS covering the entire subdm chart with the original points as data
3927 
3928   Input Parameter:
3929 . dm - The submesh DM
3930 
3931   Output Parameter:
3932 . subpointIS - The IS of all the points from the original mesh in this submesh, or NULL if this is not a submesh
3933 
3934   Note: This IS is guaranteed to be sorted by the construction of the submesh
3935 
3936   Level: developer
3937 
3938 .seealso: `DMPlexCreateSubmesh()`, `DMPlexGetSubpointMap()`
3939 @*/
3940 PetscErrorCode DMPlexGetSubpointIS(DM dm, IS *subpointIS) {
3941   DM_Plex         *mesh = (DM_Plex *)dm->data;
3942   DMLabel          spmap;
3943   PetscObjectState state;
3944 
3945   PetscFunctionBegin;
3946   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3947   PetscValidPointer(subpointIS, 2);
3948   PetscCall(DMPlexGetSubpointMap(dm, &spmap));
3949   PetscCall(PetscObjectStateGet((PetscObject)spmap, &state));
3950   if (state != mesh->subpointState || !mesh->subpointIS) PetscCall(DMPlexCreateSubpointIS_Internal(dm, &mesh->subpointIS));
3951   *subpointIS = mesh->subpointIS;
3952   PetscFunctionReturn(0);
3953 }
3954 
3955 /*@
3956   DMGetEnclosureRelation - Get the relationship between dmA and dmB
3957 
3958   Input Parameters:
3959 + dmA - The first DM
3960 - dmB - The second DM
3961 
3962   Output Parameter:
3963 . rel - The relation of dmA to dmB
3964 
3965   Level: intermediate
3966 
3967 .seealso: `DMGetEnclosurePoint()`
3968 @*/
3969 PetscErrorCode DMGetEnclosureRelation(DM dmA, DM dmB, DMEnclosureType *rel) {
3970   DM       plexA, plexB, sdm;
3971   DMLabel  spmap;
3972   PetscInt pStartA, pEndA, pStartB, pEndB, NpA, NpB;
3973 
3974   PetscFunctionBegin;
3975   PetscValidPointer(rel, 3);
3976   *rel = DM_ENC_NONE;
3977   if (!dmA || !dmB) PetscFunctionReturn(0);
3978   PetscValidHeaderSpecific(dmA, DM_CLASSID, 1);
3979   PetscValidHeaderSpecific(dmB, DM_CLASSID, 2);
3980   if (dmA == dmB) {
3981     *rel = DM_ENC_EQUALITY;
3982     PetscFunctionReturn(0);
3983   }
3984   PetscCall(DMConvert(dmA, DMPLEX, &plexA));
3985   PetscCall(DMConvert(dmB, DMPLEX, &plexB));
3986   PetscCall(DMPlexGetChart(plexA, &pStartA, &pEndA));
3987   PetscCall(DMPlexGetChart(plexB, &pStartB, &pEndB));
3988   /* Assumption 1: subDMs have smaller charts than the DMs that they originate from
3989     - The degenerate case of a subdomain which includes all of the domain on some process can be treated as equality */
3990   if ((pStartA == pStartB) && (pEndA == pEndB)) {
3991     *rel = DM_ENC_EQUALITY;
3992     goto end;
3993   }
3994   NpA = pEndA - pStartA;
3995   NpB = pEndB - pStartB;
3996   if (NpA == NpB) goto end;
3997   sdm = NpA > NpB ? plexB : plexA; /* The other is the original, enclosing dm */
3998   PetscCall(DMPlexGetSubpointMap(sdm, &spmap));
3999   if (!spmap) goto end;
4000   /* TODO Check the space mapped to by subpointMap is same size as dm */
4001   if (NpA > NpB) {
4002     *rel = DM_ENC_SUPERMESH;
4003   } else {
4004     *rel = DM_ENC_SUBMESH;
4005   }
4006 end:
4007   PetscCall(DMDestroy(&plexA));
4008   PetscCall(DMDestroy(&plexB));
4009   PetscFunctionReturn(0);
4010 }
4011 
4012 /*@
4013   DMGetEnclosurePoint - Get the point pA in dmA which corresponds to the point pB in dmB
4014 
4015   Input Parameters:
4016 + dmA   - The first DM
4017 . dmB   - The second DM
4018 . etype - The type of enclosure relation that dmA has to dmB
4019 - pB    - A point of dmB
4020 
4021   Output Parameter:
4022 . pA    - The corresponding point of dmA
4023 
4024   Level: intermediate
4025 
4026 .seealso: `DMGetEnclosureRelation()`
4027 @*/
4028 PetscErrorCode DMGetEnclosurePoint(DM dmA, DM dmB, DMEnclosureType etype, PetscInt pB, PetscInt *pA) {
4029   DM              sdm;
4030   IS              subpointIS;
4031   const PetscInt *subpoints;
4032   PetscInt        numSubpoints;
4033 
4034   PetscFunctionBegin;
4035   /* TODO Cache the IS, making it look like an index */
4036   switch (etype) {
4037   case DM_ENC_SUPERMESH:
4038     sdm = dmB;
4039     PetscCall(DMPlexGetSubpointIS(sdm, &subpointIS));
4040     PetscCall(ISGetIndices(subpointIS, &subpoints));
4041     *pA = subpoints[pB];
4042     PetscCall(ISRestoreIndices(subpointIS, &subpoints));
4043     break;
4044   case DM_ENC_SUBMESH:
4045     sdm = dmA;
4046     PetscCall(DMPlexGetSubpointIS(sdm, &subpointIS));
4047     PetscCall(ISGetLocalSize(subpointIS, &numSubpoints));
4048     PetscCall(ISGetIndices(subpointIS, &subpoints));
4049     PetscCall(PetscFindInt(pB, numSubpoints, subpoints, pA));
4050     if (*pA < 0) {
4051       PetscCall(DMViewFromOptions(dmA, NULL, "-dm_enc_A_view"));
4052       PetscCall(DMViewFromOptions(dmB, NULL, "-dm_enc_B_view"));
4053       SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Point %" PetscInt_FMT " not found in submesh", pB);
4054     }
4055     PetscCall(ISRestoreIndices(subpointIS, &subpoints));
4056     break;
4057   case DM_ENC_EQUALITY:
4058   case DM_ENC_NONE: *pA = pB; break;
4059   case DM_ENC_UNKNOWN: {
4060     DMEnclosureType enc;
4061 
4062     PetscCall(DMGetEnclosureRelation(dmA, dmB, &enc));
4063     PetscCall(DMGetEnclosurePoint(dmA, dmB, enc, pB, pA));
4064   } break;
4065   default: SETERRQ(PetscObjectComm((PetscObject)dmA), PETSC_ERR_ARG_OUTOFRANGE, "Invalid enclosure type %d", (int)etype);
4066   }
4067   PetscFunctionReturn(0);
4068 }
4069