xref: /petsc/src/dm/impls/plex/plexsubmesh.c (revision 11a5261e40035b7c793f2783a2ba6c7cd4f3b077)
1 #include <petsc/private/dmpleximpl.h>  /*I      "petscdmplex.h"    I*/
2 #include <petsc/private/dmlabelimpl.h> /*I      "petscdmlabel.h"   I*/
3 #include <petscsf.h>
4 
5 static PetscErrorCode DMPlexCellIsHybrid_Internal(DM dm, PetscInt p, PetscBool *isHybrid) {
6   DMPolytopeType ct;
7 
8   PetscFunctionBegin;
9   PetscCall(DMPlexGetCellType(dm, p, &ct));
10   switch (ct) {
11   case DM_POLYTOPE_POINT_PRISM_TENSOR:
12   case DM_POLYTOPE_SEG_PRISM_TENSOR:
13   case DM_POLYTOPE_TRI_PRISM_TENSOR:
14   case DM_POLYTOPE_QUAD_PRISM_TENSOR: *isHybrid = PETSC_TRUE;
15   default: *isHybrid = PETSC_FALSE;
16   }
17   PetscFunctionReturn(0);
18 }
19 
20 static PetscErrorCode DMPlexGetTensorPrismBounds_Internal(DM dm, PetscInt dim, PetscInt *cStart, PetscInt *cEnd) {
21   DMLabel ctLabel;
22 
23   PetscFunctionBegin;
24   if (cStart) *cStart = -1;
25   if (cEnd) *cEnd = -1;
26   PetscCall(DMPlexGetCellTypeLabel(dm, &ctLabel));
27   switch (dim) {
28   case 1: PetscCall(DMLabelGetStratumBounds(ctLabel, DM_POLYTOPE_POINT_PRISM_TENSOR, cStart, cEnd)); break;
29   case 2: PetscCall(DMLabelGetStratumBounds(ctLabel, DM_POLYTOPE_SEG_PRISM_TENSOR, cStart, cEnd)); break;
30   case 3:
31     PetscCall(DMLabelGetStratumBounds(ctLabel, DM_POLYTOPE_TRI_PRISM_TENSOR, cStart, cEnd));
32     if (*cStart < 0) PetscCall(DMLabelGetStratumBounds(ctLabel, DM_POLYTOPE_QUAD_PRISM_TENSOR, cStart, cEnd));
33     break;
34   default: PetscFunctionReturn(0);
35   }
36   PetscFunctionReturn(0);
37 }
38 
39 static PetscErrorCode DMPlexMarkBoundaryFaces_Internal(DM dm, PetscInt val, PetscInt cellHeight, DMLabel label) {
40   PetscInt fStart, fEnd, f;
41 
42   PetscFunctionBegin;
43   PetscCall(DMPlexGetHeightStratum(dm, cellHeight + 1, &fStart, &fEnd));
44   for (f = fStart; f < fEnd; ++f) {
45     PetscInt supportSize;
46 
47     PetscCall(DMPlexGetSupportSize(dm, f, &supportSize));
48     if (supportSize == 1) {
49       if (val < 0) {
50         PetscInt *closure = NULL;
51         PetscInt  clSize, cl, cval;
52 
53         PetscCall(DMPlexGetTransitiveClosure(dm, f, PETSC_TRUE, &clSize, &closure));
54         for (cl = 0; cl < clSize * 2; cl += 2) {
55           PetscCall(DMLabelGetValue(label, closure[cl], &cval));
56           if (cval < 0) continue;
57           PetscCall(DMLabelSetValue(label, f, cval));
58           break;
59         }
60         if (cl == clSize * 2) PetscCall(DMLabelSetValue(label, f, 1));
61         PetscCall(DMPlexRestoreTransitiveClosure(dm, f, PETSC_TRUE, &clSize, &closure));
62       } else {
63         PetscCall(DMLabelSetValue(label, f, val));
64       }
65     }
66   }
67   PetscFunctionReturn(0);
68 }
69 
70 /*@
71   DMPlexMarkBoundaryFaces - Mark all faces on the boundary
72 
73   Not Collective
74 
75   Input Parameters:
76 + dm - The original DM
77 - val - The marker value, or PETSC_DETERMINE to use some value in the closure (or 1 if none are found)
78 
79   Output Parameter:
80 . label - The DMLabel marking boundary faces with the given value
81 
82   Level: developer
83 
84 .seealso: `DMLabelCreate()`, `DMCreateLabel()`
85 @*/
86 PetscErrorCode DMPlexMarkBoundaryFaces(DM dm, PetscInt val, DMLabel label) {
87   DMPlexInterpolatedFlag flg;
88 
89   PetscFunctionBegin;
90   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
91   PetscCall(DMPlexIsInterpolated(dm, &flg));
92   PetscCheck(flg == DMPLEX_INTERPOLATED_FULL, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM is not fully interpolated on this rank");
93   PetscCall(DMPlexMarkBoundaryFaces_Internal(dm, val, 0, label));
94   PetscFunctionReturn(0);
95 }
96 
97 static PetscErrorCode DMPlexLabelComplete_Internal(DM dm, DMLabel label, PetscBool completeCells) {
98   IS              valueIS;
99   PetscSF         sfPoint;
100   const PetscInt *values;
101   PetscInt        numValues, v, cStart, cEnd, nroots;
102 
103   PetscFunctionBegin;
104   PetscCall(DMLabelGetNumValues(label, &numValues));
105   PetscCall(DMLabelGetValueIS(label, &valueIS));
106   PetscCall(DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd));
107   PetscCall(ISGetIndices(valueIS, &values));
108   for (v = 0; v < numValues; ++v) {
109     IS              pointIS;
110     const PetscInt *points;
111     PetscInt        numPoints, p;
112 
113     PetscCall(DMLabelGetStratumSize(label, values[v], &numPoints));
114     PetscCall(DMLabelGetStratumIS(label, values[v], &pointIS));
115     PetscCall(ISGetIndices(pointIS, &points));
116     for (p = 0; p < numPoints; ++p) {
117       PetscInt  q       = points[p];
118       PetscInt *closure = NULL;
119       PetscInt  closureSize, c;
120 
121       if (cStart <= q && q < cEnd && !completeCells) { /* skip cells */
122         continue;
123       }
124       PetscCall(DMPlexGetTransitiveClosure(dm, q, PETSC_TRUE, &closureSize, &closure));
125       for (c = 0; c < closureSize * 2; c += 2) PetscCall(DMLabelSetValue(label, closure[c], values[v]));
126       PetscCall(DMPlexRestoreTransitiveClosure(dm, q, PETSC_TRUE, &closureSize, &closure));
127     }
128     PetscCall(ISRestoreIndices(pointIS, &points));
129     PetscCall(ISDestroy(&pointIS));
130   }
131   PetscCall(ISRestoreIndices(valueIS, &values));
132   PetscCall(ISDestroy(&valueIS));
133   PetscCall(DMGetPointSF(dm, &sfPoint));
134   PetscCall(PetscSFGetGraph(sfPoint, &nroots, NULL, NULL, NULL));
135   if (nroots >= 0) {
136     DMLabel         lblRoots, lblLeaves;
137     IS              valueIS, pointIS;
138     const PetscInt *values;
139     PetscInt        numValues, v;
140 
141     /* Pull point contributions from remote leaves into local roots */
142     PetscCall(DMLabelGather(label, sfPoint, &lblLeaves));
143     PetscCall(DMLabelGetValueIS(lblLeaves, &valueIS));
144     PetscCall(ISGetLocalSize(valueIS, &numValues));
145     PetscCall(ISGetIndices(valueIS, &values));
146     for (v = 0; v < numValues; ++v) {
147       const PetscInt value = values[v];
148 
149       PetscCall(DMLabelGetStratumIS(lblLeaves, value, &pointIS));
150       PetscCall(DMLabelInsertIS(label, pointIS, value));
151       PetscCall(ISDestroy(&pointIS));
152     }
153     PetscCall(ISRestoreIndices(valueIS, &values));
154     PetscCall(ISDestroy(&valueIS));
155     PetscCall(DMLabelDestroy(&lblLeaves));
156     /* Push point contributions from roots into remote leaves */
157     PetscCall(DMLabelDistribute(label, sfPoint, &lblRoots));
158     PetscCall(DMLabelGetValueIS(lblRoots, &valueIS));
159     PetscCall(ISGetLocalSize(valueIS, &numValues));
160     PetscCall(ISGetIndices(valueIS, &values));
161     for (v = 0; v < numValues; ++v) {
162       const PetscInt value = values[v];
163 
164       PetscCall(DMLabelGetStratumIS(lblRoots, value, &pointIS));
165       PetscCall(DMLabelInsertIS(label, pointIS, value));
166       PetscCall(ISDestroy(&pointIS));
167     }
168     PetscCall(ISRestoreIndices(valueIS, &values));
169     PetscCall(ISDestroy(&valueIS));
170     PetscCall(DMLabelDestroy(&lblRoots));
171   }
172   PetscFunctionReturn(0);
173 }
174 
175 /*@
176   DMPlexLabelComplete - Starting with a label marking points on a surface, we add the transitive closure to the surface
177 
178   Input Parameters:
179 + dm - The DM
180 - label - A DMLabel marking the surface points
181 
182   Output Parameter:
183 . label - A DMLabel marking all surface points in the transitive closure
184 
185   Level: developer
186 
187 .seealso: `DMPlexLabelCohesiveComplete()`
188 @*/
189 PetscErrorCode DMPlexLabelComplete(DM dm, DMLabel label) {
190   PetscFunctionBegin;
191   PetscCall(DMPlexLabelComplete_Internal(dm, label, PETSC_TRUE));
192   PetscFunctionReturn(0);
193 }
194 
195 /*@
196   DMPlexLabelAddCells - Starting with a label marking points on a surface, we add a cell for each point
197 
198   Input Parameters:
199 + dm - The DM
200 - label - A DMLabel marking the surface points
201 
202   Output Parameter:
203 . label - A DMLabel incorporating cells
204 
205   Level: developer
206 
207   Note: The cells allow FEM boundary conditions to be applied using the cell geometry
208 
209 .seealso: `DMPlexLabelAddFaceCells()`, `DMPlexLabelComplete()`, `DMPlexLabelCohesiveComplete()`
210 @*/
211 PetscErrorCode DMPlexLabelAddCells(DM dm, DMLabel label) {
212   IS              valueIS;
213   const PetscInt *values;
214   PetscInt        numValues, v, cStart, cEnd;
215 
216   PetscFunctionBegin;
217   PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd));
218   PetscCall(DMLabelGetNumValues(label, &numValues));
219   PetscCall(DMLabelGetValueIS(label, &valueIS));
220   PetscCall(ISGetIndices(valueIS, &values));
221   for (v = 0; v < numValues; ++v) {
222     IS              pointIS;
223     const PetscInt *points;
224     PetscInt        numPoints, p;
225 
226     PetscCall(DMLabelGetStratumSize(label, values[v], &numPoints));
227     PetscCall(DMLabelGetStratumIS(label, values[v], &pointIS));
228     PetscCall(ISGetIndices(pointIS, &points));
229     for (p = 0; p < numPoints; ++p) {
230       PetscInt *closure = NULL;
231       PetscInt  closureSize, cl;
232 
233       PetscCall(DMPlexGetTransitiveClosure(dm, points[p], PETSC_FALSE, &closureSize, &closure));
234       for (cl = closureSize - 1; cl > 0; --cl) {
235         const PetscInt cell = closure[cl * 2];
236         if ((cell >= cStart) && (cell < cEnd)) {
237           PetscCall(DMLabelSetValue(label, cell, values[v]));
238           break;
239         }
240       }
241       PetscCall(DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_FALSE, &closureSize, &closure));
242     }
243     PetscCall(ISRestoreIndices(pointIS, &points));
244     PetscCall(ISDestroy(&pointIS));
245   }
246   PetscCall(ISRestoreIndices(valueIS, &values));
247   PetscCall(ISDestroy(&valueIS));
248   PetscFunctionReturn(0);
249 }
250 
251 /*@
252   DMPlexLabelAddFaceCells - Starting with a label marking faces on a surface, we add a cell for each face
253 
254   Input Parameters:
255 + dm - The DM
256 - label - A DMLabel marking the surface points
257 
258   Output Parameter:
259 . label - A DMLabel incorporating cells
260 
261   Level: developer
262 
263   Note: The cells allow FEM boundary conditions to be applied using the cell geometry
264 
265 .seealso: `DMPlexLabelAddCells()`, `DMPlexLabelComplete()`, `DMPlexLabelCohesiveComplete()`
266 @*/
267 PetscErrorCode DMPlexLabelAddFaceCells(DM dm, DMLabel label) {
268   IS              valueIS;
269   const PetscInt *values;
270   PetscInt        numValues, v, cStart, cEnd, fStart, fEnd;
271 
272   PetscFunctionBegin;
273   PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd));
274   PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
275   PetscCall(DMLabelGetNumValues(label, &numValues));
276   PetscCall(DMLabelGetValueIS(label, &valueIS));
277   PetscCall(ISGetIndices(valueIS, &values));
278   for (v = 0; v < numValues; ++v) {
279     IS              pointIS;
280     const PetscInt *points;
281     PetscInt        numPoints, p;
282 
283     PetscCall(DMLabelGetStratumSize(label, values[v], &numPoints));
284     PetscCall(DMLabelGetStratumIS(label, values[v], &pointIS));
285     PetscCall(ISGetIndices(pointIS, &points));
286     for (p = 0; p < numPoints; ++p) {
287       const PetscInt face    = points[p];
288       PetscInt      *closure = NULL;
289       PetscInt       closureSize, cl;
290 
291       if ((face < fStart) || (face >= fEnd)) continue;
292       PetscCall(DMPlexGetTransitiveClosure(dm, face, PETSC_FALSE, &closureSize, &closure));
293       for (cl = closureSize - 1; cl > 0; --cl) {
294         const PetscInt cell = closure[cl * 2];
295         if ((cell >= cStart) && (cell < cEnd)) {
296           PetscCall(DMLabelSetValue(label, cell, values[v]));
297           break;
298         }
299       }
300       PetscCall(DMPlexRestoreTransitiveClosure(dm, face, PETSC_FALSE, &closureSize, &closure));
301     }
302     PetscCall(ISRestoreIndices(pointIS, &points));
303     PetscCall(ISDestroy(&pointIS));
304   }
305   PetscCall(ISRestoreIndices(valueIS, &values));
306   PetscCall(ISDestroy(&valueIS));
307   PetscFunctionReturn(0);
308 }
309 
310 /*@
311   DMPlexLabelClearCells - Remove cells from a label
312 
313   Input Parameters:
314 + dm - The DM
315 - label - A DMLabel marking surface points and their adjacent cells
316 
317   Output Parameter:
318 . label - A DMLabel without cells
319 
320   Level: developer
321 
322   Note: This undoes DMPlexLabelAddCells() or DMPlexLabelAddFaceCells()
323 
324 .seealso: `DMPlexLabelComplete()`, `DMPlexLabelCohesiveComplete()`, `DMPlexLabelAddCells()`
325 @*/
326 PetscErrorCode DMPlexLabelClearCells(DM dm, DMLabel label) {
327   IS              valueIS;
328   const PetscInt *values;
329   PetscInt        numValues, v, cStart, cEnd;
330 
331   PetscFunctionBegin;
332   PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd));
333   PetscCall(DMLabelGetNumValues(label, &numValues));
334   PetscCall(DMLabelGetValueIS(label, &valueIS));
335   PetscCall(ISGetIndices(valueIS, &values));
336   for (v = 0; v < numValues; ++v) {
337     IS              pointIS;
338     const PetscInt *points;
339     PetscInt        numPoints, p;
340 
341     PetscCall(DMLabelGetStratumSize(label, values[v], &numPoints));
342     PetscCall(DMLabelGetStratumIS(label, values[v], &pointIS));
343     PetscCall(ISGetIndices(pointIS, &points));
344     for (p = 0; p < numPoints; ++p) {
345       PetscInt point = points[p];
346 
347       if (point >= cStart && point < cEnd) PetscCall(DMLabelClearValue(label, point, values[v]));
348     }
349     PetscCall(ISRestoreIndices(pointIS, &points));
350     PetscCall(ISDestroy(&pointIS));
351   }
352   PetscCall(ISRestoreIndices(valueIS, &values));
353   PetscCall(ISDestroy(&valueIS));
354   PetscFunctionReturn(0);
355 }
356 
357 /* take (oldEnd, added) pairs, ordered by height and convert them to (oldstart, newstart) pairs, ordered by ascending
358  * index (skipping first, which is (0,0)) */
359 static inline PetscErrorCode DMPlexShiftPointSetUp_Internal(PetscInt depth, PetscInt depthShift[]) {
360   PetscInt d, off = 0;
361 
362   PetscFunctionBegin;
363   /* sort by (oldend): yes this is an O(n^2) sort, we expect depth <= 3 */
364   for (d = 0; d < depth; d++) {
365     PetscInt firstd     = d;
366     PetscInt firstStart = depthShift[2 * d];
367     PetscInt e;
368 
369     for (e = d + 1; e <= depth; e++) {
370       if (depthShift[2 * e] < firstStart) {
371         firstd     = e;
372         firstStart = depthShift[2 * d];
373       }
374     }
375     if (firstd != d) {
376       PetscInt swap[2];
377 
378       e                     = firstd;
379       swap[0]               = depthShift[2 * d];
380       swap[1]               = depthShift[2 * d + 1];
381       depthShift[2 * d]     = depthShift[2 * e];
382       depthShift[2 * d + 1] = depthShift[2 * e + 1];
383       depthShift[2 * e]     = swap[0];
384       depthShift[2 * e + 1] = swap[1];
385     }
386   }
387   /* convert (oldstart, added) to (oldstart, newstart) */
388   for (d = 0; d <= depth; d++) {
389     off += depthShift[2 * d + 1];
390     depthShift[2 * d + 1] = depthShift[2 * d] + off;
391   }
392   PetscFunctionReturn(0);
393 }
394 
395 /* depthShift is a list of (old, new) pairs */
396 static inline PetscInt DMPlexShiftPoint_Internal(PetscInt p, PetscInt depth, PetscInt depthShift[]) {
397   PetscInt d;
398   PetscInt newOff = 0;
399 
400   for (d = 0; d <= depth; d++) {
401     if (p < depthShift[2 * d]) return p + newOff;
402     else newOff = depthShift[2 * d + 1] - depthShift[2 * d];
403   }
404   return p + newOff;
405 }
406 
407 /* depthShift is a list of (old, new) pairs */
408 static inline PetscInt DMPlexShiftPointInverse_Internal(PetscInt p, PetscInt depth, PetscInt depthShift[]) {
409   PetscInt d;
410   PetscInt newOff = 0;
411 
412   for (d = 0; d <= depth; d++) {
413     if (p < depthShift[2 * d + 1]) return p + newOff;
414     else newOff = depthShift[2 * d] - depthShift[2 * d + 1];
415   }
416   return p + newOff;
417 }
418 
419 static PetscErrorCode DMPlexShiftSizes_Internal(DM dm, PetscInt depthShift[], DM dmNew) {
420   PetscInt depth = 0, d, pStart, pEnd, p;
421   DMLabel  depthLabel;
422 
423   PetscFunctionBegin;
424   PetscCall(DMPlexGetDepth(dm, &depth));
425   if (depth < 0) PetscFunctionReturn(0);
426   /* Step 1: Expand chart */
427   PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
428   pEnd = DMPlexShiftPoint_Internal(pEnd, depth, depthShift);
429   PetscCall(DMPlexSetChart(dmNew, pStart, pEnd));
430   PetscCall(DMCreateLabel(dmNew, "depth"));
431   PetscCall(DMPlexGetDepthLabel(dmNew, &depthLabel));
432   PetscCall(DMCreateLabel(dmNew, "celltype"));
433   /* Step 2: Set cone and support sizes */
434   for (d = 0; d <= depth; ++d) {
435     PetscInt pStartNew, pEndNew;
436     IS       pIS;
437 
438     PetscCall(DMPlexGetDepthStratum(dm, d, &pStart, &pEnd));
439     pStartNew = DMPlexShiftPoint_Internal(pStart, depth, depthShift);
440     pEndNew   = DMPlexShiftPoint_Internal(pEnd, depth, depthShift);
441     PetscCall(ISCreateStride(PETSC_COMM_SELF, pEndNew - pStartNew, pStartNew, 1, &pIS));
442     PetscCall(DMLabelSetStratumIS(depthLabel, d, pIS));
443     PetscCall(ISDestroy(&pIS));
444     for (p = pStart; p < pEnd; ++p) {
445       PetscInt       newp = DMPlexShiftPoint_Internal(p, depth, depthShift);
446       PetscInt       size;
447       DMPolytopeType ct;
448 
449       PetscCall(DMPlexGetConeSize(dm, p, &size));
450       PetscCall(DMPlexSetConeSize(dmNew, newp, size));
451       PetscCall(DMPlexGetSupportSize(dm, p, &size));
452       PetscCall(DMPlexSetSupportSize(dmNew, newp, size));
453       PetscCall(DMPlexGetCellType(dm, p, &ct));
454       PetscCall(DMPlexSetCellType(dmNew, newp, ct));
455     }
456   }
457   PetscFunctionReturn(0);
458 }
459 
460 static PetscErrorCode DMPlexShiftPoints_Internal(DM dm, PetscInt depthShift[], DM dmNew) {
461   PetscInt *newpoints;
462   PetscInt  depth = 0, maxConeSize, maxSupportSize, maxConeSizeNew, maxSupportSizeNew, pStart, pEnd, p;
463 
464   PetscFunctionBegin;
465   PetscCall(DMPlexGetDepth(dm, &depth));
466   if (depth < 0) PetscFunctionReturn(0);
467   PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize));
468   PetscCall(DMPlexGetMaxSizes(dmNew, &maxConeSizeNew, &maxSupportSizeNew));
469   PetscCall(PetscMalloc1(PetscMax(PetscMax(maxConeSize, maxSupportSize), PetscMax(maxConeSizeNew, maxSupportSizeNew)), &newpoints));
470   /* Step 5: Set cones and supports */
471   PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
472   for (p = pStart; p < pEnd; ++p) {
473     const PetscInt *points = NULL, *orientations = NULL;
474     PetscInt        size, sizeNew, i, newp = DMPlexShiftPoint_Internal(p, depth, depthShift);
475 
476     PetscCall(DMPlexGetConeSize(dm, p, &size));
477     PetscCall(DMPlexGetCone(dm, p, &points));
478     PetscCall(DMPlexGetConeOrientation(dm, p, &orientations));
479     for (i = 0; i < size; ++i) newpoints[i] = DMPlexShiftPoint_Internal(points[i], depth, depthShift);
480     PetscCall(DMPlexSetCone(dmNew, newp, newpoints));
481     PetscCall(DMPlexSetConeOrientation(dmNew, newp, orientations));
482     PetscCall(DMPlexGetSupportSize(dm, p, &size));
483     PetscCall(DMPlexGetSupportSize(dmNew, newp, &sizeNew));
484     PetscCall(DMPlexGetSupport(dm, p, &points));
485     for (i = 0; i < size; ++i) newpoints[i] = DMPlexShiftPoint_Internal(points[i], depth, depthShift);
486     for (i = size; i < sizeNew; ++i) newpoints[i] = 0;
487     PetscCall(DMPlexSetSupport(dmNew, newp, newpoints));
488   }
489   PetscCall(PetscFree(newpoints));
490   PetscFunctionReturn(0);
491 }
492 
493 static PetscErrorCode DMPlexShiftCoordinates_Internal(DM dm, PetscInt depthShift[], DM dmNew) {
494   PetscSection coordSection, newCoordSection;
495   Vec          coordinates, newCoordinates;
496   PetscScalar *coords, *newCoords;
497   PetscInt     coordSize, sStart, sEnd;
498   PetscInt     dim, depth = 0, cStart, cEnd, cStartNew, cEndNew, c, vStart, vEnd, vStartNew, vEndNew, v;
499   PetscBool    hasCells;
500 
501   PetscFunctionBegin;
502   PetscCall(DMGetCoordinateDim(dm, &dim));
503   PetscCall(DMSetCoordinateDim(dmNew, dim));
504   PetscCall(DMPlexGetDepth(dm, &depth));
505   /* Step 8: Convert coordinates */
506   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
507   PetscCall(DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd));
508   PetscCall(DMPlexGetDepthStratum(dmNew, 0, &vStartNew, &vEndNew));
509   PetscCall(DMPlexGetHeightStratum(dmNew, 0, &cStartNew, &cEndNew));
510   PetscCall(DMGetCoordinateSection(dm, &coordSection));
511   PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dm), &newCoordSection));
512   PetscCall(PetscSectionSetNumFields(newCoordSection, 1));
513   PetscCall(PetscSectionSetFieldComponents(newCoordSection, 0, dim));
514   PetscCall(PetscSectionGetChart(coordSection, &sStart, &sEnd));
515   hasCells = sStart == cStart ? PETSC_TRUE : PETSC_FALSE;
516   PetscCall(PetscSectionSetChart(newCoordSection, hasCells ? cStartNew : vStartNew, vEndNew));
517   if (hasCells) {
518     for (c = cStart; c < cEnd; ++c) {
519       PetscInt cNew = DMPlexShiftPoint_Internal(c, depth, depthShift), dof;
520 
521       PetscCall(PetscSectionGetDof(coordSection, c, &dof));
522       PetscCall(PetscSectionSetDof(newCoordSection, cNew, dof));
523       PetscCall(PetscSectionSetFieldDof(newCoordSection, cNew, 0, dof));
524     }
525   }
526   for (v = vStartNew; v < vEndNew; ++v) {
527     PetscCall(PetscSectionSetDof(newCoordSection, v, dim));
528     PetscCall(PetscSectionSetFieldDof(newCoordSection, v, 0, dim));
529   }
530   PetscCall(PetscSectionSetUp(newCoordSection));
531   PetscCall(DMSetCoordinateSection(dmNew, PETSC_DETERMINE, newCoordSection));
532   PetscCall(PetscSectionGetStorageSize(newCoordSection, &coordSize));
533   PetscCall(VecCreate(PETSC_COMM_SELF, &newCoordinates));
534   PetscCall(PetscObjectSetName((PetscObject)newCoordinates, "coordinates"));
535   PetscCall(VecSetSizes(newCoordinates, coordSize, PETSC_DETERMINE));
536   PetscCall(VecSetBlockSize(newCoordinates, dim));
537   PetscCall(VecSetType(newCoordinates, VECSTANDARD));
538   PetscCall(DMSetCoordinatesLocal(dmNew, newCoordinates));
539   PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
540   PetscCall(VecGetArray(coordinates, &coords));
541   PetscCall(VecGetArray(newCoordinates, &newCoords));
542   if (hasCells) {
543     for (c = cStart; c < cEnd; ++c) {
544       PetscInt cNew = DMPlexShiftPoint_Internal(c, depth, depthShift), dof, off, noff, d;
545 
546       PetscCall(PetscSectionGetDof(coordSection, c, &dof));
547       PetscCall(PetscSectionGetOffset(coordSection, c, &off));
548       PetscCall(PetscSectionGetOffset(newCoordSection, cNew, &noff));
549       for (d = 0; d < dof; ++d) newCoords[noff + d] = coords[off + d];
550     }
551   }
552   for (v = vStart; v < vEnd; ++v) {
553     PetscInt dof, off, noff, d;
554 
555     PetscCall(PetscSectionGetDof(coordSection, v, &dof));
556     PetscCall(PetscSectionGetOffset(coordSection, v, &off));
557     PetscCall(PetscSectionGetOffset(newCoordSection, DMPlexShiftPoint_Internal(v, depth, depthShift), &noff));
558     for (d = 0; d < dof; ++d) newCoords[noff + d] = coords[off + d];
559   }
560   PetscCall(VecRestoreArray(coordinates, &coords));
561   PetscCall(VecRestoreArray(newCoordinates, &newCoords));
562   PetscCall(VecDestroy(&newCoordinates));
563   PetscCall(PetscSectionDestroy(&newCoordSection));
564   PetscFunctionReturn(0);
565 }
566 
567 static PetscErrorCode DMPlexShiftSF_Single(DM dm, PetscInt depthShift[], PetscSF sf, PetscSF sfNew) {
568   const PetscSFNode *remotePoints;
569   PetscSFNode       *gremotePoints;
570   const PetscInt    *localPoints;
571   PetscInt          *glocalPoints, *newLocation, *newRemoteLocation;
572   PetscInt           numRoots, numLeaves, l, pStart, pEnd, depth = 0, totShift = 0;
573 
574   PetscFunctionBegin;
575   PetscCall(DMPlexGetDepth(dm, &depth));
576   PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
577   PetscCall(PetscSFGetGraph(sf, &numRoots, &numLeaves, &localPoints, &remotePoints));
578   totShift = DMPlexShiftPoint_Internal(pEnd, depth, depthShift) - pEnd;
579   if (numRoots >= 0) {
580     PetscCall(PetscMalloc2(numRoots, &newLocation, pEnd - pStart, &newRemoteLocation));
581     for (l = 0; l < numRoots; ++l) newLocation[l] = DMPlexShiftPoint_Internal(l, depth, depthShift);
582     PetscCall(PetscSFBcastBegin(sf, MPIU_INT, newLocation, newRemoteLocation, MPI_REPLACE));
583     PetscCall(PetscSFBcastEnd(sf, MPIU_INT, newLocation, newRemoteLocation, MPI_REPLACE));
584     PetscCall(PetscMalloc1(numLeaves, &glocalPoints));
585     PetscCall(PetscMalloc1(numLeaves, &gremotePoints));
586     for (l = 0; l < numLeaves; ++l) {
587       glocalPoints[l]        = DMPlexShiftPoint_Internal(localPoints[l], depth, depthShift);
588       gremotePoints[l].rank  = remotePoints[l].rank;
589       gremotePoints[l].index = newRemoteLocation[localPoints[l]];
590     }
591     PetscCall(PetscFree2(newLocation, newRemoteLocation));
592     PetscCall(PetscSFSetGraph(sfNew, numRoots + totShift, numLeaves, glocalPoints, PETSC_OWN_POINTER, gremotePoints, PETSC_OWN_POINTER));
593   }
594   PetscFunctionReturn(0);
595 }
596 
597 static PetscErrorCode DMPlexShiftSF_Internal(DM dm, PetscInt depthShift[], DM dmNew) {
598   PetscSF   sfPoint, sfPointNew;
599   PetscBool useNatural;
600 
601   PetscFunctionBegin;
602   /* Step 9: Convert pointSF */
603   PetscCall(DMGetPointSF(dm, &sfPoint));
604   PetscCall(DMGetPointSF(dmNew, &sfPointNew));
605   PetscCall(DMPlexShiftSF_Single(dm, depthShift, sfPoint, sfPointNew));
606   /* Step 9b: Convert naturalSF */
607   PetscCall(DMGetUseNatural(dm, &useNatural));
608   if (useNatural) {
609     PetscSF sfNat, sfNatNew;
610 
611     PetscCall(DMSetUseNatural(dmNew, useNatural));
612     PetscCall(DMGetNaturalSF(dm, &sfNat));
613     PetscCall(DMGetNaturalSF(dmNew, &sfNatNew));
614     PetscCall(DMPlexShiftSF_Single(dm, depthShift, sfNat, sfNatNew));
615   }
616   PetscFunctionReturn(0);
617 }
618 
619 static PetscErrorCode DMPlexShiftLabels_Internal(DM dm, PetscInt depthShift[], DM dmNew) {
620   PetscInt depth = 0, numLabels, l;
621 
622   PetscFunctionBegin;
623   PetscCall(DMPlexGetDepth(dm, &depth));
624   /* Step 10: Convert labels */
625   PetscCall(DMGetNumLabels(dm, &numLabels));
626   for (l = 0; l < numLabels; ++l) {
627     DMLabel         label, newlabel;
628     const char     *lname;
629     PetscBool       isDepth, isDim;
630     IS              valueIS;
631     const PetscInt *values;
632     PetscInt        numValues, val;
633 
634     PetscCall(DMGetLabelName(dm, l, &lname));
635     PetscCall(PetscStrcmp(lname, "depth", &isDepth));
636     if (isDepth) continue;
637     PetscCall(PetscStrcmp(lname, "dim", &isDim));
638     if (isDim) continue;
639     PetscCall(DMCreateLabel(dmNew, lname));
640     PetscCall(DMGetLabel(dm, lname, &label));
641     PetscCall(DMGetLabel(dmNew, lname, &newlabel));
642     PetscCall(DMLabelGetDefaultValue(label, &val));
643     PetscCall(DMLabelSetDefaultValue(newlabel, val));
644     PetscCall(DMLabelGetValueIS(label, &valueIS));
645     PetscCall(ISGetLocalSize(valueIS, &numValues));
646     PetscCall(ISGetIndices(valueIS, &values));
647     for (val = 0; val < numValues; ++val) {
648       IS              pointIS;
649       const PetscInt *points;
650       PetscInt        numPoints, p;
651 
652       PetscCall(DMLabelGetStratumIS(label, values[val], &pointIS));
653       PetscCall(ISGetLocalSize(pointIS, &numPoints));
654       PetscCall(ISGetIndices(pointIS, &points));
655       for (p = 0; p < numPoints; ++p) {
656         const PetscInt newpoint = DMPlexShiftPoint_Internal(points[p], depth, depthShift);
657 
658         PetscCall(DMLabelSetValue(newlabel, newpoint, values[val]));
659       }
660       PetscCall(ISRestoreIndices(pointIS, &points));
661       PetscCall(ISDestroy(&pointIS));
662     }
663     PetscCall(ISRestoreIndices(valueIS, &values));
664     PetscCall(ISDestroy(&valueIS));
665   }
666   PetscFunctionReturn(0);
667 }
668 
669 static PetscErrorCode DMPlexCreateVTKLabel_Internal(DM dm, PetscBool createGhostLabel, DM dmNew) {
670   PetscSF            sfPoint;
671   DMLabel            vtkLabel, ghostLabel = NULL;
672   const PetscSFNode *leafRemote;
673   const PetscInt    *leafLocal;
674   PetscInt           cellHeight, cStart, cEnd, c, fStart, fEnd, f, numLeaves, l;
675   PetscMPIInt        rank;
676 
677   PetscFunctionBegin;
678   /* Step 11: Make label for output (vtk) and to mark ghost points (ghost) */
679   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
680   PetscCall(DMGetPointSF(dm, &sfPoint));
681   PetscCall(DMPlexGetVTKCellHeight(dmNew, &cellHeight));
682   PetscCall(DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd));
683   PetscCall(PetscSFGetGraph(sfPoint, NULL, &numLeaves, &leafLocal, &leafRemote));
684   PetscCall(DMCreateLabel(dmNew, "vtk"));
685   PetscCall(DMGetLabel(dmNew, "vtk", &vtkLabel));
686   if (createGhostLabel) {
687     PetscCall(DMCreateLabel(dmNew, "ghost"));
688     PetscCall(DMGetLabel(dmNew, "ghost", &ghostLabel));
689   }
690   for (l = 0, c = cStart; l < numLeaves && c < cEnd; ++l, ++c) {
691     for (; c < leafLocal[l] && c < cEnd; ++c) PetscCall(DMLabelSetValue(vtkLabel, c, 1));
692     if (leafLocal[l] >= cEnd) break;
693     if (leafRemote[l].rank == rank) {
694       PetscCall(DMLabelSetValue(vtkLabel, c, 1));
695     } else if (ghostLabel) PetscCall(DMLabelSetValue(ghostLabel, c, 2));
696   }
697   for (; c < cEnd; ++c) PetscCall(DMLabelSetValue(vtkLabel, c, 1));
698   if (ghostLabel) {
699     PetscCall(DMPlexGetHeightStratum(dmNew, 1, &fStart, &fEnd));
700     for (f = fStart; f < fEnd; ++f) {
701       PetscInt numCells;
702 
703       PetscCall(DMPlexGetSupportSize(dmNew, f, &numCells));
704       if (numCells < 2) {
705         PetscCall(DMLabelSetValue(ghostLabel, f, 1));
706       } else {
707         const PetscInt *cells = NULL;
708         PetscInt        vA, vB;
709 
710         PetscCall(DMPlexGetSupport(dmNew, f, &cells));
711         PetscCall(DMLabelGetValue(vtkLabel, cells[0], &vA));
712         PetscCall(DMLabelGetValue(vtkLabel, cells[1], &vB));
713         if (vA != 1 && vB != 1) PetscCall(DMLabelSetValue(ghostLabel, f, 1));
714       }
715     }
716   }
717   PetscFunctionReturn(0);
718 }
719 
720 static PetscErrorCode DMPlexShiftTree_Internal(DM dm, PetscInt depthShift[], DM dmNew) {
721   DM           refTree;
722   PetscSection pSec;
723   PetscInt    *parents, *childIDs;
724 
725   PetscFunctionBegin;
726   PetscCall(DMPlexGetReferenceTree(dm, &refTree));
727   PetscCall(DMPlexSetReferenceTree(dmNew, refTree));
728   PetscCall(DMPlexGetTree(dm, &pSec, &parents, &childIDs, NULL, NULL));
729   if (pSec) {
730     PetscInt     p, pStart, pEnd, *parentsShifted, pStartShifted, pEndShifted, depth;
731     PetscInt    *childIDsShifted;
732     PetscSection pSecShifted;
733 
734     PetscCall(PetscSectionGetChart(pSec, &pStart, &pEnd));
735     PetscCall(DMPlexGetDepth(dm, &depth));
736     pStartShifted = DMPlexShiftPoint_Internal(pStart, depth, depthShift);
737     pEndShifted   = DMPlexShiftPoint_Internal(pEnd, depth, depthShift);
738     PetscCall(PetscMalloc2(pEndShifted - pStartShifted, &parentsShifted, pEndShifted - pStartShifted, &childIDsShifted));
739     PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dmNew), &pSecShifted));
740     PetscCall(PetscSectionSetChart(pSecShifted, pStartShifted, pEndShifted));
741     for (p = pStartShifted; p < pEndShifted; p++) {
742       /* start off assuming no children */
743       PetscCall(PetscSectionSetDof(pSecShifted, p, 0));
744     }
745     for (p = pStart; p < pEnd; p++) {
746       PetscInt dof;
747       PetscInt pNew = DMPlexShiftPoint_Internal(p, depth, depthShift);
748 
749       PetscCall(PetscSectionGetDof(pSec, p, &dof));
750       PetscCall(PetscSectionSetDof(pSecShifted, pNew, dof));
751     }
752     PetscCall(PetscSectionSetUp(pSecShifted));
753     for (p = pStart; p < pEnd; p++) {
754       PetscInt dof;
755       PetscInt pNew = DMPlexShiftPoint_Internal(p, depth, depthShift);
756 
757       PetscCall(PetscSectionGetDof(pSec, p, &dof));
758       if (dof) {
759         PetscInt off, offNew;
760 
761         PetscCall(PetscSectionGetOffset(pSec, p, &off));
762         PetscCall(PetscSectionGetOffset(pSecShifted, pNew, &offNew));
763         parentsShifted[offNew]  = DMPlexShiftPoint_Internal(parents[off], depth, depthShift);
764         childIDsShifted[offNew] = childIDs[off];
765       }
766     }
767     PetscCall(DMPlexSetTree(dmNew, pSecShifted, parentsShifted, childIDsShifted));
768     PetscCall(PetscFree2(parentsShifted, childIDsShifted));
769     PetscCall(PetscSectionDestroy(&pSecShifted));
770   }
771   PetscFunctionReturn(0);
772 }
773 
774 static PetscErrorCode DMPlexConstructGhostCells_Internal(DM dm, DMLabel label, PetscInt *numGhostCells, DM gdm) {
775   PetscSF          sf;
776   IS               valueIS;
777   const PetscInt  *values, *leaves;
778   PetscInt        *depthShift;
779   PetscInt         d, depth = 0, nleaves, loc, Ng, numFS, fs, fStart, fEnd, ghostCell, cEnd, c;
780   const PetscReal *maxCell, *Lstart, *L;
781 
782   PetscFunctionBegin;
783   PetscCall(DMGetPointSF(dm, &sf));
784   PetscCall(PetscSFGetGraph(sf, NULL, &nleaves, &leaves, NULL));
785   nleaves = PetscMax(0, nleaves);
786   PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
787   /* Count ghost cells */
788   PetscCall(DMLabelGetValueIS(label, &valueIS));
789   PetscCall(ISGetLocalSize(valueIS, &numFS));
790   PetscCall(ISGetIndices(valueIS, &values));
791   Ng = 0;
792   for (fs = 0; fs < numFS; ++fs) {
793     IS              faceIS;
794     const PetscInt *faces;
795     PetscInt        numFaces, f, numBdFaces = 0;
796 
797     PetscCall(DMLabelGetStratumIS(label, values[fs], &faceIS));
798     PetscCall(ISGetLocalSize(faceIS, &numFaces));
799     PetscCall(ISGetIndices(faceIS, &faces));
800     for (f = 0; f < numFaces; ++f) {
801       PetscInt numChildren;
802 
803       PetscCall(PetscFindInt(faces[f], nleaves, leaves, &loc));
804       PetscCall(DMPlexGetTreeChildren(dm, faces[f], &numChildren, NULL));
805       /* non-local and ancestors points don't get to register ghosts */
806       if (loc >= 0 || numChildren) continue;
807       if ((faces[f] >= fStart) && (faces[f] < fEnd)) ++numBdFaces;
808     }
809     Ng += numBdFaces;
810     PetscCall(ISRestoreIndices(faceIS, &faces));
811     PetscCall(ISDestroy(&faceIS));
812   }
813   PetscCall(DMPlexGetDepth(dm, &depth));
814   PetscCall(PetscMalloc1(2 * (depth + 1), &depthShift));
815   for (d = 0; d <= depth; d++) {
816     PetscInt dEnd;
817 
818     PetscCall(DMPlexGetDepthStratum(dm, d, NULL, &dEnd));
819     depthShift[2 * d]     = dEnd;
820     depthShift[2 * d + 1] = 0;
821   }
822   if (depth >= 0) depthShift[2 * depth + 1] = Ng;
823   PetscCall(DMPlexShiftPointSetUp_Internal(depth, depthShift));
824   PetscCall(DMPlexShiftSizes_Internal(dm, depthShift, gdm));
825   /* Step 3: Set cone/support sizes for new points */
826   PetscCall(DMPlexGetHeightStratum(dm, 0, NULL, &cEnd));
827   for (c = cEnd; c < cEnd + Ng; ++c) PetscCall(DMPlexSetConeSize(gdm, c, 1));
828   for (fs = 0; fs < numFS; ++fs) {
829     IS              faceIS;
830     const PetscInt *faces;
831     PetscInt        numFaces, f;
832 
833     PetscCall(DMLabelGetStratumIS(label, values[fs], &faceIS));
834     PetscCall(ISGetLocalSize(faceIS, &numFaces));
835     PetscCall(ISGetIndices(faceIS, &faces));
836     for (f = 0; f < numFaces; ++f) {
837       PetscInt size, numChildren;
838 
839       PetscCall(PetscFindInt(faces[f], nleaves, leaves, &loc));
840       PetscCall(DMPlexGetTreeChildren(dm, faces[f], &numChildren, NULL));
841       if (loc >= 0 || numChildren) continue;
842       if ((faces[f] < fStart) || (faces[f] >= fEnd)) continue;
843       PetscCall(DMPlexGetSupportSize(dm, faces[f], &size));
844       PetscCheck(size == 1, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM has boundary face %" PetscInt_FMT " with %" PetscInt_FMT " support cells", faces[f], size);
845       PetscCall(DMPlexSetSupportSize(gdm, faces[f] + Ng, 2));
846     }
847     PetscCall(ISRestoreIndices(faceIS, &faces));
848     PetscCall(ISDestroy(&faceIS));
849   }
850   /* Step 4: Setup ghosted DM */
851   PetscCall(DMSetUp(gdm));
852   PetscCall(DMPlexShiftPoints_Internal(dm, depthShift, gdm));
853   /* Step 6: Set cones and supports for new points */
854   ghostCell = cEnd;
855   for (fs = 0; fs < numFS; ++fs) {
856     IS              faceIS;
857     const PetscInt *faces;
858     PetscInt        numFaces, f;
859 
860     PetscCall(DMLabelGetStratumIS(label, values[fs], &faceIS));
861     PetscCall(ISGetLocalSize(faceIS, &numFaces));
862     PetscCall(ISGetIndices(faceIS, &faces));
863     for (f = 0; f < numFaces; ++f) {
864       PetscInt newFace = faces[f] + Ng, numChildren;
865 
866       PetscCall(PetscFindInt(faces[f], nleaves, leaves, &loc));
867       PetscCall(DMPlexGetTreeChildren(dm, faces[f], &numChildren, NULL));
868       if (loc >= 0 || numChildren) continue;
869       if ((faces[f] < fStart) || (faces[f] >= fEnd)) continue;
870       PetscCall(DMPlexSetCone(gdm, ghostCell, &newFace));
871       PetscCall(DMPlexInsertSupport(gdm, newFace, 1, ghostCell));
872       ++ghostCell;
873     }
874     PetscCall(ISRestoreIndices(faceIS, &faces));
875     PetscCall(ISDestroy(&faceIS));
876   }
877   PetscCall(ISRestoreIndices(valueIS, &values));
878   PetscCall(ISDestroy(&valueIS));
879   PetscCall(DMPlexShiftCoordinates_Internal(dm, depthShift, gdm));
880   PetscCall(DMPlexShiftSF_Internal(dm, depthShift, gdm));
881   PetscCall(DMPlexShiftLabels_Internal(dm, depthShift, gdm));
882   PetscCall(DMPlexCreateVTKLabel_Internal(dm, PETSC_TRUE, gdm));
883   PetscCall(DMPlexShiftTree_Internal(dm, depthShift, gdm));
884   PetscCall(PetscFree(depthShift));
885   for (c = cEnd; c < cEnd + Ng; ++c) PetscCall(DMPlexSetCellType(gdm, c, DM_POLYTOPE_FV_GHOST));
886   /* Step 7: Periodicity */
887   PetscCall(DMGetPeriodicity(dm, &maxCell, &Lstart, &L));
888   PetscCall(DMSetPeriodicity(gdm, maxCell, Lstart, L));
889   if (numGhostCells) *numGhostCells = Ng;
890   PetscFunctionReturn(0);
891 }
892 
893 /*@C
894   DMPlexConstructGhostCells - Construct ghost cells which connect to every boundary face
895 
896   Collective on dm
897 
898   Input Parameters:
899 + dm - The original DM
900 - labelName - The label specifying the boundary faces, or "Face Sets" if this is NULL
901 
902   Output Parameters:
903 + numGhostCells - The number of ghost cells added to the DM
904 - dmGhosted - The new DM
905 
906   Note: If no label exists of that name, one will be created marking all boundary faces
907 
908   Level: developer
909 
910 .seealso: `DMCreate()`
911 @*/
912 PetscErrorCode DMPlexConstructGhostCells(DM dm, const char labelName[], PetscInt *numGhostCells, DM *dmGhosted) {
913   DM          gdm;
914   DMLabel     label;
915   const char *name = labelName ? labelName : "Face Sets";
916   PetscInt    dim, Ng = 0;
917   PetscBool   useCone, useClosure;
918 
919   PetscFunctionBegin;
920   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
921   if (numGhostCells) PetscValidIntPointer(numGhostCells, 3);
922   PetscValidPointer(dmGhosted, 4);
923   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &gdm));
924   PetscCall(DMSetType(gdm, DMPLEX));
925   PetscCall(DMGetDimension(dm, &dim));
926   PetscCall(DMSetDimension(gdm, dim));
927   PetscCall(DMGetBasicAdjacency(dm, &useCone, &useClosure));
928   PetscCall(DMSetBasicAdjacency(gdm, useCone, useClosure));
929   PetscCall(DMGetLabel(dm, name, &label));
930   if (!label) {
931     /* Get label for boundary faces */
932     PetscCall(DMCreateLabel(dm, name));
933     PetscCall(DMGetLabel(dm, name, &label));
934     PetscCall(DMPlexMarkBoundaryFaces(dm, 1, label));
935   }
936   PetscCall(DMPlexConstructGhostCells_Internal(dm, label, &Ng, gdm));
937   PetscCall(DMCopyDisc(dm, gdm));
938   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_TRUE, gdm));
939   gdm->setfromoptionscalled = dm->setfromoptionscalled;
940   if (numGhostCells) *numGhostCells = Ng;
941   *dmGhosted = gdm;
942   PetscFunctionReturn(0);
943 }
944 
945 static PetscErrorCode DivideCells_Private(DM dm, DMLabel label, DMPlexPointQueue queue) {
946   PetscInt dim, d, shift = 100, *pStart, *pEnd;
947 
948   PetscFunctionBegin;
949   PetscCall(DMGetDimension(dm, &dim));
950   PetscCall(PetscMalloc2(dim, &pStart, dim, &pEnd));
951   for (d = 0; d < dim; ++d) PetscCall(DMPlexGetDepthStratum(dm, d, &pStart[d], &pEnd[d]));
952   while (!DMPlexPointQueueEmpty(queue)) {
953     PetscInt  cell    = -1;
954     PetscInt *closure = NULL;
955     PetscInt  closureSize, cl, cval;
956 
957     PetscCall(DMPlexPointQueueDequeue(queue, &cell));
958     PetscCall(DMLabelGetValue(label, cell, &cval));
959     PetscCall(DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
960     /* Mark points in the cell closure that touch the fault */
961     for (d = 0; d < dim; ++d) {
962       for (cl = 0; cl < closureSize * 2; cl += 2) {
963         const PetscInt clp = closure[cl];
964         PetscInt       clval;
965 
966         if ((clp < pStart[d]) || (clp >= pEnd[d])) continue;
967         PetscCall(DMLabelGetValue(label, clp, &clval));
968         if (clval == -1) {
969           const PetscInt *cone;
970           PetscInt        coneSize, c;
971 
972           /* If a cone point touches the fault, then this point touches the fault */
973           PetscCall(DMPlexGetCone(dm, clp, &cone));
974           PetscCall(DMPlexGetConeSize(dm, clp, &coneSize));
975           for (c = 0; c < coneSize; ++c) {
976             PetscInt cpval;
977 
978             PetscCall(DMLabelGetValue(label, cone[c], &cpval));
979             if (cpval != -1) {
980               PetscInt dep;
981 
982               PetscCall(DMPlexGetPointDepth(dm, clp, &dep));
983               clval = cval < 0 ? -(shift + dep) : shift + dep;
984               PetscCall(DMLabelSetValue(label, clp, clval));
985               break;
986             }
987           }
988         }
989         /* Mark neighbor cells through marked faces (these cells must also touch the fault) */
990         if (d == dim - 1 && clval != -1) {
991           const PetscInt *support;
992           PetscInt        supportSize, s, nval;
993 
994           PetscCall(DMPlexGetSupport(dm, clp, &support));
995           PetscCall(DMPlexGetSupportSize(dm, clp, &supportSize));
996           for (s = 0; s < supportSize; ++s) {
997             PetscCall(DMLabelGetValue(label, support[s], &nval));
998             if (nval == -1) {
999               PetscCall(DMLabelSetValue(label, support[s], clval < 0 ? clval - 1 : clval + 1));
1000               PetscCall(DMPlexPointQueueEnqueue(queue, support[s]));
1001             }
1002           }
1003         }
1004       }
1005     }
1006     PetscCall(DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
1007   }
1008   PetscCall(PetscFree2(pStart, pEnd));
1009   PetscFunctionReturn(0);
1010 }
1011 
1012 typedef struct {
1013   DM               dm;
1014   DMPlexPointQueue queue;
1015 } PointDivision;
1016 
1017 static PetscErrorCode divideCell(DMLabel label, PetscInt p, PetscInt val, void *ctx) {
1018   PointDivision  *div  = (PointDivision *)ctx;
1019   PetscInt        cval = val < 0 ? val - 1 : val + 1;
1020   const PetscInt *support;
1021   PetscInt        supportSize, s;
1022 
1023   PetscFunctionBegin;
1024   PetscCall(DMPlexGetSupport(div->dm, p, &support));
1025   PetscCall(DMPlexGetSupportSize(div->dm, p, &supportSize));
1026   for (s = 0; s < supportSize; ++s) {
1027     PetscCall(DMLabelSetValue(label, support[s], cval));
1028     PetscCall(DMPlexPointQueueEnqueue(div->queue, support[s]));
1029   }
1030   PetscFunctionReturn(0);
1031 }
1032 
1033 /* Mark cells by label propagation */
1034 static PetscErrorCode DMPlexLabelFaultHalo(DM dm, DMLabel faultLabel) {
1035   DMPlexPointQueue queue = NULL;
1036   PointDivision    div;
1037   PetscSF          pointSF;
1038   IS               pointIS;
1039   const PetscInt  *points;
1040   PetscBool        empty;
1041   PetscInt         dim, shift = 100, n, i;
1042 
1043   PetscFunctionBegin;
1044   PetscCall(DMGetDimension(dm, &dim));
1045   PetscCall(DMPlexPointQueueCreate(1024, &queue));
1046   div.dm    = dm;
1047   div.queue = queue;
1048   /* Enqueue cells on fault */
1049   PetscCall(DMLabelGetStratumIS(faultLabel, shift + dim, &pointIS));
1050   if (pointIS) {
1051     PetscCall(ISGetLocalSize(pointIS, &n));
1052     PetscCall(ISGetIndices(pointIS, &points));
1053     for (i = 0; i < n; ++i) PetscCall(DMPlexPointQueueEnqueue(queue, points[i]));
1054     PetscCall(ISRestoreIndices(pointIS, &points));
1055     PetscCall(ISDestroy(&pointIS));
1056   }
1057   PetscCall(DMLabelGetStratumIS(faultLabel, -(shift + dim), &pointIS));
1058   if (pointIS) {
1059     PetscCall(ISGetLocalSize(pointIS, &n));
1060     PetscCall(ISGetIndices(pointIS, &points));
1061     for (i = 0; i < n; ++i) PetscCall(DMPlexPointQueueEnqueue(queue, points[i]));
1062     PetscCall(ISRestoreIndices(pointIS, &points));
1063     PetscCall(ISDestroy(&pointIS));
1064   }
1065 
1066   PetscCall(DMGetPointSF(dm, &pointSF));
1067   PetscCall(DMLabelPropagateBegin(faultLabel, pointSF));
1068   /* While edge queue is not empty: */
1069   PetscCall(DMPlexPointQueueEmptyCollective((PetscObject)dm, queue, &empty));
1070   while (!empty) {
1071     PetscCall(DivideCells_Private(dm, faultLabel, queue));
1072     PetscCall(DMLabelPropagatePush(faultLabel, pointSF, divideCell, &div));
1073     PetscCall(DMPlexPointQueueEmptyCollective((PetscObject)dm, queue, &empty));
1074   }
1075   PetscCall(DMLabelPropagateEnd(faultLabel, pointSF));
1076   PetscCall(DMPlexPointQueueDestroy(&queue));
1077   PetscFunctionReturn(0);
1078 }
1079 
1080 /*
1081   We are adding three kinds of points here:
1082     Replicated:     Copies of points which exist in the mesh, such as vertices identified across a fault
1083     Non-replicated: Points which exist on the fault, but are not replicated
1084     Ghost:          These are shared fault faces which are not owned by this process. These do not produce hybrid cells and do not replicate
1085     Hybrid:         Entirely new points, such as cohesive cells
1086 
1087   When creating subsequent cohesive cells, we shift the old hybrid cells to the end of the numbering at
1088   each depth so that the new split/hybrid points can be inserted as a block.
1089 */
1090 static PetscErrorCode DMPlexConstructCohesiveCells_Internal(DM dm, DMLabel label, DMLabel splitLabel, DM sdm) {
1091   MPI_Comm         comm;
1092   IS               valueIS;
1093   PetscInt         numSP = 0; /* The number of depths for which we have replicated points */
1094   const PetscInt  *values;    /* List of depths for which we have replicated points */
1095   IS              *splitIS;
1096   IS              *unsplitIS;
1097   IS               ghostIS;
1098   PetscInt        *numSplitPoints;     /* The number of replicated points at each depth */
1099   PetscInt        *numUnsplitPoints;   /* The number of non-replicated points at each depth which still give rise to hybrid points */
1100   PetscInt        *numHybridPoints;    /* The number of new hybrid points at each depth */
1101   PetscInt        *numHybridPointsOld; /* The number of existing hybrid points at each depth */
1102   PetscInt         numGhostPoints;     /* The number of unowned, shared fault faces */
1103   const PetscInt **splitPoints;        /* Replicated points for each depth */
1104   const PetscInt **unsplitPoints;      /* Non-replicated points for each depth */
1105   const PetscInt  *ghostPoints;        /* Ghost fault faces */
1106   PetscSection     coordSection;
1107   Vec              coordinates;
1108   PetscScalar     *coords;
1109   PetscInt        *depthMax;   /* The first hybrid point at each depth in the original mesh */
1110   PetscInt        *depthEnd;   /* The point limit at each depth in the original mesh */
1111   PetscInt        *depthShift; /* Number of replicated+hybrid points at each depth */
1112   PetscInt        *pMaxNew;    /* The first replicated point at each depth in the new mesh, hybrids come after this */
1113   PetscInt        *coneNew, *coneONew, *supportNew;
1114   PetscInt         shift = 100, shift2 = 200, depth = 0, dep, dim, d, sp, maxConeSize, maxSupportSize, maxConeSizeNew, maxSupportSizeNew, numLabels, vStart, vEnd, pEnd, p, v;
1115 
1116   PetscFunctionBegin;
1117   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
1118   PetscCall(DMGetDimension(dm, &dim));
1119   PetscCall(DMPlexGetDepth(dm, &depth));
1120   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
1121   /* We do not want this label automatically computed, instead we compute it here */
1122   PetscCall(DMCreateLabel(sdm, "celltype"));
1123   /* Count split points and add cohesive cells */
1124   PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize));
1125   PetscCall(PetscMalloc5(depth + 1, &depthMax, depth + 1, &depthEnd, 2 * (depth + 1), &depthShift, depth + 1, &pMaxNew, depth + 1, &numHybridPointsOld));
1126   PetscCall(PetscMalloc7(depth + 1, &splitIS, depth + 1, &unsplitIS, depth + 1, &numSplitPoints, depth + 1, &numUnsplitPoints, depth + 1, &numHybridPoints, depth + 1, &splitPoints, depth + 1, &unsplitPoints));
1127   for (d = 0; d <= depth; ++d) {
1128     PetscCall(DMPlexGetDepthStratum(dm, d, NULL, &pMaxNew[d]));
1129     PetscCall(DMPlexGetTensorPrismBounds_Internal(dm, d, &depthMax[d], NULL));
1130     depthEnd[d]           = pMaxNew[d];
1131     depthMax[d]           = depthMax[d] < 0 ? depthEnd[d] : depthMax[d];
1132     numSplitPoints[d]     = 0;
1133     numUnsplitPoints[d]   = 0;
1134     numHybridPoints[d]    = 0;
1135     numHybridPointsOld[d] = depthMax[d] < 0 ? 0 : depthEnd[d] - depthMax[d];
1136     splitPoints[d]        = NULL;
1137     unsplitPoints[d]      = NULL;
1138     splitIS[d]            = NULL;
1139     unsplitIS[d]          = NULL;
1140     /* we are shifting the existing hybrid points with the stratum behind them, so
1141      * the split comes at the end of the normal points, i.e., at depthMax[d] */
1142     depthShift[2 * d]     = depthMax[d];
1143     depthShift[2 * d + 1] = 0;
1144   }
1145   numGhostPoints = 0;
1146   ghostPoints    = NULL;
1147   if (label) {
1148     PetscCall(DMLabelGetValueIS(label, &valueIS));
1149     PetscCall(ISGetLocalSize(valueIS, &numSP));
1150     PetscCall(ISGetIndices(valueIS, &values));
1151   }
1152   for (sp = 0; sp < numSP; ++sp) {
1153     const PetscInt dep = values[sp];
1154 
1155     if ((dep < 0) || (dep > depth)) continue;
1156     PetscCall(DMLabelGetStratumIS(label, dep, &splitIS[dep]));
1157     if (splitIS[dep]) {
1158       PetscCall(ISGetLocalSize(splitIS[dep], &numSplitPoints[dep]));
1159       PetscCall(ISGetIndices(splitIS[dep], &splitPoints[dep]));
1160     }
1161     PetscCall(DMLabelGetStratumIS(label, shift2 + dep, &unsplitIS[dep]));
1162     if (unsplitIS[dep]) {
1163       PetscCall(ISGetLocalSize(unsplitIS[dep], &numUnsplitPoints[dep]));
1164       PetscCall(ISGetIndices(unsplitIS[dep], &unsplitPoints[dep]));
1165     }
1166   }
1167   PetscCall(DMLabelGetStratumIS(label, shift2 + dim - 1, &ghostIS));
1168   if (ghostIS) {
1169     PetscCall(ISGetLocalSize(ghostIS, &numGhostPoints));
1170     PetscCall(ISGetIndices(ghostIS, &ghostPoints));
1171   }
1172   /* Calculate number of hybrid points */
1173   for (d = 1; d <= depth; ++d) numHybridPoints[d] = numSplitPoints[d - 1] + numUnsplitPoints[d - 1]; /* There is a hybrid cell/face/edge for every split face/edge/vertex   */
1174   for (d = 0; d <= depth; ++d) depthShift[2 * d + 1] = numSplitPoints[d] + numHybridPoints[d];
1175   PetscCall(DMPlexShiftPointSetUp_Internal(depth, depthShift));
1176   /* the end of the points in this stratum that come before the new points:
1177    * shifting pMaxNew[d] gets the new start of the next stratum, then count back the old hybrid points and the newly
1178    * added points */
1179   for (d = 0; d <= depth; ++d) pMaxNew[d] = DMPlexShiftPoint_Internal(pMaxNew[d], depth, depthShift) - (numHybridPointsOld[d] + numSplitPoints[d] + numHybridPoints[d]);
1180   PetscCall(DMPlexShiftSizes_Internal(dm, depthShift, sdm));
1181   /* Step 3: Set cone/support sizes for new points */
1182   for (dep = 0; dep <= depth; ++dep) {
1183     for (p = 0; p < numSplitPoints[dep]; ++p) {
1184       const PetscInt  oldp   = splitPoints[dep][p];
1185       const PetscInt  newp   = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*oldp + depthOffset[dep]*/;
1186       const PetscInt  splitp = p + pMaxNew[dep];
1187       const PetscInt *support;
1188       DMPolytopeType  ct;
1189       PetscInt        coneSize, supportSize, qf, qn, qp, e;
1190 
1191       PetscCall(DMPlexGetConeSize(dm, oldp, &coneSize));
1192       PetscCall(DMPlexSetConeSize(sdm, splitp, coneSize));
1193       PetscCall(DMPlexGetSupportSize(dm, oldp, &supportSize));
1194       PetscCall(DMPlexSetSupportSize(sdm, splitp, supportSize));
1195       PetscCall(DMPlexGetCellType(dm, oldp, &ct));
1196       PetscCall(DMPlexSetCellType(sdm, splitp, ct));
1197       if (dep == depth - 1) {
1198         const PetscInt hybcell = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1];
1199 
1200         /* Add cohesive cells, they are prisms */
1201         PetscCall(DMPlexSetConeSize(sdm, hybcell, 2 + coneSize));
1202         switch (coneSize) {
1203         case 2: PetscCall(DMPlexSetCellType(sdm, hybcell, DM_POLYTOPE_SEG_PRISM_TENSOR)); break;
1204         case 3: PetscCall(DMPlexSetCellType(sdm, hybcell, DM_POLYTOPE_TRI_PRISM_TENSOR)); break;
1205         case 4: PetscCall(DMPlexSetCellType(sdm, hybcell, DM_POLYTOPE_QUAD_PRISM_TENSOR)); break;
1206         }
1207         /* Shared fault faces with only one support cell now have two with the cohesive cell */
1208         /*   TODO Check thaat oldp has rootdegree == 1 */
1209         if (supportSize == 1) {
1210           const PetscInt *support;
1211           PetscInt        val;
1212 
1213           PetscCall(DMPlexGetSupport(dm, oldp, &support));
1214           PetscCall(DMLabelGetValue(label, support[0], &val));
1215           if (val < 0) PetscCall(DMPlexSetSupportSize(sdm, splitp, 2));
1216           else PetscCall(DMPlexSetSupportSize(sdm, newp, 2));
1217         }
1218       } else if (dep == 0) {
1219         const PetscInt hybedge = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1];
1220 
1221         PetscCall(DMPlexGetSupport(dm, oldp, &support));
1222         for (e = 0, qn = 0, qp = 0, qf = 0; e < supportSize; ++e) {
1223           PetscInt val;
1224 
1225           PetscCall(DMLabelGetValue(label, support[e], &val));
1226           if (val == 1) ++qf;
1227           if ((val == 1) || (val == (shift + 1))) ++qn;
1228           if ((val == 1) || (val == -(shift + 1))) ++qp;
1229         }
1230         /* Split old vertex: Edges into original vertex and new cohesive edge */
1231         PetscCall(DMPlexSetSupportSize(sdm, newp, qn + 1));
1232         /* Split new vertex: Edges into split vertex and new cohesive edge */
1233         PetscCall(DMPlexSetSupportSize(sdm, splitp, qp + 1));
1234         /* Add hybrid edge */
1235         PetscCall(DMPlexSetConeSize(sdm, hybedge, 2));
1236         PetscCall(DMPlexSetSupportSize(sdm, hybedge, qf));
1237         PetscCall(DMPlexSetCellType(sdm, hybedge, DM_POLYTOPE_POINT_PRISM_TENSOR));
1238       } else if (dep == dim - 2) {
1239         const PetscInt hybface = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1];
1240 
1241         PetscCall(DMPlexGetSupport(dm, oldp, &support));
1242         for (e = 0, qn = 0, qp = 0, qf = 0; e < supportSize; ++e) {
1243           PetscInt val;
1244 
1245           PetscCall(DMLabelGetValue(label, support[e], &val));
1246           if (val == dim - 1) ++qf;
1247           if ((val == dim - 1) || (val == (shift + dim - 1))) ++qn;
1248           if ((val == dim - 1) || (val == -(shift + dim - 1))) ++qp;
1249         }
1250         /* Split old edge: Faces into original edge and cohesive face (positive side?) */
1251         PetscCall(DMPlexSetSupportSize(sdm, newp, qn + 1));
1252         /* Split new edge: Faces into split edge and cohesive face (negative side?) */
1253         PetscCall(DMPlexSetSupportSize(sdm, splitp, qp + 1));
1254         /* Add hybrid face */
1255         PetscCall(DMPlexSetConeSize(sdm, hybface, 4));
1256         PetscCall(DMPlexSetSupportSize(sdm, hybface, qf));
1257         PetscCall(DMPlexSetCellType(sdm, hybface, DM_POLYTOPE_SEG_PRISM_TENSOR));
1258       }
1259     }
1260   }
1261   for (dep = 0; dep <= depth; ++dep) {
1262     for (p = 0; p < numUnsplitPoints[dep]; ++p) {
1263       const PetscInt  oldp = unsplitPoints[dep][p];
1264       const PetscInt  newp = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*oldp + depthOffset[dep]*/;
1265       const PetscInt *support;
1266       PetscInt        coneSize, supportSize, qf, e, s;
1267 
1268       PetscCall(DMPlexGetConeSize(dm, oldp, &coneSize));
1269       PetscCall(DMPlexGetSupportSize(dm, oldp, &supportSize));
1270       PetscCall(DMPlexGetSupport(dm, oldp, &support));
1271       if (dep == 0) {
1272         const PetscInt hybedge = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1] + numSplitPoints[dep];
1273 
1274         /* Unsplit vertex: Edges into original vertex, split edges, and new cohesive edge twice */
1275         for (s = 0, qf = 0; s < supportSize; ++s, ++qf) {
1276           PetscCall(PetscFindInt(support[s], numSplitPoints[dep + 1], splitPoints[dep + 1], &e));
1277           if (e >= 0) ++qf;
1278         }
1279         PetscCall(DMPlexSetSupportSize(sdm, newp, qf + 2));
1280         /* Add hybrid edge */
1281         PetscCall(DMPlexSetConeSize(sdm, hybedge, 2));
1282         for (e = 0, qf = 0; e < supportSize; ++e) {
1283           PetscInt val;
1284 
1285           PetscCall(DMLabelGetValue(label, support[e], &val));
1286           /* Split and unsplit edges produce hybrid faces */
1287           if (val == 1) ++qf;
1288           if (val == (shift2 + 1)) ++qf;
1289         }
1290         PetscCall(DMPlexSetSupportSize(sdm, hybedge, qf));
1291         PetscCall(DMPlexSetCellType(sdm, hybedge, DM_POLYTOPE_POINT_PRISM_TENSOR));
1292       } else if (dep == dim - 2) {
1293         const PetscInt hybface = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1] + numSplitPoints[dep];
1294         PetscInt       val;
1295 
1296         for (e = 0, qf = 0; e < supportSize; ++e) {
1297           PetscCall(DMLabelGetValue(label, support[e], &val));
1298           if (val == dim - 1) qf += 2;
1299           else ++qf;
1300         }
1301         /* Unsplit edge: Faces into original edge, split face, and cohesive face twice */
1302         PetscCall(DMPlexSetSupportSize(sdm, newp, qf + 2));
1303         /* Add hybrid face */
1304         for (e = 0, qf = 0; e < supportSize; ++e) {
1305           PetscCall(DMLabelGetValue(label, support[e], &val));
1306           if (val == dim - 1) ++qf;
1307         }
1308         PetscCall(DMPlexSetConeSize(sdm, hybface, 4));
1309         PetscCall(DMPlexSetSupportSize(sdm, hybface, qf));
1310         PetscCall(DMPlexSetCellType(sdm, hybface, DM_POLYTOPE_SEG_PRISM_TENSOR));
1311       }
1312     }
1313   }
1314   /* Step 4: Setup split DM */
1315   PetscCall(DMSetUp(sdm));
1316   PetscCall(DMPlexShiftPoints_Internal(dm, depthShift, sdm));
1317   PetscCall(DMPlexGetMaxSizes(sdm, &maxConeSizeNew, &maxSupportSizeNew));
1318   PetscCall(PetscMalloc3(PetscMax(maxConeSize, maxConeSizeNew) * 3, &coneNew, PetscMax(maxConeSize, maxConeSizeNew) * 3, &coneONew, PetscMax(maxSupportSize, maxSupportSizeNew), &supportNew));
1319   /* Step 6: Set cones and supports for new points */
1320   for (dep = 0; dep <= depth; ++dep) {
1321     for (p = 0; p < numSplitPoints[dep]; ++p) {
1322       const PetscInt  oldp   = splitPoints[dep][p];
1323       const PetscInt  newp   = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*oldp + depthOffset[dep]*/;
1324       const PetscInt  splitp = p + pMaxNew[dep];
1325       const PetscInt *cone, *support, *ornt;
1326       DMPolytopeType  ct;
1327       PetscInt        coneSize, supportSize, q, qf, qn, qp, v, e, s;
1328 
1329       PetscCall(DMPlexGetCellType(dm, oldp, &ct));
1330       PetscCall(DMPlexGetConeSize(dm, oldp, &coneSize));
1331       PetscCall(DMPlexGetCone(dm, oldp, &cone));
1332       PetscCall(DMPlexGetConeOrientation(dm, oldp, &ornt));
1333       PetscCall(DMPlexGetSupportSize(dm, oldp, &supportSize));
1334       PetscCall(DMPlexGetSupport(dm, oldp, &support));
1335       if (dep == depth - 1) {
1336         PetscBool       hasUnsplit = PETSC_FALSE;
1337         const PetscInt  hybcell    = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1];
1338         const PetscInt *supportF;
1339 
1340         coneONew[0] = coneONew[1] = -1000;
1341         /* Split face:       copy in old face to new face to start */
1342         PetscCall(DMPlexGetSupport(sdm, newp, &supportF));
1343         PetscCall(DMPlexSetSupport(sdm, splitp, supportF));
1344         /* Split old face:   old vertices/edges in cone so no change */
1345         /* Split new face:   new vertices/edges in cone */
1346         for (q = 0; q < coneSize; ++q) {
1347           PetscCall(PetscFindInt(cone[q], numSplitPoints[dep - 1], splitPoints[dep - 1], &v));
1348           if (v < 0) {
1349             PetscCall(PetscFindInt(cone[q], numUnsplitPoints[dep - 1], unsplitPoints[dep - 1], &v));
1350             PetscCheck(v >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %" PetscInt_FMT " in split or unsplit points of depth %" PetscInt_FMT, cone[q], dep - 1);
1351             coneNew[2 + q] = DMPlexShiftPoint_Internal(cone[q], depth, depthShift) /*cone[q] + depthOffset[dep-1]*/;
1352             hasUnsplit     = PETSC_TRUE;
1353           } else {
1354             coneNew[2 + q] = v + pMaxNew[dep - 1];
1355             if (dep > 1) {
1356               const PetscInt *econe;
1357               PetscInt        econeSize, r, vs, vu;
1358 
1359               PetscCall(DMPlexGetConeSize(dm, cone[q], &econeSize));
1360               PetscCall(DMPlexGetCone(dm, cone[q], &econe));
1361               for (r = 0; r < econeSize; ++r) {
1362                 PetscCall(PetscFindInt(econe[r], numSplitPoints[dep - 2], splitPoints[dep - 2], &vs));
1363                 PetscCall(PetscFindInt(econe[r], numUnsplitPoints[dep - 2], unsplitPoints[dep - 2], &vu));
1364                 if (vs >= 0) continue;
1365                 PetscCheck(vu >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %" PetscInt_FMT " in split or unsplit points of depth %" PetscInt_FMT, econe[r], dep - 2);
1366                 hasUnsplit = PETSC_TRUE;
1367               }
1368             }
1369           }
1370         }
1371         PetscCall(DMPlexSetCone(sdm, splitp, &coneNew[2]));
1372         PetscCall(DMPlexSetConeOrientation(sdm, splitp, ornt));
1373         /* Face support */
1374         PetscInt vals[2];
1375 
1376         PetscCall(DMLabelGetValue(label, support[0], &vals[0]));
1377         if (supportSize > 1) PetscCall(DMLabelGetValue(label, support[1], &vals[1]));
1378         else vals[1] = -vals[0];
1379         PetscCheck(vals[0] * vals[1] < 0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid support labels %" PetscInt_FMT " %" PetscInt_FMT, vals[0], vals[1]);
1380 
1381         for (s = 0; s < 2; ++s) {
1382           if (s >= supportSize) {
1383             if (vals[s] < 0) {
1384               /* Ghost old face:   Replace negative side cell with cohesive cell */
1385               PetscCall(DMPlexInsertSupport(sdm, newp, s, hybcell));
1386             } else {
1387               /* Ghost new face:   Replace positive side cell with cohesive cell */
1388               PetscCall(DMPlexInsertSupport(sdm, splitp, s, hybcell));
1389             }
1390           } else {
1391             if (vals[s] < 0) {
1392               /* Split old face:   Replace negative side cell with cohesive cell */
1393               PetscCall(DMPlexInsertSupport(sdm, newp, s, hybcell));
1394             } else {
1395               /* Split new face:   Replace positive side cell with cohesive cell */
1396               PetscCall(DMPlexInsertSupport(sdm, splitp, s, hybcell));
1397             }
1398           }
1399         }
1400         /* Get orientation for cohesive face using the positive side cell */
1401         {
1402           const PetscInt *ncone, *nconeO;
1403           PetscInt        nconeSize, nc, ocell;
1404           PetscBool       flip = PETSC_FALSE;
1405 
1406           if (supportSize > 1) {
1407             ocell = vals[0] < 0 ? support[1] : support[0];
1408           } else {
1409             ocell = support[0];
1410             flip  = vals[0] < 0 ? PETSC_TRUE : PETSC_FALSE;
1411           }
1412           PetscCall(DMPlexGetConeSize(dm, ocell, &nconeSize));
1413           PetscCall(DMPlexGetCone(dm, ocell, &ncone));
1414           PetscCall(DMPlexGetConeOrientation(dm, ocell, &nconeO));
1415           for (nc = 0; nc < nconeSize; ++nc) {
1416             if (ncone[nc] == oldp) {
1417               coneONew[0] = flip ? -(nconeO[nc] + 1) : nconeO[nc];
1418               break;
1419             }
1420           }
1421           PetscCheck(nc < nconeSize, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate face %" PetscInt_FMT " in neighboring cell %" PetscInt_FMT, oldp, ocell);
1422         }
1423         /* Cohesive cell:    Old and new split face, then new cohesive faces */
1424         {
1425           const PetscInt No = DMPolytopeTypeGetNumArrangments(ct) / 2;
1426           PetscCheck((coneONew[0] >= -No) && (coneONew[0] < No), PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid %s orientation %" PetscInt_FMT, DMPolytopeTypes[ct], coneONew[0]);
1427         }
1428         const PetscInt *arr = DMPolytopeTypeGetArrangment(ct, coneONew[0]);
1429 
1430         coneNew[0]  = newp; /* Extracted negative side orientation above */
1431         coneNew[1]  = splitp;
1432         coneONew[1] = coneONew[0];
1433         for (q = 0; q < coneSize; ++q) {
1434           /* Hybrid faces must follow order from oriented end face */
1435           const PetscInt qa = arr[q * 2 + 0];
1436           const PetscInt qo = arr[q * 2 + 1];
1437           DMPolytopeType ft = dep == 2 ? DM_POLYTOPE_SEGMENT : DM_POLYTOPE_POINT;
1438 
1439           PetscCall(PetscFindInt(cone[qa], numSplitPoints[dep - 1], splitPoints[dep - 1], &v));
1440           if (v < 0) {
1441             PetscCall(PetscFindInt(cone[qa], numUnsplitPoints[dep - 1], unsplitPoints[dep - 1], &v));
1442             coneNew[2 + q] = v + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep - 1];
1443           } else {
1444             coneNew[2 + q] = v + pMaxNew[dep] + numSplitPoints[dep];
1445           }
1446           coneONew[2 + q] = DMPolytopeTypeComposeOrientation(ft, qo, ornt[qa]);
1447         }
1448         PetscCall(DMPlexSetCone(sdm, hybcell, coneNew));
1449         PetscCall(DMPlexSetConeOrientation(sdm, hybcell, coneONew));
1450         /* Label the hybrid cells on the boundary of the split */
1451         if (hasUnsplit) PetscCall(DMLabelSetValue(label, -hybcell, dim));
1452       } else if (dep == 0) {
1453         const PetscInt hybedge = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1];
1454 
1455         /* Split old vertex: Edges in old split faces and new cohesive edge */
1456         for (e = 0, qn = 0; e < supportSize; ++e) {
1457           PetscInt val;
1458 
1459           PetscCall(DMLabelGetValue(label, support[e], &val));
1460           if ((val == 1) || (val == (shift + 1))) supportNew[qn++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1461         }
1462         supportNew[qn] = hybedge;
1463         PetscCall(DMPlexSetSupport(sdm, newp, supportNew));
1464         /* Split new vertex: Edges in new split faces and new cohesive edge */
1465         for (e = 0, qp = 0; e < supportSize; ++e) {
1466           PetscInt val, edge;
1467 
1468           PetscCall(DMLabelGetValue(label, support[e], &val));
1469           if (val == 1) {
1470             PetscCall(PetscFindInt(support[e], numSplitPoints[dep + 1], splitPoints[dep + 1], &edge));
1471             PetscCheck(edge >= 0, comm, PETSC_ERR_ARG_WRONG, "Edge %" PetscInt_FMT " is not a split edge", support[e]);
1472             supportNew[qp++] = edge + pMaxNew[dep + 1];
1473           } else if (val == -(shift + 1)) {
1474             supportNew[qp++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1475           }
1476         }
1477         supportNew[qp] = hybedge;
1478         PetscCall(DMPlexSetSupport(sdm, splitp, supportNew));
1479         /* Hybrid edge:    Old and new split vertex */
1480         coneNew[0] = newp;
1481         coneNew[1] = splitp;
1482         PetscCall(DMPlexSetCone(sdm, hybedge, coneNew));
1483         for (e = 0, qf = 0; e < supportSize; ++e) {
1484           PetscInt val, edge;
1485 
1486           PetscCall(DMLabelGetValue(label, support[e], &val));
1487           if (val == 1) {
1488             PetscCall(PetscFindInt(support[e], numSplitPoints[dep + 1], splitPoints[dep + 1], &edge));
1489             PetscCheck(edge >= 0, comm, PETSC_ERR_ARG_WRONG, "Edge %" PetscInt_FMT " is not a split edge", support[e]);
1490             supportNew[qf++] = edge + pMaxNew[dep + 2] + numSplitPoints[dep + 2];
1491           }
1492         }
1493         PetscCall(DMPlexSetSupport(sdm, hybedge, supportNew));
1494       } else if (dep == dim - 2) {
1495         const PetscInt hybface = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1];
1496 
1497         /* Split old edge:   old vertices in cone so no change */
1498         /* Split new edge:   new vertices in cone */
1499         for (q = 0; q < coneSize; ++q) {
1500           PetscCall(PetscFindInt(cone[q], numSplitPoints[dep - 1], splitPoints[dep - 1], &v));
1501           if (v < 0) {
1502             PetscCall(PetscFindInt(cone[q], numUnsplitPoints[dep - 1], unsplitPoints[dep - 1], &v));
1503             PetscCheck(v >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %" PetscInt_FMT " in split or unsplit points of depth %" PetscInt_FMT, cone[q], dep - 1);
1504             coneNew[q] = DMPlexShiftPoint_Internal(cone[q], depth, depthShift) /*cone[q] + depthOffset[dep-1]*/;
1505           } else {
1506             coneNew[q] = v + pMaxNew[dep - 1];
1507           }
1508         }
1509         PetscCall(DMPlexSetCone(sdm, splitp, coneNew));
1510         /* Split old edge: Faces in positive side cells and old split faces */
1511         for (e = 0, q = 0; e < supportSize; ++e) {
1512           PetscInt val;
1513 
1514           PetscCall(DMLabelGetValue(label, support[e], &val));
1515           if (val == dim - 1) {
1516             supportNew[q++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1517           } else if (val == (shift + dim - 1)) {
1518             supportNew[q++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1519           }
1520         }
1521         supportNew[q++] = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1];
1522         PetscCall(DMPlexSetSupport(sdm, newp, supportNew));
1523         /* Split new edge: Faces in negative side cells and new split faces */
1524         for (e = 0, q = 0; e < supportSize; ++e) {
1525           PetscInt val, face;
1526 
1527           PetscCall(DMLabelGetValue(label, support[e], &val));
1528           if (val == dim - 1) {
1529             PetscCall(PetscFindInt(support[e], numSplitPoints[dep + 1], splitPoints[dep + 1], &face));
1530             PetscCheck(face >= 0, comm, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " is not a split face", support[e]);
1531             supportNew[q++] = face + pMaxNew[dep + 1];
1532           } else if (val == -(shift + dim - 1)) {
1533             supportNew[q++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1534           }
1535         }
1536         supportNew[q++] = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1];
1537         PetscCall(DMPlexSetSupport(sdm, splitp, supportNew));
1538         /* Hybrid face */
1539         coneNew[0] = newp;
1540         coneNew[1] = splitp;
1541         for (v = 0; v < coneSize; ++v) {
1542           PetscInt vertex;
1543           PetscCall(PetscFindInt(cone[v], numSplitPoints[dep - 1], splitPoints[dep - 1], &vertex));
1544           if (vertex < 0) {
1545             PetscCall(PetscFindInt(cone[v], numUnsplitPoints[dep - 1], unsplitPoints[dep - 1], &vertex));
1546             PetscCheck(vertex >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %" PetscInt_FMT " in split or unsplit points of depth %" PetscInt_FMT, cone[v], dep - 1);
1547             coneNew[2 + v] = vertex + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep - 1];
1548           } else {
1549             coneNew[2 + v] = vertex + pMaxNew[dep] + numSplitPoints[dep];
1550           }
1551         }
1552         PetscCall(DMPlexSetCone(sdm, hybface, coneNew));
1553         for (e = 0, qf = 0; e < supportSize; ++e) {
1554           PetscInt val, face;
1555 
1556           PetscCall(DMLabelGetValue(label, support[e], &val));
1557           if (val == dim - 1) {
1558             PetscCall(PetscFindInt(support[e], numSplitPoints[dep + 1], splitPoints[dep + 1], &face));
1559             PetscCheck(face >= 0, comm, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " is not a split face", support[e]);
1560             supportNew[qf++] = face + pMaxNew[dep + 2] + numSplitPoints[dep + 2];
1561           }
1562         }
1563         PetscCall(DMPlexSetSupport(sdm, hybface, supportNew));
1564       }
1565     }
1566   }
1567   for (dep = 0; dep <= depth; ++dep) {
1568     for (p = 0; p < numUnsplitPoints[dep]; ++p) {
1569       const PetscInt  oldp = unsplitPoints[dep][p];
1570       const PetscInt  newp = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*oldp + depthOffset[dep]*/;
1571       const PetscInt *cone, *support;
1572       PetscInt        coneSize, supportSize, supportSizeNew, q, qf, e, f, s;
1573 
1574       PetscCall(DMPlexGetConeSize(dm, oldp, &coneSize));
1575       PetscCall(DMPlexGetCone(dm, oldp, &cone));
1576       PetscCall(DMPlexGetSupportSize(dm, oldp, &supportSize));
1577       PetscCall(DMPlexGetSupport(dm, oldp, &support));
1578       if (dep == 0) {
1579         const PetscInt hybedge = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1] + numSplitPoints[dep];
1580 
1581         /* Unsplit vertex */
1582         PetscCall(DMPlexGetSupportSize(sdm, newp, &supportSizeNew));
1583         for (s = 0, q = 0; s < supportSize; ++s) {
1584           supportNew[q++] = DMPlexShiftPoint_Internal(support[s], depth, depthShift) /*support[s] + depthOffset[dep+1]*/;
1585           PetscCall(PetscFindInt(support[s], numSplitPoints[dep + 1], splitPoints[dep + 1], &e));
1586           if (e >= 0) supportNew[q++] = e + pMaxNew[dep + 1];
1587         }
1588         supportNew[q++] = hybedge;
1589         supportNew[q++] = hybedge;
1590         PetscCheck(q == supportSizeNew, comm, PETSC_ERR_ARG_WRONG, "Support size %" PetscInt_FMT " != %" PetscInt_FMT " for vertex %" PetscInt_FMT, q, supportSizeNew, newp);
1591         PetscCall(DMPlexSetSupport(sdm, newp, supportNew));
1592         /* Hybrid edge */
1593         coneNew[0] = newp;
1594         coneNew[1] = newp;
1595         PetscCall(DMPlexSetCone(sdm, hybedge, coneNew));
1596         for (e = 0, qf = 0; e < supportSize; ++e) {
1597           PetscInt val, edge;
1598 
1599           PetscCall(DMLabelGetValue(label, support[e], &val));
1600           if (val == 1) {
1601             PetscCall(PetscFindInt(support[e], numSplitPoints[dep + 1], splitPoints[dep + 1], &edge));
1602             PetscCheck(edge >= 0, comm, PETSC_ERR_ARG_WRONG, "Edge %" PetscInt_FMT " is not a split edge", support[e]);
1603             supportNew[qf++] = edge + pMaxNew[dep + 2] + numSplitPoints[dep + 2];
1604           } else if (val == (shift2 + 1)) {
1605             PetscCall(PetscFindInt(support[e], numUnsplitPoints[dep + 1], unsplitPoints[dep + 1], &edge));
1606             PetscCheck(edge >= 0, comm, PETSC_ERR_ARG_WRONG, "Edge %" PetscInt_FMT " is not a unsplit edge", support[e]);
1607             supportNew[qf++] = edge + pMaxNew[dep + 2] + numSplitPoints[dep + 2] + numSplitPoints[dep + 1];
1608           }
1609         }
1610         PetscCall(DMPlexSetSupport(sdm, hybedge, supportNew));
1611       } else if (dep == dim - 2) {
1612         const PetscInt hybface = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1] + numSplitPoints[dep];
1613 
1614         /* Unsplit edge: Faces into original edge, split face, and hybrid face twice */
1615         for (f = 0, qf = 0; f < supportSize; ++f) {
1616           PetscInt val, face;
1617 
1618           PetscCall(DMLabelGetValue(label, support[f], &val));
1619           if (val == dim - 1) {
1620             PetscCall(PetscFindInt(support[f], numSplitPoints[dep + 1], splitPoints[dep + 1], &face));
1621             PetscCheck(face >= 0, comm, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " is not a split face", support[f]);
1622             supportNew[qf++] = DMPlexShiftPoint_Internal(support[f], depth, depthShift) /*support[f] + depthOffset[dep+1]*/;
1623             supportNew[qf++] = face + pMaxNew[dep + 1];
1624           } else {
1625             supportNew[qf++] = DMPlexShiftPoint_Internal(support[f], depth, depthShift) /*support[f] + depthOffset[dep+1]*/;
1626           }
1627         }
1628         supportNew[qf++] = hybface;
1629         supportNew[qf++] = hybface;
1630         PetscCall(DMPlexGetSupportSize(sdm, newp, &supportSizeNew));
1631         PetscCheck(qf == supportSizeNew, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Support size for unsplit edge %" PetscInt_FMT " is %" PetscInt_FMT " != %" PetscInt_FMT, newp, qf, supportSizeNew);
1632         PetscCall(DMPlexSetSupport(sdm, newp, supportNew));
1633         /* Add hybrid face */
1634         coneNew[0] = newp;
1635         coneNew[1] = newp;
1636         PetscCall(PetscFindInt(cone[0], numUnsplitPoints[dep - 1], unsplitPoints[dep - 1], &v));
1637         PetscCheck(v >= 0, comm, PETSC_ERR_ARG_WRONG, "Vertex %" PetscInt_FMT " is not an unsplit vertex", cone[0]);
1638         coneNew[2] = v + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep - 1];
1639         PetscCall(PetscFindInt(cone[1], numUnsplitPoints[dep - 1], unsplitPoints[dep - 1], &v));
1640         PetscCheck(v >= 0, comm, PETSC_ERR_ARG_WRONG, "Vertex %" PetscInt_FMT " is not an unsplit vertex", cone[1]);
1641         coneNew[3] = v + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep - 1];
1642         PetscCall(DMPlexSetCone(sdm, hybface, coneNew));
1643         for (f = 0, qf = 0; f < supportSize; ++f) {
1644           PetscInt val, face;
1645 
1646           PetscCall(DMLabelGetValue(label, support[f], &val));
1647           if (val == dim - 1) {
1648             PetscCall(PetscFindInt(support[f], numSplitPoints[dep + 1], splitPoints[dep + 1], &face));
1649             supportNew[qf++] = face + pMaxNew[dep + 2] + numSplitPoints[dep + 2];
1650           }
1651         }
1652         PetscCall(DMPlexGetSupportSize(sdm, hybface, &supportSizeNew));
1653         PetscCheck(qf == supportSizeNew, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Support size for hybrid face %" PetscInt_FMT " is %" PetscInt_FMT " != %" PetscInt_FMT, hybface, qf, supportSizeNew);
1654         PetscCall(DMPlexSetSupport(sdm, hybface, supportNew));
1655       }
1656     }
1657   }
1658   /* Step 6b: Replace split points in negative side cones */
1659   for (sp = 0; sp < numSP; ++sp) {
1660     PetscInt        dep = values[sp];
1661     IS              pIS;
1662     PetscInt        numPoints;
1663     const PetscInt *points;
1664 
1665     if (dep >= 0) continue;
1666     PetscCall(DMLabelGetStratumIS(label, dep, &pIS));
1667     if (!pIS) continue;
1668     dep = -dep - shift;
1669     PetscCall(ISGetLocalSize(pIS, &numPoints));
1670     PetscCall(ISGetIndices(pIS, &points));
1671     for (p = 0; p < numPoints; ++p) {
1672       const PetscInt  oldp = points[p];
1673       const PetscInt  newp = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*depthOffset[dep] + oldp*/;
1674       const PetscInt *cone;
1675       PetscInt        coneSize, c;
1676       /* PetscBool       replaced = PETSC_FALSE; */
1677 
1678       /* Negative edge: replace split vertex */
1679       /* Negative cell: replace split face */
1680       PetscCall(DMPlexGetConeSize(sdm, newp, &coneSize));
1681       PetscCall(DMPlexGetCone(sdm, newp, &cone));
1682       for (c = 0; c < coneSize; ++c) {
1683         const PetscInt coldp = DMPlexShiftPointInverse_Internal(cone[c], depth, depthShift);
1684         PetscInt       csplitp, cp, val;
1685 
1686         PetscCall(DMLabelGetValue(label, coldp, &val));
1687         if (val == dep - 1) {
1688           PetscCall(PetscFindInt(coldp, numSplitPoints[dep - 1], splitPoints[dep - 1], &cp));
1689           PetscCheck(cp >= 0, comm, PETSC_ERR_ARG_WRONG, "Point %" PetscInt_FMT " is not a split point of dimension %" PetscInt_FMT, oldp, dep - 1);
1690           csplitp = pMaxNew[dep - 1] + cp;
1691           PetscCall(DMPlexInsertCone(sdm, newp, c, csplitp));
1692           /* replaced = PETSC_TRUE; */
1693         }
1694       }
1695       /* Cells with only a vertex or edge on the submesh have no replacement */
1696       /* PetscCheck(replaced,comm, PETSC_ERR_ARG_WRONG, "The cone of point %d does not contain split points", oldp); */
1697     }
1698     PetscCall(ISRestoreIndices(pIS, &points));
1699     PetscCall(ISDestroy(&pIS));
1700   }
1701   /* Step 7: Coordinates */
1702   PetscCall(DMPlexShiftCoordinates_Internal(dm, depthShift, sdm));
1703   PetscCall(DMGetCoordinateSection(sdm, &coordSection));
1704   PetscCall(DMGetCoordinatesLocal(sdm, &coordinates));
1705   PetscCall(VecGetArray(coordinates, &coords));
1706   for (v = 0; v < (numSplitPoints ? numSplitPoints[0] : 0); ++v) {
1707     const PetscInt newp   = DMPlexShiftPoint_Internal(splitPoints[0][v], depth, depthShift) /*depthOffset[0] + splitPoints[0][v]*/;
1708     const PetscInt splitp = pMaxNew[0] + v;
1709     PetscInt       dof, off, soff, d;
1710 
1711     PetscCall(PetscSectionGetDof(coordSection, newp, &dof));
1712     PetscCall(PetscSectionGetOffset(coordSection, newp, &off));
1713     PetscCall(PetscSectionGetOffset(coordSection, splitp, &soff));
1714     for (d = 0; d < dof; ++d) coords[soff + d] = coords[off + d];
1715   }
1716   PetscCall(VecRestoreArray(coordinates, &coords));
1717   /* Step 8: SF, if I can figure this out we can split the mesh in parallel */
1718   PetscCall(DMPlexShiftSF_Internal(dm, depthShift, sdm));
1719   /*   TODO We need to associate the ghost points with the correct replica */
1720   /* Step 9: Labels */
1721   PetscCall(DMPlexShiftLabels_Internal(dm, depthShift, sdm));
1722   PetscCall(DMPlexCreateVTKLabel_Internal(dm, PETSC_FALSE, sdm));
1723   PetscCall(DMGetNumLabels(sdm, &numLabels));
1724   for (dep = 0; dep <= depth; ++dep) {
1725     for (p = 0; p < numSplitPoints[dep]; ++p) {
1726       const PetscInt newp   = DMPlexShiftPoint_Internal(splitPoints[dep][p], depth, depthShift) /*depthOffset[dep] + splitPoints[dep][p]*/;
1727       const PetscInt splitp = pMaxNew[dep] + p;
1728       PetscInt       l;
1729 
1730       if (splitLabel) {
1731         const PetscInt val = 100 + dep;
1732 
1733         PetscCall(DMLabelSetValue(splitLabel, newp, val));
1734         PetscCall(DMLabelSetValue(splitLabel, splitp, -val));
1735       }
1736       for (l = 0; l < numLabels; ++l) {
1737         DMLabel     mlabel;
1738         const char *lname;
1739         PetscInt    val;
1740         PetscBool   isDepth;
1741 
1742         PetscCall(DMGetLabelName(sdm, l, &lname));
1743         PetscCall(PetscStrcmp(lname, "depth", &isDepth));
1744         if (isDepth) continue;
1745         PetscCall(DMGetLabel(sdm, lname, &mlabel));
1746         PetscCall(DMLabelGetValue(mlabel, newp, &val));
1747         if (val >= 0) PetscCall(DMLabelSetValue(mlabel, splitp, val));
1748       }
1749     }
1750   }
1751   for (sp = 0; sp < numSP; ++sp) {
1752     const PetscInt dep = values[sp];
1753 
1754     if ((dep < 0) || (dep > depth)) continue;
1755     if (splitIS[dep]) PetscCall(ISRestoreIndices(splitIS[dep], &splitPoints[dep]));
1756     PetscCall(ISDestroy(&splitIS[dep]));
1757     if (unsplitIS[dep]) PetscCall(ISRestoreIndices(unsplitIS[dep], &unsplitPoints[dep]));
1758     PetscCall(ISDestroy(&unsplitIS[dep]));
1759   }
1760   if (ghostIS) PetscCall(ISRestoreIndices(ghostIS, &ghostPoints));
1761   PetscCall(ISDestroy(&ghostIS));
1762   if (label) {
1763     PetscCall(ISRestoreIndices(valueIS, &values));
1764     PetscCall(ISDestroy(&valueIS));
1765   }
1766   for (d = 0; d <= depth; ++d) {
1767     PetscCall(DMPlexGetDepthStratum(sdm, d, NULL, &pEnd));
1768     pMaxNew[d] = pEnd - numHybridPoints[d] - numHybridPointsOld[d];
1769   }
1770   PetscCall(PetscFree3(coneNew, coneONew, supportNew));
1771   PetscCall(PetscFree5(depthMax, depthEnd, depthShift, pMaxNew, numHybridPointsOld));
1772   PetscCall(PetscFree7(splitIS, unsplitIS, numSplitPoints, numUnsplitPoints, numHybridPoints, splitPoints, unsplitPoints));
1773   PetscFunctionReturn(0);
1774 }
1775 
1776 /*@C
1777   DMPlexConstructCohesiveCells - Construct cohesive cells which split the face along an internal interface
1778 
1779   Collective on dm
1780 
1781   Input Parameters:
1782 + dm - The original DM
1783 - label - The label specifying the boundary faces (this could be auto-generated)
1784 
1785   Output Parameters:
1786 + splitLabel - The label containing the split points, or NULL if no output is desired
1787 - dmSplit - The new DM
1788 
1789   Level: developer
1790 
1791 .seealso: `DMCreate()`, `DMPlexLabelCohesiveComplete()`
1792 @*/
1793 PetscErrorCode DMPlexConstructCohesiveCells(DM dm, DMLabel label, DMLabel splitLabel, DM *dmSplit) {
1794   DM       sdm;
1795   PetscInt dim;
1796 
1797   PetscFunctionBegin;
1798   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1799   PetscValidPointer(dmSplit, 4);
1800   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &sdm));
1801   PetscCall(DMSetType(sdm, DMPLEX));
1802   PetscCall(DMGetDimension(dm, &dim));
1803   PetscCall(DMSetDimension(sdm, dim));
1804   switch (dim) {
1805   case 2:
1806   case 3: PetscCall(DMPlexConstructCohesiveCells_Internal(dm, label, splitLabel, sdm)); break;
1807   default: SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cannot construct cohesive cells for dimension %" PetscInt_FMT, dim);
1808   }
1809   *dmSplit = sdm;
1810   PetscFunctionReturn(0);
1811 }
1812 
1813 /* Returns the side of the surface for a given cell with a face on the surface */
1814 static PetscErrorCode GetSurfaceSide_Static(DM dm, DM subdm, PetscInt numSubpoints, const PetscInt *subpoints, PetscInt cell, PetscInt face, PetscBool *pos) {
1815   const PetscInt *cone, *ornt;
1816   PetscInt        dim, coneSize, c;
1817 
1818   PetscFunctionBegin;
1819   *pos = PETSC_TRUE;
1820   PetscCall(DMGetDimension(dm, &dim));
1821   PetscCall(DMPlexGetConeSize(dm, cell, &coneSize));
1822   PetscCall(DMPlexGetCone(dm, cell, &cone));
1823   PetscCall(DMPlexGetConeOrientation(dm, cell, &ornt));
1824   for (c = 0; c < coneSize; ++c) {
1825     if (cone[c] == face) {
1826       PetscInt o = ornt[c];
1827 
1828       if (subdm) {
1829         const PetscInt *subcone, *subornt;
1830         PetscInt        subpoint, subface, subconeSize, sc;
1831 
1832         PetscCall(PetscFindInt(cell, numSubpoints, subpoints, &subpoint));
1833         PetscCall(PetscFindInt(face, numSubpoints, subpoints, &subface));
1834         PetscCall(DMPlexGetConeSize(subdm, subpoint, &subconeSize));
1835         PetscCall(DMPlexGetCone(subdm, subpoint, &subcone));
1836         PetscCall(DMPlexGetConeOrientation(subdm, subpoint, &subornt));
1837         for (sc = 0; sc < subconeSize; ++sc) {
1838           if (subcone[sc] == subface) {
1839             o = subornt[0];
1840             break;
1841           }
1842         }
1843         PetscCheck(sc < subconeSize, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not find subpoint %" PetscInt_FMT " (%" PetscInt_FMT ") in cone for subpoint %" PetscInt_FMT " (%" PetscInt_FMT ")", subface, face, subpoint, cell);
1844       }
1845       if (o >= 0) *pos = PETSC_TRUE;
1846       else *pos = PETSC_FALSE;
1847       break;
1848     }
1849   }
1850   PetscCheck(c != coneSize, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Cell %" PetscInt_FMT " in split face %" PetscInt_FMT " support does not have it in the cone", cell, face);
1851   PetscFunctionReturn(0);
1852 }
1853 
1854 static PetscErrorCode CheckFaultEdge_Private(DM dm, DMLabel label) {
1855   IS              facePosIS, faceNegIS, dimIS;
1856   const PetscInt *points;
1857   PetscInt        dim, numPoints, p, shift = 100, shift2 = 200;
1858 
1859   PetscFunctionBegin;
1860   PetscCall(DMGetDimension(dm, &dim));
1861   /* If any faces touching the fault divide cells on either side, split them */
1862   PetscCall(DMLabelGetStratumIS(label, shift + dim - 1, &facePosIS));
1863   PetscCall(DMLabelGetStratumIS(label, -(shift + dim - 1), &faceNegIS));
1864   if (!facePosIS || !faceNegIS) {
1865     PetscCall(ISDestroy(&facePosIS));
1866     PetscCall(ISDestroy(&faceNegIS));
1867     PetscFunctionReturn(0);
1868   }
1869   PetscCall(ISExpand(facePosIS, faceNegIS, &dimIS));
1870   PetscCall(ISDestroy(&facePosIS));
1871   PetscCall(ISDestroy(&faceNegIS));
1872   PetscCall(ISGetLocalSize(dimIS, &numPoints));
1873   PetscCall(ISGetIndices(dimIS, &points));
1874   for (p = 0; p < numPoints; ++p) {
1875     const PetscInt  point = points[p];
1876     const PetscInt *support;
1877     PetscInt        supportSize, valA, valB;
1878 
1879     PetscCall(DMPlexGetSupportSize(dm, point, &supportSize));
1880     if (supportSize != 2) continue;
1881     PetscCall(DMPlexGetSupport(dm, point, &support));
1882     PetscCall(DMLabelGetValue(label, support[0], &valA));
1883     PetscCall(DMLabelGetValue(label, support[1], &valB));
1884     if ((valA == -1) || (valB == -1)) continue;
1885     if (valA * valB > 0) continue;
1886     /* Check that this face is not incident on only unsplit faces, meaning has at least one split face */
1887     {
1888       PetscInt *closure = NULL;
1889       PetscBool split   = PETSC_FALSE;
1890       PetscInt  closureSize, cl;
1891 
1892       PetscCall(DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
1893       for (cl = 0; cl < closureSize * 2; cl += 2) {
1894         PetscCall(DMLabelGetValue(label, closure[cl], &valA));
1895         if ((valA >= 0) && (valA <= dim)) {
1896           split = PETSC_TRUE;
1897           break;
1898         }
1899       }
1900       PetscCall(DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
1901       if (!split) continue;
1902     }
1903     /* Split the face */
1904     PetscCall(DMLabelGetValue(label, point, &valA));
1905     PetscCall(DMLabelClearValue(label, point, valA));
1906     PetscCall(DMLabelSetValue(label, point, dim - 1));
1907     /* Label its closure:
1908       unmarked: label as unsplit
1909       incident: relabel as split
1910       split:    do nothing
1911     */
1912     {
1913       PetscInt *closure = NULL;
1914       PetscInt  closureSize, cl, dep;
1915 
1916       PetscCall(DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
1917       for (cl = 0; cl < closureSize * 2; cl += 2) {
1918         PetscCall(DMLabelGetValue(label, closure[cl], &valA));
1919         if (valA == -1) { /* Mark as unsplit */
1920           PetscCall(DMPlexGetPointDepth(dm, closure[cl], &dep));
1921           PetscCall(DMLabelSetValue(label, closure[cl], shift2 + dep));
1922         } else if (((valA >= shift) && (valA < shift2)) || ((valA <= -shift) && (valA > -shift2))) {
1923           PetscCall(DMPlexGetPointDepth(dm, closure[cl], &dep));
1924           PetscCall(DMLabelClearValue(label, closure[cl], valA));
1925           PetscCall(DMLabelSetValue(label, closure[cl], dep));
1926         }
1927       }
1928       PetscCall(DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
1929     }
1930   }
1931   PetscCall(ISRestoreIndices(dimIS, &points));
1932   PetscCall(ISDestroy(&dimIS));
1933   PetscFunctionReturn(0);
1934 }
1935 
1936 /*@
1937   DMPlexLabelCohesiveComplete - Starting with a label marking points on an internal surface, we add all other mesh pieces
1938   to complete the surface
1939 
1940   Input Parameters:
1941 + dm     - The DM
1942 . label  - A DMLabel marking the surface
1943 . blabel - A DMLabel marking the vertices on the boundary which will not be duplicated, or NULL to find them automatically
1944 . bvalue - Value of DMLabel marking the vertices on the boundary
1945 . flip   - Flag to flip the submesh normal and replace points on the other side
1946 - subdm  - The subDM associated with the label, or NULL
1947 
1948   Output Parameter:
1949 . label - A DMLabel marking all surface points
1950 
1951   Note: The vertices in blabel are called "unsplit" in the terminology from hybrid cell creation.
1952 
1953   Level: developer
1954 
1955 .seealso: `DMPlexConstructCohesiveCells()`, `DMPlexLabelComplete()`
1956 @*/
1957 PetscErrorCode DMPlexLabelCohesiveComplete(DM dm, DMLabel label, DMLabel blabel, PetscInt bvalue, PetscBool flip, DM subdm) {
1958   DMLabel         depthLabel;
1959   IS              dimIS, subpointIS = NULL;
1960   const PetscInt *points, *subpoints;
1961   const PetscInt  rev   = flip ? -1 : 1;
1962   PetscInt        shift = 100, shift2 = 200, shift3 = 300, dim, depth, numPoints, numSubpoints, p, val;
1963 
1964   PetscFunctionBegin;
1965   PetscCall(DMPlexGetDepth(dm, &depth));
1966   PetscCall(DMGetDimension(dm, &dim));
1967   PetscCall(DMPlexGetDepthLabel(dm, &depthLabel));
1968   if (subdm) {
1969     PetscCall(DMPlexGetSubpointIS(subdm, &subpointIS));
1970     if (subpointIS) {
1971       PetscCall(ISGetLocalSize(subpointIS, &numSubpoints));
1972       PetscCall(ISGetIndices(subpointIS, &subpoints));
1973     }
1974   }
1975   /* Mark cell on the fault, and its faces which touch the fault: cell orientation for face gives the side of the fault */
1976   PetscCall(DMLabelGetStratumIS(label, dim - 1, &dimIS));
1977   if (!dimIS) goto divide;
1978   PetscCall(ISGetLocalSize(dimIS, &numPoints));
1979   PetscCall(ISGetIndices(dimIS, &points));
1980   for (p = 0; p < numPoints; ++p) { /* Loop over fault faces */
1981     const PetscInt *support;
1982     PetscInt        supportSize, s;
1983 
1984     PetscCall(DMPlexGetSupportSize(dm, points[p], &supportSize));
1985 #if 0
1986     if (supportSize != 2) {
1987       const PetscInt *lp;
1988       PetscInt        Nlp, pind;
1989 
1990       /* Check that for a cell with a single support face, that face is in the SF */
1991       /*   THis check only works for the remote side. We would need root side information */
1992       PetscCall(PetscSFGetGraph(dm->sf, NULL, &Nlp, &lp, NULL));
1993       PetscCall(PetscFindInt(points[p], Nlp, lp, &pind));
1994       PetscCheck(pind >= 0,PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Split face %" PetscInt_FMT " has %" PetscInt_FMT " != 2 supports, and the face is not shared with another process", points[p], supportSize);
1995     }
1996 #endif
1997     PetscCall(DMPlexGetSupport(dm, points[p], &support));
1998     for (s = 0; s < supportSize; ++s) {
1999       const PetscInt *cone;
2000       PetscInt        coneSize, c;
2001       PetscBool       pos;
2002 
2003       PetscCall(GetSurfaceSide_Static(dm, subdm, numSubpoints, subpoints, support[s], points[p], &pos));
2004       if (pos) PetscCall(DMLabelSetValue(label, support[s], rev * (shift + dim)));
2005       else PetscCall(DMLabelSetValue(label, support[s], -rev * (shift + dim)));
2006       if (rev < 0) pos = !pos ? PETSC_TRUE : PETSC_FALSE;
2007       /* Put faces touching the fault in the label */
2008       PetscCall(DMPlexGetConeSize(dm, support[s], &coneSize));
2009       PetscCall(DMPlexGetCone(dm, support[s], &cone));
2010       for (c = 0; c < coneSize; ++c) {
2011         const PetscInt point = cone[c];
2012 
2013         PetscCall(DMLabelGetValue(label, point, &val));
2014         if (val == -1) {
2015           PetscInt *closure = NULL;
2016           PetscInt  closureSize, cl;
2017 
2018           PetscCall(DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
2019           for (cl = 0; cl < closureSize * 2; cl += 2) {
2020             const PetscInt clp  = closure[cl];
2021             PetscInt       bval = -1;
2022 
2023             PetscCall(DMLabelGetValue(label, clp, &val));
2024             if (blabel) PetscCall(DMLabelGetValue(blabel, clp, &bval));
2025             if ((val >= 0) && (val < dim - 1) && (bval < 0)) {
2026               PetscCall(DMLabelSetValue(label, point, pos == PETSC_TRUE ? shift + dim - 1 : -(shift + dim - 1)));
2027               break;
2028             }
2029           }
2030           PetscCall(DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
2031         }
2032       }
2033     }
2034   }
2035   PetscCall(ISRestoreIndices(dimIS, &points));
2036   PetscCall(ISDestroy(&dimIS));
2037   /* Mark boundary points as unsplit */
2038   if (blabel) {
2039     IS bdIS;
2040 
2041     PetscCall(DMLabelGetStratumIS(blabel, bvalue, &bdIS));
2042     PetscCall(ISGetLocalSize(bdIS, &numPoints));
2043     PetscCall(ISGetIndices(bdIS, &points));
2044     for (p = 0; p < numPoints; ++p) {
2045       const PetscInt point = points[p];
2046       PetscInt       val, bval;
2047 
2048       PetscCall(DMLabelGetValue(blabel, point, &bval));
2049       if (bval >= 0) {
2050         PetscCall(DMLabelGetValue(label, point, &val));
2051         if ((val < 0) || (val > dim)) {
2052           /* This could be a point added from splitting a vertex on an adjacent fault, otherwise its just wrong */
2053           PetscCall(DMLabelClearValue(blabel, point, bval));
2054         }
2055       }
2056     }
2057     for (p = 0; p < numPoints; ++p) {
2058       const PetscInt point = points[p];
2059       PetscInt       val, bval;
2060 
2061       PetscCall(DMLabelGetValue(blabel, point, &bval));
2062       if (bval >= 0) {
2063         const PetscInt *cone, *support;
2064         PetscInt        coneSize, supportSize, s, valA, valB, valE;
2065 
2066         /* Mark as unsplit */
2067         PetscCall(DMLabelGetValue(label, point, &val));
2068         PetscCheck(val >= 0 && val <= dim, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %" PetscInt_FMT " has label value %" PetscInt_FMT ", should be part of the fault", point, val);
2069         PetscCall(DMLabelClearValue(label, point, val));
2070         PetscCall(DMLabelSetValue(label, point, shift2 + val));
2071         /* Check for cross-edge
2072              A cross-edge has endpoints which are both on the boundary of the surface, but the edge itself is not. */
2073         if (val != 0) continue;
2074         PetscCall(DMPlexGetSupport(dm, point, &support));
2075         PetscCall(DMPlexGetSupportSize(dm, point, &supportSize));
2076         for (s = 0; s < supportSize; ++s) {
2077           PetscCall(DMPlexGetCone(dm, support[s], &cone));
2078           PetscCall(DMPlexGetConeSize(dm, support[s], &coneSize));
2079           PetscCheck(coneSize == 2, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Edge %" PetscInt_FMT " has %" PetscInt_FMT " vertices != 2", support[s], coneSize);
2080           PetscCall(DMLabelGetValue(blabel, cone[0], &valA));
2081           PetscCall(DMLabelGetValue(blabel, cone[1], &valB));
2082           PetscCall(DMLabelGetValue(blabel, support[s], &valE));
2083           if ((valE < 0) && (valA >= 0) && (valB >= 0) && (cone[0] != cone[1])) PetscCall(DMLabelSetValue(blabel, support[s], 2));
2084         }
2085       }
2086     }
2087     PetscCall(ISRestoreIndices(bdIS, &points));
2088     PetscCall(ISDestroy(&bdIS));
2089   }
2090   /* Mark ghost fault cells */
2091   {
2092     PetscSF         sf;
2093     const PetscInt *leaves;
2094     PetscInt        Nl, l;
2095 
2096     PetscCall(DMGetPointSF(dm, &sf));
2097     PetscCall(PetscSFGetGraph(sf, NULL, &Nl, &leaves, NULL));
2098     PetscCall(DMLabelGetStratumIS(label, dim - 1, &dimIS));
2099     if (!dimIS) goto divide;
2100     PetscCall(ISGetLocalSize(dimIS, &numPoints));
2101     PetscCall(ISGetIndices(dimIS, &points));
2102     if (Nl > 0) {
2103       for (p = 0; p < numPoints; ++p) {
2104         const PetscInt point = points[p];
2105         PetscInt       val;
2106 
2107         PetscCall(PetscFindInt(point, Nl, leaves, &l));
2108         if (l >= 0) {
2109           PetscInt *closure = NULL;
2110           PetscInt  closureSize, cl;
2111 
2112           PetscCall(DMLabelGetValue(label, point, &val));
2113           PetscCheck((val == dim - 1) || (val == shift2 + dim - 1), PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %" PetscInt_FMT " has label value %" PetscInt_FMT ", should be a fault face", point, val);
2114           PetscCall(DMLabelClearValue(label, point, val));
2115           PetscCall(DMLabelSetValue(label, point, shift3 + val));
2116           PetscCall(DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
2117           for (cl = 2; cl < closureSize * 2; cl += 2) {
2118             const PetscInt clp = closure[cl];
2119 
2120             PetscCall(DMLabelGetValue(label, clp, &val));
2121             PetscCheck(val != -1, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %" PetscInt_FMT " is missing from label, but is in the closure of a fault face", point);
2122             PetscCall(DMLabelClearValue(label, clp, val));
2123             PetscCall(DMLabelSetValue(label, clp, shift3 + val));
2124           }
2125           PetscCall(DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
2126         }
2127       }
2128     }
2129     PetscCall(ISRestoreIndices(dimIS, &points));
2130     PetscCall(ISDestroy(&dimIS));
2131   }
2132 divide:
2133   if (subpointIS) PetscCall(ISRestoreIndices(subpointIS, &subpoints));
2134   PetscCall(DMPlexLabelFaultHalo(dm, label));
2135   PetscCall(CheckFaultEdge_Private(dm, label));
2136   PetscFunctionReturn(0);
2137 }
2138 
2139 /* Check that no cell have all vertices on the fault */
2140 PetscErrorCode DMPlexCheckValidSubmesh_Private(DM dm, DMLabel label, DM subdm) {
2141   IS              subpointIS;
2142   const PetscInt *dmpoints;
2143   PetscInt        defaultValue, cStart, cEnd, c, vStart, vEnd;
2144 
2145   PetscFunctionBegin;
2146   if (!label) PetscFunctionReturn(0);
2147   PetscCall(DMLabelGetDefaultValue(label, &defaultValue));
2148   PetscCall(DMPlexGetSubpointIS(subdm, &subpointIS));
2149   if (!subpointIS) PetscFunctionReturn(0);
2150   PetscCall(DMPlexGetHeightStratum(subdm, 0, &cStart, &cEnd));
2151   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
2152   PetscCall(ISGetIndices(subpointIS, &dmpoints));
2153   for (c = cStart; c < cEnd; ++c) {
2154     PetscBool invalidCell = PETSC_TRUE;
2155     PetscInt *closure     = NULL;
2156     PetscInt  closureSize, cl;
2157 
2158     PetscCall(DMPlexGetTransitiveClosure(dm, dmpoints[c], PETSC_TRUE, &closureSize, &closure));
2159     for (cl = 0; cl < closureSize * 2; cl += 2) {
2160       PetscInt value = 0;
2161 
2162       if ((closure[cl] < vStart) || (closure[cl] >= vEnd)) continue;
2163       PetscCall(DMLabelGetValue(label, closure[cl], &value));
2164       if (value == defaultValue) {
2165         invalidCell = PETSC_FALSE;
2166         break;
2167       }
2168     }
2169     PetscCall(DMPlexRestoreTransitiveClosure(dm, dmpoints[c], PETSC_TRUE, &closureSize, &closure));
2170     if (invalidCell) {
2171       PetscCall(ISRestoreIndices(subpointIS, &dmpoints));
2172       PetscCall(ISDestroy(&subpointIS));
2173       PetscCall(DMDestroy(&subdm));
2174       SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Ambiguous submesh. Cell %" PetscInt_FMT " has all of its vertices on the submesh.", dmpoints[c]);
2175     }
2176   }
2177   PetscCall(ISRestoreIndices(subpointIS, &dmpoints));
2178   PetscFunctionReturn(0);
2179 }
2180 
2181 /*@
2182   DMPlexCreateHybridMesh - Create a mesh with hybrid cells along an internal interface
2183 
2184   Collective on dm
2185 
2186   Input Parameters:
2187 + dm - The original DM
2188 . label - The label specifying the interface vertices
2189 . bdlabel - The optional label specifying the interface boundary vertices
2190 - bdvalue - Value of optional label specifying the interface boundary vertices
2191 
2192   Output Parameters:
2193 + hybridLabel - The label fully marking the interface, or NULL if no output is desired
2194 . splitLabel - The label containing the split points, or NULL if no output is desired
2195 . dmInterface - The new interface DM, or NULL
2196 - dmHybrid - The new DM with cohesive cells
2197 
2198   Note: The hybridLabel indicates what parts of the original mesh impinged on the on division surface. For points
2199   directly on the division surface, they are labeled with their dimension, so an edge 7 on the division surface would be
2200   7 (1) in hybridLabel. For points that impinge from the positive side, they are labeled with 100+dim, so an edge 6 with
2201   one vertex 3 on the surface would be 6 (101) and 3 (0) in hybridLabel. If an edge 9 from the negative side of the
2202   surface also hits vertex 3, it would be 9 (-101) in hybridLabel.
2203 
2204   The splitLabel indicates what points in the new hybrid mesh were the result of splitting points in the original
2205   mesh. The label value is +=100+dim for each point. For example, if two edges 10 and 14 in the hybrid resulting from
2206   splitting an edge in the original mesh, you would have 10 (101) and 14 (-101) in the splitLabel.
2207 
2208   The dmInterface is a DM built from the original division surface. It has a label which can be retrieved using
2209   DMPlexGetSubpointMap() which maps each point back to the point in the surface of the original mesh.
2210 
2211   Level: developer
2212 
2213 .seealso: `DMPlexConstructCohesiveCells()`, `DMPlexLabelCohesiveComplete()`, `DMPlexGetSubpointMap()`, `DMCreate()`
2214 @*/
2215 PetscErrorCode DMPlexCreateHybridMesh(DM dm, DMLabel label, DMLabel bdlabel, PetscInt bdvalue, DMLabel *hybridLabel, DMLabel *splitLabel, DM *dmInterface, DM *dmHybrid) {
2216   DM       idm;
2217   DMLabel  subpointMap, hlabel, slabel = NULL;
2218   PetscInt dim;
2219 
2220   PetscFunctionBegin;
2221   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2222   if (label) PetscValidPointer(label, 2);
2223   if (bdlabel) PetscValidPointer(bdlabel, 3);
2224   if (hybridLabel) PetscValidPointer(hybridLabel, 5);
2225   if (splitLabel) PetscValidPointer(splitLabel, 6);
2226   if (dmInterface) PetscValidPointer(dmInterface, 7);
2227   PetscValidPointer(dmHybrid, 8);
2228   PetscCall(DMGetDimension(dm, &dim));
2229   PetscCall(DMPlexCreateSubmesh(dm, label, 1, PETSC_FALSE, &idm));
2230   PetscCall(DMPlexCheckValidSubmesh_Private(dm, label, idm));
2231   PetscCall(DMPlexOrient(idm));
2232   PetscCall(DMPlexGetSubpointMap(idm, &subpointMap));
2233   PetscCall(DMLabelDuplicate(subpointMap, &hlabel));
2234   PetscCall(DMLabelClearStratum(hlabel, dim));
2235   if (splitLabel) {
2236     const char *name;
2237     char        sname[PETSC_MAX_PATH_LEN];
2238 
2239     PetscCall(PetscObjectGetName((PetscObject)hlabel, &name));
2240     PetscCall(PetscStrncpy(sname, name, PETSC_MAX_PATH_LEN));
2241     PetscCall(PetscStrcat(sname, " split"));
2242     PetscCall(DMLabelCreate(PETSC_COMM_SELF, sname, &slabel));
2243   }
2244   PetscCall(DMPlexLabelCohesiveComplete(dm, hlabel, bdlabel, bdvalue, PETSC_FALSE, idm));
2245   if (dmInterface) {
2246     *dmInterface = idm;
2247   } else PetscCall(DMDestroy(&idm));
2248   PetscCall(DMPlexConstructCohesiveCells(dm, hlabel, slabel, dmHybrid));
2249   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_TRUE, *dmHybrid));
2250   if (hybridLabel) *hybridLabel = hlabel;
2251   else PetscCall(DMLabelDestroy(&hlabel));
2252   if (splitLabel) *splitLabel = slabel;
2253   {
2254     DM      cdm;
2255     DMLabel ctLabel;
2256 
2257     /* We need to somehow share the celltype label with the coordinate dm */
2258     PetscCall(DMGetCoordinateDM(*dmHybrid, &cdm));
2259     PetscCall(DMPlexGetCellTypeLabel(*dmHybrid, &ctLabel));
2260     PetscCall(DMSetLabel(cdm, ctLabel));
2261   }
2262   PetscFunctionReturn(0);
2263 }
2264 
2265 /* Here we need the explicit assumption that:
2266 
2267      For any marked cell, the marked vertices constitute a single face
2268 */
2269 static PetscErrorCode DMPlexMarkSubmesh_Uninterpolated(DM dm, DMLabel vertexLabel, PetscInt value, DMLabel subpointMap, PetscInt *numFaces, PetscInt *nFV, DM subdm) {
2270   IS              subvertexIS = NULL;
2271   const PetscInt *subvertices;
2272   PetscInt       *pStart, *pEnd, pSize;
2273   PetscInt        depth, dim, d, numSubVerticesInitial = 0, v;
2274 
2275   PetscFunctionBegin;
2276   *numFaces = 0;
2277   *nFV      = 0;
2278   PetscCall(DMPlexGetDepth(dm, &depth));
2279   PetscCall(DMGetDimension(dm, &dim));
2280   pSize = PetscMax(depth, dim) + 1;
2281   PetscCall(PetscMalloc2(pSize, &pStart, pSize, &pEnd));
2282   for (d = 0; d <= depth; ++d) PetscCall(DMPlexGetSimplexOrBoxCells(dm, depth - d, &pStart[d], &pEnd[d]));
2283   /* Loop over initial vertices and mark all faces in the collective star() */
2284   if (vertexLabel) PetscCall(DMLabelGetStratumIS(vertexLabel, value, &subvertexIS));
2285   if (subvertexIS) {
2286     PetscCall(ISGetSize(subvertexIS, &numSubVerticesInitial));
2287     PetscCall(ISGetIndices(subvertexIS, &subvertices));
2288   }
2289   for (v = 0; v < numSubVerticesInitial; ++v) {
2290     const PetscInt vertex = subvertices[v];
2291     PetscInt      *star   = NULL;
2292     PetscInt       starSize, s, numCells = 0, c;
2293 
2294     PetscCall(DMPlexGetTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star));
2295     for (s = 0; s < starSize * 2; s += 2) {
2296       const PetscInt point = star[s];
2297       if ((point >= pStart[depth]) && (point < pEnd[depth])) star[numCells++] = point;
2298     }
2299     for (c = 0; c < numCells; ++c) {
2300       const PetscInt cell    = star[c];
2301       PetscInt      *closure = NULL;
2302       PetscInt       closureSize, cl;
2303       PetscInt       cellLoc, numCorners = 0, faceSize = 0;
2304 
2305       PetscCall(DMLabelGetValue(subpointMap, cell, &cellLoc));
2306       if (cellLoc == 2) continue;
2307       PetscCheck(cellLoc < 0, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Cell %" PetscInt_FMT " has dimension %" PetscInt_FMT " in the surface label", cell, cellLoc);
2308       PetscCall(DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
2309       for (cl = 0; cl < closureSize * 2; cl += 2) {
2310         const PetscInt point = closure[cl];
2311         PetscInt       vertexLoc;
2312 
2313         if ((point >= pStart[0]) && (point < pEnd[0])) {
2314           ++numCorners;
2315           PetscCall(DMLabelGetValue(vertexLabel, point, &vertexLoc));
2316           if (vertexLoc == value) closure[faceSize++] = point;
2317         }
2318       }
2319       if (!(*nFV)) PetscCall(DMPlexGetNumFaceVertices(dm, dim, numCorners, nFV));
2320       PetscCheck(faceSize <= *nFV, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Invalid submesh: Too many vertices %" PetscInt_FMT " of an element on the surface", faceSize);
2321       if (faceSize == *nFV) {
2322         const PetscInt *cells = NULL;
2323         PetscInt        numCells, nc;
2324 
2325         ++(*numFaces);
2326         for (cl = 0; cl < faceSize; ++cl) PetscCall(DMLabelSetValue(subpointMap, closure[cl], 0));
2327         PetscCall(DMPlexGetJoin(dm, faceSize, closure, &numCells, &cells));
2328         for (nc = 0; nc < numCells; ++nc) PetscCall(DMLabelSetValue(subpointMap, cells[nc], 2));
2329         PetscCall(DMPlexRestoreJoin(dm, faceSize, closure, &numCells, &cells));
2330       }
2331       PetscCall(DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
2332     }
2333     PetscCall(DMPlexRestoreTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star));
2334   }
2335   if (subvertexIS) PetscCall(ISRestoreIndices(subvertexIS, &subvertices));
2336   PetscCall(ISDestroy(&subvertexIS));
2337   PetscCall(PetscFree2(pStart, pEnd));
2338   PetscFunctionReturn(0);
2339 }
2340 
2341 static PetscErrorCode DMPlexMarkSubmesh_Interpolated(DM dm, DMLabel vertexLabel, PetscInt value, PetscBool markedFaces, DMLabel subpointMap, DM subdm) {
2342   IS              subvertexIS = NULL;
2343   const PetscInt *subvertices;
2344   PetscInt       *pStart, *pEnd;
2345   PetscInt        dim, d, numSubVerticesInitial = 0, v;
2346 
2347   PetscFunctionBegin;
2348   PetscCall(DMGetDimension(dm, &dim));
2349   PetscCall(PetscMalloc2(dim + 1, &pStart, dim + 1, &pEnd));
2350   for (d = 0; d <= dim; ++d) PetscCall(DMPlexGetSimplexOrBoxCells(dm, dim - d, &pStart[d], &pEnd[d]));
2351   /* Loop over initial vertices and mark all faces in the collective star() */
2352   if (vertexLabel) {
2353     PetscCall(DMLabelGetStratumIS(vertexLabel, value, &subvertexIS));
2354     if (subvertexIS) {
2355       PetscCall(ISGetSize(subvertexIS, &numSubVerticesInitial));
2356       PetscCall(ISGetIndices(subvertexIS, &subvertices));
2357     }
2358   }
2359   for (v = 0; v < numSubVerticesInitial; ++v) {
2360     const PetscInt vertex = subvertices[v];
2361     PetscInt      *star   = NULL;
2362     PetscInt       starSize, s, numFaces = 0, f;
2363 
2364     PetscCall(DMPlexGetTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star));
2365     for (s = 0; s < starSize * 2; s += 2) {
2366       const PetscInt point = star[s];
2367       PetscInt       faceLoc;
2368 
2369       if ((point >= pStart[dim - 1]) && (point < pEnd[dim - 1])) {
2370         if (markedFaces) {
2371           PetscCall(DMLabelGetValue(vertexLabel, point, &faceLoc));
2372           if (faceLoc < 0) continue;
2373         }
2374         star[numFaces++] = point;
2375       }
2376     }
2377     for (f = 0; f < numFaces; ++f) {
2378       const PetscInt face    = star[f];
2379       PetscInt      *closure = NULL;
2380       PetscInt       closureSize, c;
2381       PetscInt       faceLoc;
2382 
2383       PetscCall(DMLabelGetValue(subpointMap, face, &faceLoc));
2384       if (faceLoc == dim - 1) continue;
2385       PetscCheck(faceLoc < 0, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Face %" PetscInt_FMT " has dimension %" PetscInt_FMT " in the surface label", face, faceLoc);
2386       PetscCall(DMPlexGetTransitiveClosure(dm, face, PETSC_TRUE, &closureSize, &closure));
2387       for (c = 0; c < closureSize * 2; c += 2) {
2388         const PetscInt point = closure[c];
2389         PetscInt       vertexLoc;
2390 
2391         if ((point >= pStart[0]) && (point < pEnd[0])) {
2392           PetscCall(DMLabelGetValue(vertexLabel, point, &vertexLoc));
2393           if (vertexLoc != value) break;
2394         }
2395       }
2396       if (c == closureSize * 2) {
2397         const PetscInt *support;
2398         PetscInt        supportSize, s;
2399 
2400         for (c = 0; c < closureSize * 2; c += 2) {
2401           const PetscInt point = closure[c];
2402 
2403           for (d = 0; d < dim; ++d) {
2404             if ((point >= pStart[d]) && (point < pEnd[d])) {
2405               PetscCall(DMLabelSetValue(subpointMap, point, d));
2406               break;
2407             }
2408           }
2409         }
2410         PetscCall(DMPlexGetSupportSize(dm, face, &supportSize));
2411         PetscCall(DMPlexGetSupport(dm, face, &support));
2412         for (s = 0; s < supportSize; ++s) PetscCall(DMLabelSetValue(subpointMap, support[s], dim));
2413       }
2414       PetscCall(DMPlexRestoreTransitiveClosure(dm, face, PETSC_TRUE, &closureSize, &closure));
2415     }
2416     PetscCall(DMPlexRestoreTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star));
2417   }
2418   if (subvertexIS) PetscCall(ISRestoreIndices(subvertexIS, &subvertices));
2419   PetscCall(ISDestroy(&subvertexIS));
2420   PetscCall(PetscFree2(pStart, pEnd));
2421   PetscFunctionReturn(0);
2422 }
2423 
2424 static PetscErrorCode DMPlexMarkCohesiveSubmesh_Uninterpolated(DM dm, PetscBool hasLagrange, const char labelname[], PetscInt value, DMLabel subpointMap, PetscInt *numFaces, PetscInt *nFV, PetscInt *subCells[], DM subdm) {
2425   DMLabel         label = NULL;
2426   const PetscInt *cone;
2427   PetscInt        dim, cMax, cEnd, c, subc = 0, p, coneSize = -1;
2428 
2429   PetscFunctionBegin;
2430   *numFaces = 0;
2431   *nFV      = 0;
2432   if (labelname) PetscCall(DMGetLabel(dm, labelname, &label));
2433   *subCells = NULL;
2434   PetscCall(DMGetDimension(dm, &dim));
2435   PetscCall(DMPlexGetTensorPrismBounds_Internal(dm, dim, &cMax, &cEnd));
2436   if (cMax < 0) PetscFunctionReturn(0);
2437   if (label) {
2438     for (c = cMax; c < cEnd; ++c) {
2439       PetscInt val;
2440 
2441       PetscCall(DMLabelGetValue(label, c, &val));
2442       if (val == value) {
2443         ++(*numFaces);
2444         PetscCall(DMPlexGetConeSize(dm, c, &coneSize));
2445       }
2446     }
2447   } else {
2448     *numFaces = cEnd - cMax;
2449     PetscCall(DMPlexGetConeSize(dm, cMax, &coneSize));
2450   }
2451   PetscCall(PetscMalloc1(*numFaces * 2, subCells));
2452   if (!(*numFaces)) PetscFunctionReturn(0);
2453   *nFV = hasLagrange ? coneSize / 3 : coneSize / 2;
2454   for (c = cMax; c < cEnd; ++c) {
2455     const PetscInt *cells;
2456     PetscInt        numCells;
2457 
2458     if (label) {
2459       PetscInt val;
2460 
2461       PetscCall(DMLabelGetValue(label, c, &val));
2462       if (val != value) continue;
2463     }
2464     PetscCall(DMPlexGetCone(dm, c, &cone));
2465     for (p = 0; p < *nFV; ++p) PetscCall(DMLabelSetValue(subpointMap, cone[p], 0));
2466     /* Negative face */
2467     PetscCall(DMPlexGetJoin(dm, *nFV, cone, &numCells, &cells));
2468     /* Not true in parallel
2469     PetscCheck(numCells == 2,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive cells should separate two cells"); */
2470     for (p = 0; p < numCells; ++p) {
2471       PetscCall(DMLabelSetValue(subpointMap, cells[p], 2));
2472       (*subCells)[subc++] = cells[p];
2473     }
2474     PetscCall(DMPlexRestoreJoin(dm, *nFV, cone, &numCells, &cells));
2475     /* Positive face is not included */
2476   }
2477   PetscFunctionReturn(0);
2478 }
2479 
2480 static PetscErrorCode DMPlexMarkCohesiveSubmesh_Interpolated(DM dm, DMLabel label, PetscInt value, DMLabel subpointMap, DM subdm) {
2481   PetscInt *pStart, *pEnd;
2482   PetscInt  dim, cMax, cEnd, c, d;
2483 
2484   PetscFunctionBegin;
2485   PetscCall(DMGetDimension(dm, &dim));
2486   PetscCall(DMPlexGetTensorPrismBounds_Internal(dm, dim, &cMax, &cEnd));
2487   if (cMax < 0) PetscFunctionReturn(0);
2488   PetscCall(PetscMalloc2(dim + 1, &pStart, dim + 1, &pEnd));
2489   for (d = 0; d <= dim; ++d) PetscCall(DMPlexGetDepthStratum(dm, d, &pStart[d], &pEnd[d]));
2490   for (c = cMax; c < cEnd; ++c) {
2491     const PetscInt *cone;
2492     PetscInt       *closure = NULL;
2493     PetscInt        fconeSize, coneSize, closureSize, cl, val;
2494 
2495     if (label) {
2496       PetscCall(DMLabelGetValue(label, c, &val));
2497       if (val != value) continue;
2498     }
2499     PetscCall(DMPlexGetConeSize(dm, c, &coneSize));
2500     PetscCall(DMPlexGetCone(dm, c, &cone));
2501     PetscCall(DMPlexGetConeSize(dm, cone[0], &fconeSize));
2502     PetscCheck(coneSize == (fconeSize ? fconeSize : 1) + 2, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive cells should separate two cells");
2503     /* Negative face */
2504     PetscCall(DMPlexGetTransitiveClosure(dm, cone[0], PETSC_TRUE, &closureSize, &closure));
2505     for (cl = 0; cl < closureSize * 2; cl += 2) {
2506       const PetscInt point = closure[cl];
2507 
2508       for (d = 0; d <= dim; ++d) {
2509         if ((point >= pStart[d]) && (point < pEnd[d])) {
2510           PetscCall(DMLabelSetValue(subpointMap, point, d));
2511           break;
2512         }
2513       }
2514     }
2515     PetscCall(DMPlexRestoreTransitiveClosure(dm, cone[0], PETSC_TRUE, &closureSize, &closure));
2516     /* Cells -- positive face is not included */
2517     for (cl = 0; cl < 1; ++cl) {
2518       const PetscInt *support;
2519       PetscInt        supportSize, s;
2520 
2521       PetscCall(DMPlexGetSupportSize(dm, cone[cl], &supportSize));
2522       /* PetscCheck(supportSize == 2,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive faces should separate two cells"); */
2523       PetscCall(DMPlexGetSupport(dm, cone[cl], &support));
2524       for (s = 0; s < supportSize; ++s) PetscCall(DMLabelSetValue(subpointMap, support[s], dim));
2525     }
2526   }
2527   PetscCall(PetscFree2(pStart, pEnd));
2528   PetscFunctionReturn(0);
2529 }
2530 
2531 static PetscErrorCode DMPlexGetFaceOrientation(DM dm, PetscInt cell, PetscInt numCorners, PetscInt indices[], PetscInt oppositeVertex, PetscInt origVertices[], PetscInt faceVertices[], PetscBool *posOriented) {
2532   MPI_Comm       comm;
2533   PetscBool      posOrient = PETSC_FALSE;
2534   const PetscInt debug     = 0;
2535   PetscInt       cellDim, faceSize, f;
2536 
2537   PetscFunctionBegin;
2538   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
2539   PetscCall(DMGetDimension(dm, &cellDim));
2540   if (debug) PetscCall(PetscPrintf(comm, "cellDim: %" PetscInt_FMT " numCorners: %" PetscInt_FMT "\n", cellDim, numCorners));
2541 
2542   if (cellDim == 1 && numCorners == 2) {
2543     /* Triangle */
2544     faceSize  = numCorners - 1;
2545     posOrient = !(oppositeVertex % 2) ? PETSC_TRUE : PETSC_FALSE;
2546   } else if (cellDim == 2 && numCorners == 3) {
2547     /* Triangle */
2548     faceSize  = numCorners - 1;
2549     posOrient = !(oppositeVertex % 2) ? PETSC_TRUE : PETSC_FALSE;
2550   } else if (cellDim == 3 && numCorners == 4) {
2551     /* Tetrahedron */
2552     faceSize  = numCorners - 1;
2553     posOrient = (oppositeVertex % 2) ? PETSC_TRUE : PETSC_FALSE;
2554   } else if (cellDim == 1 && numCorners == 3) {
2555     /* Quadratic line */
2556     faceSize  = 1;
2557     posOrient = PETSC_TRUE;
2558   } else if (cellDim == 2 && numCorners == 4) {
2559     /* Quads */
2560     faceSize = 2;
2561     if ((indices[1] > indices[0]) && (indices[1] - indices[0] == 1)) {
2562       posOrient = PETSC_TRUE;
2563     } else if ((indices[0] == 3) && (indices[1] == 0)) {
2564       posOrient = PETSC_TRUE;
2565     } else {
2566       if (((indices[0] > indices[1]) && (indices[0] - indices[1] == 1)) || ((indices[0] == 0) && (indices[1] == 3))) {
2567         posOrient = PETSC_FALSE;
2568       } else SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid quad crossedge");
2569     }
2570   } else if (cellDim == 2 && numCorners == 6) {
2571     /* Quadratic triangle (I hate this) */
2572     /* Edges are determined by the first 2 vertices (corners of edges) */
2573     const PetscInt faceSizeTri = 3;
2574     PetscInt       sortedIndices[3], i, iFace;
2575     PetscBool      found                    = PETSC_FALSE;
2576     PetscInt       faceVerticesTriSorted[9] = {
2577             0, 3, 4, /* bottom */
2578             1, 4, 5, /* right */
2579             2, 3, 5, /* left */
2580     };
2581     PetscInt faceVerticesTri[9] = {
2582       0, 3, 4, /* bottom */
2583       1, 4, 5, /* right */
2584       2, 5, 3, /* left */
2585     };
2586 
2587     for (i = 0; i < faceSizeTri; ++i) sortedIndices[i] = indices[i];
2588     PetscCall(PetscSortInt(faceSizeTri, sortedIndices));
2589     for (iFace = 0; iFace < 3; ++iFace) {
2590       const PetscInt ii = iFace * faceSizeTri;
2591       PetscInt       fVertex, cVertex;
2592 
2593       if ((sortedIndices[0] == faceVerticesTriSorted[ii + 0]) && (sortedIndices[1] == faceVerticesTriSorted[ii + 1])) {
2594         for (fVertex = 0; fVertex < faceSizeTri; ++fVertex) {
2595           for (cVertex = 0; cVertex < faceSizeTri; ++cVertex) {
2596             if (indices[cVertex] == faceVerticesTri[ii + fVertex]) {
2597               faceVertices[fVertex] = origVertices[cVertex];
2598               break;
2599             }
2600           }
2601         }
2602         found = PETSC_TRUE;
2603         break;
2604       }
2605     }
2606     PetscCheck(found, comm, PETSC_ERR_ARG_WRONG, "Invalid tri crossface");
2607     if (posOriented) *posOriented = PETSC_TRUE;
2608     PetscFunctionReturn(0);
2609   } else if (cellDim == 2 && numCorners == 9) {
2610     /* Quadratic quad (I hate this) */
2611     /* Edges are determined by the first 2 vertices (corners of edges) */
2612     const PetscInt faceSizeQuad = 3;
2613     PetscInt       sortedIndices[3], i, iFace;
2614     PetscBool      found                      = PETSC_FALSE;
2615     PetscInt       faceVerticesQuadSorted[12] = {
2616             0, 1, 4, /* bottom */
2617             1, 2, 5, /* right */
2618             2, 3, 6, /* top */
2619             0, 3, 7, /* left */
2620     };
2621     PetscInt faceVerticesQuad[12] = {
2622       0, 1, 4, /* bottom */
2623       1, 2, 5, /* right */
2624       2, 3, 6, /* top */
2625       3, 0, 7, /* left */
2626     };
2627 
2628     for (i = 0; i < faceSizeQuad; ++i) sortedIndices[i] = indices[i];
2629     PetscCall(PetscSortInt(faceSizeQuad, sortedIndices));
2630     for (iFace = 0; iFace < 4; ++iFace) {
2631       const PetscInt ii = iFace * faceSizeQuad;
2632       PetscInt       fVertex, cVertex;
2633 
2634       if ((sortedIndices[0] == faceVerticesQuadSorted[ii + 0]) && (sortedIndices[1] == faceVerticesQuadSorted[ii + 1])) {
2635         for (fVertex = 0; fVertex < faceSizeQuad; ++fVertex) {
2636           for (cVertex = 0; cVertex < faceSizeQuad; ++cVertex) {
2637             if (indices[cVertex] == faceVerticesQuad[ii + fVertex]) {
2638               faceVertices[fVertex] = origVertices[cVertex];
2639               break;
2640             }
2641           }
2642         }
2643         found = PETSC_TRUE;
2644         break;
2645       }
2646     }
2647     PetscCheck(found, comm, PETSC_ERR_ARG_WRONG, "Invalid quad crossface");
2648     if (posOriented) *posOriented = PETSC_TRUE;
2649     PetscFunctionReturn(0);
2650   } else if (cellDim == 3 && numCorners == 8) {
2651     /* Hexes
2652        A hex is two oriented quads with the normal of the first
2653        pointing up at the second.
2654 
2655           7---6
2656          /|  /|
2657         4---5 |
2658         | 1-|-2
2659         |/  |/
2660         0---3
2661 
2662         Faces are determined by the first 4 vertices (corners of faces) */
2663     const PetscInt faceSizeHex = 4;
2664     PetscInt       sortedIndices[4], i, iFace;
2665     PetscBool      found                     = PETSC_FALSE;
2666     PetscInt       faceVerticesHexSorted[24] = {
2667             0, 1, 2, 3, /* bottom */
2668             4, 5, 6, 7, /* top */
2669             0, 3, 4, 5, /* front */
2670             2, 3, 5, 6, /* right */
2671             1, 2, 6, 7, /* back */
2672             0, 1, 4, 7, /* left */
2673     };
2674     PetscInt faceVerticesHex[24] = {
2675       1, 2, 3, 0, /* bottom */
2676       4, 5, 6, 7, /* top */
2677       0, 3, 5, 4, /* front */
2678       3, 2, 6, 5, /* right */
2679       2, 1, 7, 6, /* back */
2680       1, 0, 4, 7, /* left */
2681     };
2682 
2683     for (i = 0; i < faceSizeHex; ++i) sortedIndices[i] = indices[i];
2684     PetscCall(PetscSortInt(faceSizeHex, sortedIndices));
2685     for (iFace = 0; iFace < 6; ++iFace) {
2686       const PetscInt ii = iFace * faceSizeHex;
2687       PetscInt       fVertex, cVertex;
2688 
2689       if ((sortedIndices[0] == faceVerticesHexSorted[ii + 0]) && (sortedIndices[1] == faceVerticesHexSorted[ii + 1]) && (sortedIndices[2] == faceVerticesHexSorted[ii + 2]) && (sortedIndices[3] == faceVerticesHexSorted[ii + 3])) {
2690         for (fVertex = 0; fVertex < faceSizeHex; ++fVertex) {
2691           for (cVertex = 0; cVertex < faceSizeHex; ++cVertex) {
2692             if (indices[cVertex] == faceVerticesHex[ii + fVertex]) {
2693               faceVertices[fVertex] = origVertices[cVertex];
2694               break;
2695             }
2696           }
2697         }
2698         found = PETSC_TRUE;
2699         break;
2700       }
2701     }
2702     PetscCheck(found, comm, PETSC_ERR_ARG_WRONG, "Invalid hex crossface");
2703     if (posOriented) *posOriented = PETSC_TRUE;
2704     PetscFunctionReturn(0);
2705   } else if (cellDim == 3 && numCorners == 10) {
2706     /* Quadratic tet */
2707     /* Faces are determined by the first 3 vertices (corners of faces) */
2708     const PetscInt faceSizeTet = 6;
2709     PetscInt       sortedIndices[6], i, iFace;
2710     PetscBool      found                     = PETSC_FALSE;
2711     PetscInt       faceVerticesTetSorted[24] = {
2712             0, 1, 2, 6, 7, 8, /* bottom */
2713             0, 3, 4, 6, 7, 9, /* front */
2714             1, 4, 5, 7, 8, 9, /* right */
2715             2, 3, 5, 6, 8, 9, /* left */
2716     };
2717     PetscInt faceVerticesTet[24] = {
2718       0, 1, 2, 6, 7, 8, /* bottom */
2719       0, 4, 3, 6, 7, 9, /* front */
2720       1, 5, 4, 7, 8, 9, /* right */
2721       2, 3, 5, 8, 6, 9, /* left */
2722     };
2723 
2724     for (i = 0; i < faceSizeTet; ++i) sortedIndices[i] = indices[i];
2725     PetscCall(PetscSortInt(faceSizeTet, sortedIndices));
2726     for (iFace = 0; iFace < 4; ++iFace) {
2727       const PetscInt ii = iFace * faceSizeTet;
2728       PetscInt       fVertex, cVertex;
2729 
2730       if ((sortedIndices[0] == faceVerticesTetSorted[ii + 0]) && (sortedIndices[1] == faceVerticesTetSorted[ii + 1]) && (sortedIndices[2] == faceVerticesTetSorted[ii + 2]) && (sortedIndices[3] == faceVerticesTetSorted[ii + 3])) {
2731         for (fVertex = 0; fVertex < faceSizeTet; ++fVertex) {
2732           for (cVertex = 0; cVertex < faceSizeTet; ++cVertex) {
2733             if (indices[cVertex] == faceVerticesTet[ii + fVertex]) {
2734               faceVertices[fVertex] = origVertices[cVertex];
2735               break;
2736             }
2737           }
2738         }
2739         found = PETSC_TRUE;
2740         break;
2741       }
2742     }
2743     PetscCheck(found, comm, PETSC_ERR_ARG_WRONG, "Invalid tet crossface");
2744     if (posOriented) *posOriented = PETSC_TRUE;
2745     PetscFunctionReturn(0);
2746   } else if (cellDim == 3 && numCorners == 27) {
2747     /* Quadratic hexes (I hate this)
2748        A hex is two oriented quads with the normal of the first
2749        pointing up at the second.
2750 
2751          7---6
2752         /|  /|
2753        4---5 |
2754        | 3-|-2
2755        |/  |/
2756        0---1
2757 
2758        Faces are determined by the first 4 vertices (corners of faces) */
2759     const PetscInt faceSizeQuadHex = 9;
2760     PetscInt       sortedIndices[9], i, iFace;
2761     PetscBool      found                         = PETSC_FALSE;
2762     PetscInt       faceVerticesQuadHexSorted[54] = {
2763             0, 1, 2, 3, 8,  9,  10, 11, 24, /* bottom */
2764             4, 5, 6, 7, 12, 13, 14, 15, 25, /* top */
2765             0, 1, 4, 5, 8,  12, 16, 17, 22, /* front */
2766             1, 2, 5, 6, 9,  13, 17, 18, 21, /* right */
2767             2, 3, 6, 7, 10, 14, 18, 19, 23, /* back */
2768             0, 3, 4, 7, 11, 15, 16, 19, 20, /* left */
2769     };
2770     PetscInt faceVerticesQuadHex[54] = {
2771       3, 2, 1, 0, 10, 9,  8,  11, 24, /* bottom */
2772       4, 5, 6, 7, 12, 13, 14, 15, 25, /* top */
2773       0, 1, 5, 4, 8,  17, 12, 16, 22, /* front */
2774       1, 2, 6, 5, 9,  18, 13, 17, 21, /* right */
2775       2, 3, 7, 6, 10, 19, 14, 18, 23, /* back */
2776       3, 0, 4, 7, 11, 16, 15, 19, 20  /* left */
2777     };
2778 
2779     for (i = 0; i < faceSizeQuadHex; ++i) sortedIndices[i] = indices[i];
2780     PetscCall(PetscSortInt(faceSizeQuadHex, sortedIndices));
2781     for (iFace = 0; iFace < 6; ++iFace) {
2782       const PetscInt ii = iFace * faceSizeQuadHex;
2783       PetscInt       fVertex, cVertex;
2784 
2785       if ((sortedIndices[0] == faceVerticesQuadHexSorted[ii + 0]) && (sortedIndices[1] == faceVerticesQuadHexSorted[ii + 1]) && (sortedIndices[2] == faceVerticesQuadHexSorted[ii + 2]) && (sortedIndices[3] == faceVerticesQuadHexSorted[ii + 3])) {
2786         for (fVertex = 0; fVertex < faceSizeQuadHex; ++fVertex) {
2787           for (cVertex = 0; cVertex < faceSizeQuadHex; ++cVertex) {
2788             if (indices[cVertex] == faceVerticesQuadHex[ii + fVertex]) {
2789               faceVertices[fVertex] = origVertices[cVertex];
2790               break;
2791             }
2792           }
2793         }
2794         found = PETSC_TRUE;
2795         break;
2796       }
2797     }
2798     PetscCheck(found, comm, PETSC_ERR_ARG_WRONG, "Invalid hex crossface");
2799     if (posOriented) *posOriented = PETSC_TRUE;
2800     PetscFunctionReturn(0);
2801   } else SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Unknown cell type for faceOrientation().");
2802   if (!posOrient) {
2803     if (debug) PetscCall(PetscPrintf(comm, "  Reversing initial face orientation\n"));
2804     for (f = 0; f < faceSize; ++f) faceVertices[f] = origVertices[faceSize - 1 - f];
2805   } else {
2806     if (debug) PetscCall(PetscPrintf(comm, "  Keeping initial face orientation\n"));
2807     for (f = 0; f < faceSize; ++f) faceVertices[f] = origVertices[f];
2808   }
2809   if (posOriented) *posOriented = posOrient;
2810   PetscFunctionReturn(0);
2811 }
2812 
2813 /*@
2814   DMPlexGetOrientedFace - Given a cell and a face, as a set of vertices, return the oriented face, as a set of vertices,
2815   in faceVertices. The orientation is such that the face normal points out of the cell
2816 
2817   Not collective
2818 
2819   Input Parameters:
2820 + dm           - The original mesh
2821 . cell         - The cell mesh point
2822 . faceSize     - The number of vertices on the face
2823 . face         - The face vertices
2824 . numCorners   - The number of vertices on the cell
2825 . indices      - Local numbering of face vertices in cell cone
2826 - origVertices - Original face vertices
2827 
2828   Output Parameters:
2829 + faceVertices - The face vertices properly oriented
2830 - posOriented  - PETSC_TRUE if the face was oriented with outward normal
2831 
2832   Level: developer
2833 
2834 .seealso: `DMPlexGetCone()`
2835 @*/
2836 PetscErrorCode DMPlexGetOrientedFace(DM dm, PetscInt cell, PetscInt faceSize, const PetscInt face[], PetscInt numCorners, PetscInt indices[], PetscInt origVertices[], PetscInt faceVertices[], PetscBool *posOriented) {
2837   const PetscInt *cone = NULL;
2838   PetscInt        coneSize, v, f, v2;
2839   PetscInt        oppositeVertex = -1;
2840 
2841   PetscFunctionBegin;
2842   PetscCall(DMPlexGetConeSize(dm, cell, &coneSize));
2843   PetscCall(DMPlexGetCone(dm, cell, &cone));
2844   for (v = 0, v2 = 0; v < coneSize; ++v) {
2845     PetscBool found = PETSC_FALSE;
2846 
2847     for (f = 0; f < faceSize; ++f) {
2848       if (face[f] == cone[v]) {
2849         found = PETSC_TRUE;
2850         break;
2851       }
2852     }
2853     if (found) {
2854       indices[v2]      = v;
2855       origVertices[v2] = cone[v];
2856       ++v2;
2857     } else {
2858       oppositeVertex = v;
2859     }
2860   }
2861   PetscCall(DMPlexGetFaceOrientation(dm, cell, numCorners, indices, oppositeVertex, origVertices, faceVertices, posOriented));
2862   PetscFunctionReturn(0);
2863 }
2864 
2865 /*
2866   DMPlexInsertFace_Internal - Puts a face into the mesh
2867 
2868   Not collective
2869 
2870   Input Parameters:
2871   + dm              - The DMPlex
2872   . numFaceVertex   - The number of vertices in the face
2873   . faceVertices    - The vertices in the face for dm
2874   . subfaceVertices - The vertices in the face for subdm
2875   . numCorners      - The number of vertices in the cell
2876   . cell            - A cell in dm containing the face
2877   . subcell         - A cell in subdm containing the face
2878   . firstFace       - First face in the mesh
2879   - newFacePoint    - Next face in the mesh
2880 
2881   Output Parameters:
2882   . newFacePoint - Contains next face point number on input, updated on output
2883 
2884   Level: developer
2885 */
2886 static PetscErrorCode DMPlexInsertFace_Internal(DM dm, DM subdm, PetscInt numFaceVertices, const PetscInt faceVertices[], const PetscInt subfaceVertices[], PetscInt numCorners, PetscInt cell, PetscInt subcell, PetscInt firstFace, PetscInt *newFacePoint) {
2887   MPI_Comm        comm;
2888   DM_Plex        *submesh = (DM_Plex *)subdm->data;
2889   const PetscInt *faces;
2890   PetscInt        numFaces, coneSize;
2891 
2892   PetscFunctionBegin;
2893   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
2894   PetscCall(DMPlexGetConeSize(subdm, subcell, &coneSize));
2895   PetscCheck(coneSize == 1, comm, PETSC_ERR_ARG_OUTOFRANGE, "Cone size of cell %" PetscInt_FMT " is %" PetscInt_FMT " != 1", cell, coneSize);
2896 #if 0
2897   /* Cannot use this because support() has not been constructed yet */
2898   PetscCall(DMPlexGetJoin(subdm, numFaceVertices, subfaceVertices, &numFaces, &faces));
2899 #else
2900   {
2901     PetscInt f;
2902 
2903     numFaces = 0;
2904     PetscCall(DMGetWorkArray(subdm, 1, MPIU_INT, (void **)&faces));
2905     for (f = firstFace; f < *newFacePoint; ++f) {
2906       PetscInt dof, off, d;
2907 
2908       PetscCall(PetscSectionGetDof(submesh->coneSection, f, &dof));
2909       PetscCall(PetscSectionGetOffset(submesh->coneSection, f, &off));
2910       /* Yes, I know this is quadratic, but I expect the sizes to be <5 */
2911       for (d = 0; d < dof; ++d) {
2912         const PetscInt p = submesh->cones[off + d];
2913         PetscInt       v;
2914 
2915         for (v = 0; v < numFaceVertices; ++v) {
2916           if (subfaceVertices[v] == p) break;
2917         }
2918         if (v == numFaceVertices) break;
2919       }
2920       if (d == dof) {
2921         numFaces               = 1;
2922         ((PetscInt *)faces)[0] = f;
2923       }
2924     }
2925   }
2926 #endif
2927   PetscCheck(numFaces <= 1, comm, PETSC_ERR_ARG_WRONG, "Vertex set had %" PetscInt_FMT " faces, not one", numFaces);
2928   if (numFaces == 1) {
2929     /* Add the other cell neighbor for this face */
2930     PetscCall(DMPlexSetCone(subdm, subcell, faces));
2931   } else {
2932     PetscInt *indices, *origVertices, *orientedVertices, *orientedSubVertices, v, ov;
2933     PetscBool posOriented;
2934 
2935     PetscCall(DMGetWorkArray(subdm, 4 * numFaceVertices * sizeof(PetscInt), MPIU_INT, &orientedVertices));
2936     origVertices        = &orientedVertices[numFaceVertices];
2937     indices             = &orientedVertices[numFaceVertices * 2];
2938     orientedSubVertices = &orientedVertices[numFaceVertices * 3];
2939     PetscCall(DMPlexGetOrientedFace(dm, cell, numFaceVertices, faceVertices, numCorners, indices, origVertices, orientedVertices, &posOriented));
2940     /* TODO: I know that routine should return a permutation, not the indices */
2941     for (v = 0; v < numFaceVertices; ++v) {
2942       const PetscInt vertex = faceVertices[v], subvertex = subfaceVertices[v];
2943       for (ov = 0; ov < numFaceVertices; ++ov) {
2944         if (orientedVertices[ov] == vertex) {
2945           orientedSubVertices[ov] = subvertex;
2946           break;
2947         }
2948       }
2949       PetscCheck(ov != numFaceVertices, comm, PETSC_ERR_PLIB, "Could not find face vertex %" PetscInt_FMT " in orientated set", vertex);
2950     }
2951     PetscCall(DMPlexSetCone(subdm, *newFacePoint, orientedSubVertices));
2952     PetscCall(DMPlexSetCone(subdm, subcell, newFacePoint));
2953     PetscCall(DMRestoreWorkArray(subdm, 4 * numFaceVertices * sizeof(PetscInt), MPIU_INT, &orientedVertices));
2954     ++(*newFacePoint);
2955   }
2956 #if 0
2957   PetscCall(DMPlexRestoreJoin(subdm, numFaceVertices, subfaceVertices, &numFaces, &faces));
2958 #else
2959   PetscCall(DMRestoreWorkArray(subdm, 1, MPIU_INT, (void **)&faces));
2960 #endif
2961   PetscFunctionReturn(0);
2962 }
2963 
2964 static PetscErrorCode DMPlexCreateSubmesh_Uninterpolated(DM dm, DMLabel vertexLabel, PetscInt value, DM subdm) {
2965   MPI_Comm        comm;
2966   DMLabel         subpointMap;
2967   IS              subvertexIS, subcellIS;
2968   const PetscInt *subVertices, *subCells;
2969   PetscInt        numSubVertices, firstSubVertex, numSubCells;
2970   PetscInt       *subface, maxConeSize, numSubFaces = 0, firstSubFace, newFacePoint, nFV = 0;
2971   PetscInt        vStart, vEnd, c, f;
2972 
2973   PetscFunctionBegin;
2974   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
2975   /* Create subpointMap which marks the submesh */
2976   PetscCall(DMLabelCreate(PETSC_COMM_SELF, "subpoint_map", &subpointMap));
2977   PetscCall(DMPlexSetSubpointMap(subdm, subpointMap));
2978   PetscCall(DMLabelDestroy(&subpointMap));
2979   if (vertexLabel) PetscCall(DMPlexMarkSubmesh_Uninterpolated(dm, vertexLabel, value, subpointMap, &numSubFaces, &nFV, subdm));
2980   /* Setup chart */
2981   PetscCall(DMLabelGetStratumSize(subpointMap, 0, &numSubVertices));
2982   PetscCall(DMLabelGetStratumSize(subpointMap, 2, &numSubCells));
2983   PetscCall(DMPlexSetChart(subdm, 0, numSubCells + numSubFaces + numSubVertices));
2984   PetscCall(DMPlexSetVTKCellHeight(subdm, 1));
2985   /* Set cone sizes */
2986   firstSubVertex = numSubCells;
2987   firstSubFace   = numSubCells + numSubVertices;
2988   newFacePoint   = firstSubFace;
2989   PetscCall(DMLabelGetStratumIS(subpointMap, 0, &subvertexIS));
2990   if (subvertexIS) PetscCall(ISGetIndices(subvertexIS, &subVertices));
2991   PetscCall(DMLabelGetStratumIS(subpointMap, 2, &subcellIS));
2992   if (subcellIS) PetscCall(ISGetIndices(subcellIS, &subCells));
2993   for (c = 0; c < numSubCells; ++c) PetscCall(DMPlexSetConeSize(subdm, c, 1));
2994   for (f = firstSubFace; f < firstSubFace + numSubFaces; ++f) PetscCall(DMPlexSetConeSize(subdm, f, nFV));
2995   PetscCall(DMSetUp(subdm));
2996   /* Create face cones */
2997   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
2998   PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, NULL));
2999   PetscCall(DMGetWorkArray(subdm, maxConeSize, MPIU_INT, (void **)&subface));
3000   for (c = 0; c < numSubCells; ++c) {
3001     const PetscInt cell    = subCells[c];
3002     const PetscInt subcell = c;
3003     PetscInt      *closure = NULL;
3004     PetscInt       closureSize, cl, numCorners = 0, faceSize = 0;
3005 
3006     PetscCall(DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
3007     for (cl = 0; cl < closureSize * 2; cl += 2) {
3008       const PetscInt point = closure[cl];
3009       PetscInt       subVertex;
3010 
3011       if ((point >= vStart) && (point < vEnd)) {
3012         ++numCorners;
3013         PetscCall(PetscFindInt(point, numSubVertices, subVertices, &subVertex));
3014         if (subVertex >= 0) {
3015           closure[faceSize] = point;
3016           subface[faceSize] = firstSubVertex + subVertex;
3017           ++faceSize;
3018         }
3019       }
3020     }
3021     PetscCheck(faceSize <= nFV, comm, PETSC_ERR_ARG_WRONG, "Invalid submesh: Too many vertices %" PetscInt_FMT " of an element on the surface", faceSize);
3022     if (faceSize == nFV) PetscCall(DMPlexInsertFace_Internal(dm, subdm, faceSize, closure, subface, numCorners, cell, subcell, firstSubFace, &newFacePoint));
3023     PetscCall(DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
3024   }
3025   PetscCall(DMRestoreWorkArray(subdm, maxConeSize, MPIU_INT, (void **)&subface));
3026   PetscCall(DMPlexSymmetrize(subdm));
3027   PetscCall(DMPlexStratify(subdm));
3028   /* Build coordinates */
3029   {
3030     PetscSection coordSection, subCoordSection;
3031     Vec          coordinates, subCoordinates;
3032     PetscScalar *coords, *subCoords;
3033     PetscInt     numComp, coordSize, v;
3034     const char  *name;
3035 
3036     PetscCall(DMGetCoordinateSection(dm, &coordSection));
3037     PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
3038     PetscCall(DMGetCoordinateSection(subdm, &subCoordSection));
3039     PetscCall(PetscSectionSetNumFields(subCoordSection, 1));
3040     PetscCall(PetscSectionGetFieldComponents(coordSection, 0, &numComp));
3041     PetscCall(PetscSectionSetFieldComponents(subCoordSection, 0, numComp));
3042     PetscCall(PetscSectionSetChart(subCoordSection, firstSubVertex, firstSubVertex + numSubVertices));
3043     for (v = 0; v < numSubVertices; ++v) {
3044       const PetscInt vertex    = subVertices[v];
3045       const PetscInt subvertex = firstSubVertex + v;
3046       PetscInt       dof;
3047 
3048       PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3049       PetscCall(PetscSectionSetDof(subCoordSection, subvertex, dof));
3050       PetscCall(PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof));
3051     }
3052     PetscCall(PetscSectionSetUp(subCoordSection));
3053     PetscCall(PetscSectionGetStorageSize(subCoordSection, &coordSize));
3054     PetscCall(VecCreate(PETSC_COMM_SELF, &subCoordinates));
3055     PetscCall(PetscObjectGetName((PetscObject)coordinates, &name));
3056     PetscCall(PetscObjectSetName((PetscObject)subCoordinates, name));
3057     PetscCall(VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE));
3058     PetscCall(VecSetType(subCoordinates, VECSTANDARD));
3059     if (coordSize) {
3060       PetscCall(VecGetArray(coordinates, &coords));
3061       PetscCall(VecGetArray(subCoordinates, &subCoords));
3062       for (v = 0; v < numSubVertices; ++v) {
3063         const PetscInt vertex    = subVertices[v];
3064         const PetscInt subvertex = firstSubVertex + v;
3065         PetscInt       dof, off, sdof, soff, d;
3066 
3067         PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3068         PetscCall(PetscSectionGetOffset(coordSection, vertex, &off));
3069         PetscCall(PetscSectionGetDof(subCoordSection, subvertex, &sdof));
3070         PetscCall(PetscSectionGetOffset(subCoordSection, subvertex, &soff));
3071         PetscCheck(dof == sdof, comm, PETSC_ERR_PLIB, "Coordinate dimension %" PetscInt_FMT " on subvertex %" PetscInt_FMT ", vertex %" PetscInt_FMT " should be %" PetscInt_FMT, sdof, subvertex, vertex, dof);
3072         for (d = 0; d < dof; ++d) subCoords[soff + d] = coords[off + d];
3073       }
3074       PetscCall(VecRestoreArray(coordinates, &coords));
3075       PetscCall(VecRestoreArray(subCoordinates, &subCoords));
3076     }
3077     PetscCall(DMSetCoordinatesLocal(subdm, subCoordinates));
3078     PetscCall(VecDestroy(&subCoordinates));
3079   }
3080   /* Cleanup */
3081   if (subvertexIS) PetscCall(ISRestoreIndices(subvertexIS, &subVertices));
3082   PetscCall(ISDestroy(&subvertexIS));
3083   if (subcellIS) PetscCall(ISRestoreIndices(subcellIS, &subCells));
3084   PetscCall(ISDestroy(&subcellIS));
3085   PetscFunctionReturn(0);
3086 }
3087 
3088 /* TODO: Fix this to properly propogate up error conditions it may find */
3089 static inline PetscInt DMPlexFilterPoint_Internal(PetscInt point, PetscInt firstSubPoint, PetscInt numSubPoints, const PetscInt subPoints[]) {
3090   PetscInt       subPoint;
3091   PetscErrorCode ierr;
3092 
3093   ierr = PetscFindInt(point, numSubPoints, subPoints, &subPoint);
3094   if (ierr) return -1;
3095   return subPoint < 0 ? subPoint : firstSubPoint + subPoint;
3096 }
3097 
3098 static PetscErrorCode DMPlexFilterLabels_Internal(DM dm, const PetscInt numSubPoints[], const PetscInt *subpoints[], const PetscInt firstSubPoint[], DM subdm) {
3099   DMLabel  depthLabel;
3100   PetscInt Nl, l, d;
3101 
3102   PetscFunctionBegin;
3103   // Reset depth label for fast lookup
3104   PetscCall(DMPlexGetDepthLabel(dm, &depthLabel));
3105   PetscCall(DMLabelMakeAllInvalid_Internal(depthLabel));
3106   PetscCall(DMGetNumLabels(dm, &Nl));
3107   for (l = 0; l < Nl; ++l) {
3108     DMLabel         label, newlabel;
3109     const char     *lname;
3110     PetscBool       isDepth, isDim, isCelltype, isVTK;
3111     IS              valueIS;
3112     const PetscInt *values;
3113     PetscInt        Nv, v;
3114 
3115     PetscCall(DMGetLabelName(dm, l, &lname));
3116     PetscCall(PetscStrcmp(lname, "depth", &isDepth));
3117     PetscCall(PetscStrcmp(lname, "dim", &isDim));
3118     PetscCall(PetscStrcmp(lname, "celltype", &isCelltype));
3119     PetscCall(PetscStrcmp(lname, "vtk", &isVTK));
3120     if (isDepth || isDim || isCelltype || isVTK) continue;
3121     PetscCall(DMCreateLabel(subdm, lname));
3122     PetscCall(DMGetLabel(dm, lname, &label));
3123     PetscCall(DMGetLabel(subdm, lname, &newlabel));
3124     PetscCall(DMLabelGetDefaultValue(label, &v));
3125     PetscCall(DMLabelSetDefaultValue(newlabel, v));
3126     PetscCall(DMLabelGetValueIS(label, &valueIS));
3127     PetscCall(ISGetLocalSize(valueIS, &Nv));
3128     PetscCall(ISGetIndices(valueIS, &values));
3129     for (v = 0; v < Nv; ++v) {
3130       IS              pointIS;
3131       const PetscInt *points;
3132       PetscInt        Np, p;
3133 
3134       PetscCall(DMLabelGetStratumIS(label, values[v], &pointIS));
3135       PetscCall(ISGetLocalSize(pointIS, &Np));
3136       PetscCall(ISGetIndices(pointIS, &points));
3137       for (p = 0; p < Np; ++p) {
3138         const PetscInt point = points[p];
3139         PetscInt       subp;
3140 
3141         PetscCall(DMPlexGetPointDepth(dm, point, &d));
3142         subp = DMPlexFilterPoint_Internal(point, firstSubPoint[d], numSubPoints[d], subpoints[d]);
3143         if (subp >= 0) PetscCall(DMLabelSetValue(newlabel, subp, values[v]));
3144       }
3145       PetscCall(ISRestoreIndices(pointIS, &points));
3146       PetscCall(ISDestroy(&pointIS));
3147     }
3148     PetscCall(ISRestoreIndices(valueIS, &values));
3149     PetscCall(ISDestroy(&valueIS));
3150   }
3151   PetscFunctionReturn(0);
3152 }
3153 
3154 static PetscErrorCode DMPlexCreateSubmeshGeneric_Interpolated(DM dm, DMLabel label, PetscInt value, PetscBool markedFaces, PetscBool isCohesive, PetscInt cellHeight, DM subdm) {
3155   MPI_Comm         comm;
3156   DMLabel          subpointMap;
3157   IS              *subpointIS;
3158   const PetscInt **subpoints;
3159   PetscInt        *numSubPoints, *firstSubPoint, *coneNew, *orntNew;
3160   PetscInt         totSubPoints = 0, maxConeSize, dim, p, d, v;
3161   PetscMPIInt      rank;
3162 
3163   PetscFunctionBegin;
3164   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
3165   PetscCallMPI(MPI_Comm_rank(comm, &rank));
3166   /* Create subpointMap which marks the submesh */
3167   PetscCall(DMLabelCreate(PETSC_COMM_SELF, "subpoint_map", &subpointMap));
3168   PetscCall(DMPlexSetSubpointMap(subdm, subpointMap));
3169   if (cellHeight) {
3170     if (isCohesive) PetscCall(DMPlexMarkCohesiveSubmesh_Interpolated(dm, label, value, subpointMap, subdm));
3171     else PetscCall(DMPlexMarkSubmesh_Interpolated(dm, label, value, markedFaces, subpointMap, subdm));
3172   } else {
3173     DMLabel         depth;
3174     IS              pointIS;
3175     const PetscInt *points;
3176     PetscInt        numPoints = 0;
3177 
3178     PetscCall(DMPlexGetDepthLabel(dm, &depth));
3179     PetscCall(DMLabelGetStratumIS(label, value, &pointIS));
3180     if (pointIS) {
3181       PetscCall(ISGetIndices(pointIS, &points));
3182       PetscCall(ISGetLocalSize(pointIS, &numPoints));
3183     }
3184     for (p = 0; p < numPoints; ++p) {
3185       PetscInt *closure = NULL;
3186       PetscInt  closureSize, c, pdim;
3187 
3188       PetscCall(DMPlexGetTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closure));
3189       for (c = 0; c < closureSize * 2; c += 2) {
3190         PetscCall(DMLabelGetValue(depth, closure[c], &pdim));
3191         PetscCall(DMLabelSetValue(subpointMap, closure[c], pdim));
3192       }
3193       PetscCall(DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closure));
3194     }
3195     if (pointIS) PetscCall(ISRestoreIndices(pointIS, &points));
3196     PetscCall(ISDestroy(&pointIS));
3197   }
3198   /* Setup chart */
3199   PetscCall(DMGetDimension(dm, &dim));
3200   PetscCall(PetscMalloc4(dim + 1, &numSubPoints, dim + 1, &firstSubPoint, dim + 1, &subpointIS, dim + 1, &subpoints));
3201   for (d = 0; d <= dim; ++d) {
3202     PetscCall(DMLabelGetStratumSize(subpointMap, d, &numSubPoints[d]));
3203     totSubPoints += numSubPoints[d];
3204   }
3205   PetscCall(DMPlexSetChart(subdm, 0, totSubPoints));
3206   PetscCall(DMPlexSetVTKCellHeight(subdm, cellHeight));
3207   /* Set cone sizes */
3208   firstSubPoint[dim] = 0;
3209   firstSubPoint[0]   = firstSubPoint[dim] + numSubPoints[dim];
3210   if (dim > 1) firstSubPoint[dim - 1] = firstSubPoint[0] + numSubPoints[0];
3211   if (dim > 2) firstSubPoint[dim - 2] = firstSubPoint[dim - 1] + numSubPoints[dim - 1];
3212   for (d = 0; d <= dim; ++d) {
3213     PetscCall(DMLabelGetStratumIS(subpointMap, d, &subpointIS[d]));
3214     if (subpointIS[d]) PetscCall(ISGetIndices(subpointIS[d], &subpoints[d]));
3215   }
3216   /* We do not want this label automatically computed, instead we compute it here */
3217   PetscCall(DMCreateLabel(subdm, "celltype"));
3218   for (d = 0; d <= dim; ++d) {
3219     for (p = 0; p < numSubPoints[d]; ++p) {
3220       const PetscInt  point    = subpoints[d][p];
3221       const PetscInt  subpoint = firstSubPoint[d] + p;
3222       const PetscInt *cone;
3223       PetscInt        coneSize;
3224 
3225       PetscCall(DMPlexGetConeSize(dm, point, &coneSize));
3226       if (cellHeight && (d == dim)) {
3227         PetscInt coneSizeNew, c, val;
3228 
3229         PetscCall(DMPlexGetCone(dm, point, &cone));
3230         for (c = 0, coneSizeNew = 0; c < coneSize; ++c) {
3231           PetscCall(DMLabelGetValue(subpointMap, cone[c], &val));
3232           if (val >= 0) coneSizeNew++;
3233         }
3234         PetscCall(DMPlexSetConeSize(subdm, subpoint, coneSizeNew));
3235         PetscCall(DMPlexSetCellType(subdm, subpoint, DM_POLYTOPE_FV_GHOST));
3236       } else {
3237         DMPolytopeType ct;
3238 
3239         PetscCall(DMPlexSetConeSize(subdm, subpoint, coneSize));
3240         PetscCall(DMPlexGetCellType(dm, point, &ct));
3241         PetscCall(DMPlexSetCellType(subdm, subpoint, ct));
3242       }
3243     }
3244   }
3245   PetscCall(DMLabelDestroy(&subpointMap));
3246   PetscCall(DMSetUp(subdm));
3247   /* Set cones */
3248   PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, NULL));
3249   PetscCall(PetscMalloc2(maxConeSize, &coneNew, maxConeSize, &orntNew));
3250   for (d = 0; d <= dim; ++d) {
3251     for (p = 0; p < numSubPoints[d]; ++p) {
3252       const PetscInt  point    = subpoints[d][p];
3253       const PetscInt  subpoint = firstSubPoint[d] + p;
3254       const PetscInt *cone, *ornt;
3255       PetscInt        coneSize, subconeSize, coneSizeNew, c, subc, fornt = 0;
3256 
3257       if (d == dim - 1) {
3258         const PetscInt *support, *cone, *ornt;
3259         PetscInt        supportSize, coneSize, s, subc;
3260 
3261         PetscCall(DMPlexGetSupport(dm, point, &support));
3262         PetscCall(DMPlexGetSupportSize(dm, point, &supportSize));
3263         for (s = 0; s < supportSize; ++s) {
3264           PetscBool isHybrid;
3265 
3266           PetscCall(DMPlexCellIsHybrid_Internal(dm, support[s], &isHybrid));
3267           if (!isHybrid) continue;
3268           PetscCall(PetscFindInt(support[s], numSubPoints[d + 1], subpoints[d + 1], &subc));
3269           if (subc >= 0) {
3270             const PetscInt ccell = subpoints[d + 1][subc];
3271 
3272             PetscCall(DMPlexGetCone(dm, ccell, &cone));
3273             PetscCall(DMPlexGetConeSize(dm, ccell, &coneSize));
3274             PetscCall(DMPlexGetConeOrientation(dm, ccell, &ornt));
3275             for (c = 0; c < coneSize; ++c) {
3276               if (cone[c] == point) {
3277                 fornt = ornt[c];
3278                 break;
3279               }
3280             }
3281             break;
3282           }
3283         }
3284       }
3285       PetscCall(DMPlexGetConeSize(dm, point, &coneSize));
3286       PetscCall(DMPlexGetConeSize(subdm, subpoint, &subconeSize));
3287       PetscCall(DMPlexGetCone(dm, point, &cone));
3288       PetscCall(DMPlexGetConeOrientation(dm, point, &ornt));
3289       for (c = 0, coneSizeNew = 0; c < coneSize; ++c) {
3290         PetscCall(PetscFindInt(cone[c], numSubPoints[d - 1], subpoints[d - 1], &subc));
3291         if (subc >= 0) {
3292           coneNew[coneSizeNew] = firstSubPoint[d - 1] + subc;
3293           orntNew[coneSizeNew] = ornt[c];
3294           ++coneSizeNew;
3295         }
3296       }
3297       PetscCheck(coneSizeNew == subconeSize, comm, PETSC_ERR_PLIB, "Number of cone points located %" PetscInt_FMT " does not match subcone size %" PetscInt_FMT, coneSizeNew, subconeSize);
3298       PetscCall(DMPlexSetCone(subdm, subpoint, coneNew));
3299       PetscCall(DMPlexSetConeOrientation(subdm, subpoint, orntNew));
3300       if (fornt < 0) PetscCall(DMPlexOrientPoint(subdm, subpoint, fornt));
3301     }
3302   }
3303   PetscCall(PetscFree2(coneNew, orntNew));
3304   PetscCall(DMPlexSymmetrize(subdm));
3305   PetscCall(DMPlexStratify(subdm));
3306   /* Build coordinates */
3307   {
3308     PetscSection coordSection, subCoordSection;
3309     Vec          coordinates, subCoordinates;
3310     PetscScalar *coords, *subCoords;
3311     PetscInt     cdim, numComp, coordSize;
3312     const char  *name;
3313 
3314     PetscCall(DMGetCoordinateDim(dm, &cdim));
3315     PetscCall(DMGetCoordinateSection(dm, &coordSection));
3316     PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
3317     PetscCall(DMGetCoordinateSection(subdm, &subCoordSection));
3318     PetscCall(PetscSectionSetNumFields(subCoordSection, 1));
3319     PetscCall(PetscSectionGetFieldComponents(coordSection, 0, &numComp));
3320     PetscCall(PetscSectionSetFieldComponents(subCoordSection, 0, numComp));
3321     PetscCall(PetscSectionSetChart(subCoordSection, firstSubPoint[0], firstSubPoint[0] + numSubPoints[0]));
3322     for (v = 0; v < numSubPoints[0]; ++v) {
3323       const PetscInt vertex    = subpoints[0][v];
3324       const PetscInt subvertex = firstSubPoint[0] + v;
3325       PetscInt       dof;
3326 
3327       PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3328       PetscCall(PetscSectionSetDof(subCoordSection, subvertex, dof));
3329       PetscCall(PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof));
3330     }
3331     PetscCall(PetscSectionSetUp(subCoordSection));
3332     PetscCall(PetscSectionGetStorageSize(subCoordSection, &coordSize));
3333     PetscCall(VecCreate(PETSC_COMM_SELF, &subCoordinates));
3334     PetscCall(PetscObjectGetName((PetscObject)coordinates, &name));
3335     PetscCall(PetscObjectSetName((PetscObject)subCoordinates, name));
3336     PetscCall(VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE));
3337     PetscCall(VecSetBlockSize(subCoordinates, cdim));
3338     PetscCall(VecSetType(subCoordinates, VECSTANDARD));
3339     PetscCall(VecGetArray(coordinates, &coords));
3340     PetscCall(VecGetArray(subCoordinates, &subCoords));
3341     for (v = 0; v < numSubPoints[0]; ++v) {
3342       const PetscInt vertex    = subpoints[0][v];
3343       const PetscInt subvertex = firstSubPoint[0] + v;
3344       PetscInt       dof, off, sdof, soff, d;
3345 
3346       PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3347       PetscCall(PetscSectionGetOffset(coordSection, vertex, &off));
3348       PetscCall(PetscSectionGetDof(subCoordSection, subvertex, &sdof));
3349       PetscCall(PetscSectionGetOffset(subCoordSection, subvertex, &soff));
3350       PetscCheck(dof == sdof, comm, PETSC_ERR_PLIB, "Coordinate dimension %" PetscInt_FMT " on subvertex %" PetscInt_FMT ", vertex %" PetscInt_FMT " should be %" PetscInt_FMT, sdof, subvertex, vertex, dof);
3351       for (d = 0; d < dof; ++d) subCoords[soff + d] = coords[off + d];
3352     }
3353     PetscCall(VecRestoreArray(coordinates, &coords));
3354     PetscCall(VecRestoreArray(subCoordinates, &subCoords));
3355     PetscCall(DMSetCoordinatesLocal(subdm, subCoordinates));
3356     PetscCall(VecDestroy(&subCoordinates));
3357   }
3358   /* Build SF: We need this complexity because subpoints might not be selected on the owning process */
3359   {
3360     PetscSF            sfPoint, sfPointSub;
3361     IS                 subpIS;
3362     const PetscSFNode *remotePoints;
3363     PetscSFNode       *sremotePoints = NULL, *newLocalPoints = NULL, *newOwners = NULL;
3364     const PetscInt    *localPoints, *subpoints, *rootdegree;
3365     PetscInt          *slocalPoints = NULL;
3366     PetscInt           numRoots, numLeaves, numSubpoints = 0, numSubroots, numSubleaves = 0, l, sl = 0, ll = 0, pStart, pEnd, p;
3367     PetscMPIInt        rank, size;
3368 
3369     PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
3370     PetscCallMPI(MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size));
3371     PetscCall(DMGetPointSF(dm, &sfPoint));
3372     PetscCall(DMGetPointSF(subdm, &sfPointSub));
3373     PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
3374     PetscCall(DMPlexGetChart(subdm, NULL, &numSubroots));
3375     PetscCall(DMPlexGetSubpointIS(subdm, &subpIS));
3376     if (subpIS) {
3377       PetscCall(ISGetIndices(subpIS, &subpoints));
3378       PetscCall(ISGetLocalSize(subpIS, &numSubpoints));
3379     }
3380     PetscCall(PetscSFGetGraph(sfPoint, &numRoots, &numLeaves, &localPoints, &remotePoints));
3381     if (numRoots >= 0) {
3382       PetscCall(PetscSFComputeDegreeBegin(sfPoint, &rootdegree));
3383       PetscCall(PetscSFComputeDegreeEnd(sfPoint, &rootdegree));
3384       PetscCall(PetscMalloc2(pEnd - pStart, &newLocalPoints, numRoots, &newOwners));
3385       for (p = 0; p < pEnd - pStart; ++p) {
3386         newLocalPoints[p].rank  = -2;
3387         newLocalPoints[p].index = -2;
3388       }
3389       /* Set subleaves */
3390       for (l = 0; l < numLeaves; ++l) {
3391         const PetscInt point    = localPoints[l];
3392         const PetscInt subpoint = DMPlexFilterPoint_Internal(point, 0, numSubpoints, subpoints);
3393 
3394         if (subpoint < 0) continue;
3395         newLocalPoints[point - pStart].rank  = rank;
3396         newLocalPoints[point - pStart].index = subpoint;
3397         ++numSubleaves;
3398       }
3399       /* Must put in owned subpoints */
3400       for (p = pStart; p < pEnd; ++p) {
3401         newOwners[p - pStart].rank  = -3;
3402         newOwners[p - pStart].index = -3;
3403       }
3404       for (p = 0; p < numSubpoints; ++p) {
3405         /* Hold on to currently owned points */
3406         if (rootdegree[subpoints[p] - pStart]) newOwners[subpoints[p] - pStart].rank = rank + size;
3407         else newOwners[subpoints[p] - pStart].rank = rank;
3408         newOwners[subpoints[p] - pStart].index = p;
3409       }
3410       PetscCall(PetscSFReduceBegin(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC));
3411       PetscCall(PetscSFReduceEnd(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC));
3412       for (p = pStart; p < pEnd; ++p)
3413         if (newOwners[p - pStart].rank >= size) newOwners[p - pStart].rank -= size;
3414       PetscCall(PetscSFBcastBegin(sfPoint, MPIU_2INT, newOwners, newLocalPoints, MPI_REPLACE));
3415       PetscCall(PetscSFBcastEnd(sfPoint, MPIU_2INT, newOwners, newLocalPoints, MPI_REPLACE));
3416       PetscCall(PetscMalloc1(numSubleaves, &slocalPoints));
3417       PetscCall(PetscMalloc1(numSubleaves, &sremotePoints));
3418       for (l = 0; l < numLeaves; ++l) {
3419         const PetscInt point    = localPoints[l];
3420         const PetscInt subpoint = DMPlexFilterPoint_Internal(point, 0, numSubpoints, subpoints);
3421 
3422         if (subpoint < 0) continue;
3423         if (newLocalPoints[point].rank == rank) {
3424           ++ll;
3425           continue;
3426         }
3427         slocalPoints[sl]        = subpoint;
3428         sremotePoints[sl].rank  = newLocalPoints[point].rank;
3429         sremotePoints[sl].index = newLocalPoints[point].index;
3430         PetscCheck(sremotePoints[sl].rank >= 0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote rank for local point %" PetscInt_FMT, point);
3431         PetscCheck(sremotePoints[sl].index >= 0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote subpoint for local point %" PetscInt_FMT, point);
3432         ++sl;
3433       }
3434       PetscCheck(sl + ll == numSubleaves, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatch in number of subleaves %" PetscInt_FMT " + %" PetscInt_FMT " != %" PetscInt_FMT, sl, ll, numSubleaves);
3435       PetscCall(PetscFree2(newLocalPoints, newOwners));
3436       PetscCall(PetscSFSetGraph(sfPointSub, numSubroots, sl, slocalPoints, PETSC_OWN_POINTER, sremotePoints, PETSC_OWN_POINTER));
3437     }
3438     if (subpIS) PetscCall(ISRestoreIndices(subpIS, &subpoints));
3439   }
3440   /* Filter labels */
3441   PetscCall(DMPlexFilterLabels_Internal(dm, numSubPoints, subpoints, firstSubPoint, subdm));
3442   /* Cleanup */
3443   for (d = 0; d <= dim; ++d) {
3444     if (subpointIS[d]) PetscCall(ISRestoreIndices(subpointIS[d], &subpoints[d]));
3445     PetscCall(ISDestroy(&subpointIS[d]));
3446   }
3447   PetscCall(PetscFree4(numSubPoints, firstSubPoint, subpointIS, subpoints));
3448   PetscFunctionReturn(0);
3449 }
3450 
3451 static PetscErrorCode DMPlexCreateSubmesh_Interpolated(DM dm, DMLabel vertexLabel, PetscInt value, PetscBool markedFaces, DM subdm) {
3452   PetscFunctionBegin;
3453   PetscCall(DMPlexCreateSubmeshGeneric_Interpolated(dm, vertexLabel, value, markedFaces, PETSC_FALSE, 1, subdm));
3454   PetscFunctionReturn(0);
3455 }
3456 
3457 /*@
3458   DMPlexCreateSubmesh - Extract a hypersurface from the mesh using vertices defined by a label
3459 
3460   Input Parameters:
3461 + dm           - The original mesh
3462 . vertexLabel  - The DMLabel marking points contained in the surface
3463 . value        - The label value to use
3464 - markedFaces  - PETSC_TRUE if surface faces are marked in addition to vertices, PETSC_FALSE if only vertices are marked
3465 
3466   Output Parameter:
3467 . subdm - The surface mesh
3468 
3469   Note: This function produces a DMLabel mapping original points in the submesh to their depth. This can be obtained using DMPlexGetSubpointMap().
3470 
3471   Level: developer
3472 
3473 .seealso: `DMPlexGetSubpointMap()`, `DMGetLabel()`, `DMLabelSetValue()`
3474 @*/
3475 PetscErrorCode DMPlexCreateSubmesh(DM dm, DMLabel vertexLabel, PetscInt value, PetscBool markedFaces, DM *subdm) {
3476   DMPlexInterpolatedFlag interpolated;
3477   PetscInt               dim, cdim;
3478 
3479   PetscFunctionBegin;
3480   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3481   PetscValidPointer(subdm, 5);
3482   PetscCall(DMGetDimension(dm, &dim));
3483   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), subdm));
3484   PetscCall(DMSetType(*subdm, DMPLEX));
3485   PetscCall(DMSetDimension(*subdm, dim - 1));
3486   PetscCall(DMGetCoordinateDim(dm, &cdim));
3487   PetscCall(DMSetCoordinateDim(*subdm, cdim));
3488   PetscCall(DMPlexIsInterpolated(dm, &interpolated));
3489   PetscCheck(interpolated != DMPLEX_INTERPOLATED_PARTIAL, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Not for partially interpolated meshes");
3490   if (interpolated) {
3491     PetscCall(DMPlexCreateSubmesh_Interpolated(dm, vertexLabel, value, markedFaces, *subdm));
3492   } else {
3493     PetscCall(DMPlexCreateSubmesh_Uninterpolated(dm, vertexLabel, value, *subdm));
3494   }
3495   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_TRUE, *subdm));
3496   PetscFunctionReturn(0);
3497 }
3498 
3499 static PetscErrorCode DMPlexCreateCohesiveSubmesh_Uninterpolated(DM dm, PetscBool hasLagrange, const char label[], PetscInt value, DM subdm) {
3500   MPI_Comm        comm;
3501   DMLabel         subpointMap;
3502   IS              subvertexIS;
3503   const PetscInt *subVertices;
3504   PetscInt        numSubVertices, firstSubVertex, numSubCells, *subCells = NULL;
3505   PetscInt       *subface, maxConeSize, numSubFaces, firstSubFace, newFacePoint, nFV;
3506   PetscInt        c, f;
3507 
3508   PetscFunctionBegin;
3509   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
3510   /* Create subpointMap which marks the submesh */
3511   PetscCall(DMLabelCreate(PETSC_COMM_SELF, "subpoint_map", &subpointMap));
3512   PetscCall(DMPlexSetSubpointMap(subdm, subpointMap));
3513   PetscCall(DMLabelDestroy(&subpointMap));
3514   PetscCall(DMPlexMarkCohesiveSubmesh_Uninterpolated(dm, hasLagrange, label, value, subpointMap, &numSubFaces, &nFV, &subCells, subdm));
3515   /* Setup chart */
3516   PetscCall(DMLabelGetStratumSize(subpointMap, 0, &numSubVertices));
3517   PetscCall(DMLabelGetStratumSize(subpointMap, 2, &numSubCells));
3518   PetscCall(DMPlexSetChart(subdm, 0, numSubCells + numSubFaces + numSubVertices));
3519   PetscCall(DMPlexSetVTKCellHeight(subdm, 1));
3520   /* Set cone sizes */
3521   firstSubVertex = numSubCells;
3522   firstSubFace   = numSubCells + numSubVertices;
3523   newFacePoint   = firstSubFace;
3524   PetscCall(DMLabelGetStratumIS(subpointMap, 0, &subvertexIS));
3525   if (subvertexIS) PetscCall(ISGetIndices(subvertexIS, &subVertices));
3526   for (c = 0; c < numSubCells; ++c) PetscCall(DMPlexSetConeSize(subdm, c, 1));
3527   for (f = firstSubFace; f < firstSubFace + numSubFaces; ++f) PetscCall(DMPlexSetConeSize(subdm, f, nFV));
3528   PetscCall(DMSetUp(subdm));
3529   /* Create face cones */
3530   PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, NULL));
3531   PetscCall(DMGetWorkArray(subdm, maxConeSize, MPIU_INT, (void **)&subface));
3532   for (c = 0; c < numSubCells; ++c) {
3533     const PetscInt  cell    = subCells[c];
3534     const PetscInt  subcell = c;
3535     const PetscInt *cone, *cells;
3536     PetscBool       isHybrid;
3537     PetscInt        numCells, subVertex, p, v;
3538 
3539     PetscCall(DMPlexCellIsHybrid_Internal(dm, cell, &isHybrid));
3540     if (!isHybrid) continue;
3541     PetscCall(DMPlexGetCone(dm, cell, &cone));
3542     for (v = 0; v < nFV; ++v) {
3543       PetscCall(PetscFindInt(cone[v], numSubVertices, subVertices, &subVertex));
3544       subface[v] = firstSubVertex + subVertex;
3545     }
3546     PetscCall(DMPlexSetCone(subdm, newFacePoint, subface));
3547     PetscCall(DMPlexSetCone(subdm, subcell, &newFacePoint));
3548     PetscCall(DMPlexGetJoin(dm, nFV, cone, &numCells, &cells));
3549     /* Not true in parallel
3550     PetscCheck(numCells == 2,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive cells should separate two cells"); */
3551     for (p = 0; p < numCells; ++p) {
3552       PetscInt  negsubcell;
3553       PetscBool isHybrid;
3554 
3555       PetscCall(DMPlexCellIsHybrid_Internal(dm, cells[p], &isHybrid));
3556       if (isHybrid) continue;
3557       /* I know this is a crap search */
3558       for (negsubcell = 0; negsubcell < numSubCells; ++negsubcell) {
3559         if (subCells[negsubcell] == cells[p]) break;
3560       }
3561       PetscCheck(negsubcell != numSubCells, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not find negative face neighbor for cohesive cell %" PetscInt_FMT, cell);
3562       PetscCall(DMPlexSetCone(subdm, negsubcell, &newFacePoint));
3563     }
3564     PetscCall(DMPlexRestoreJoin(dm, nFV, cone, &numCells, &cells));
3565     ++newFacePoint;
3566   }
3567   PetscCall(DMRestoreWorkArray(subdm, maxConeSize, MPIU_INT, (void **)&subface));
3568   PetscCall(DMPlexSymmetrize(subdm));
3569   PetscCall(DMPlexStratify(subdm));
3570   /* Build coordinates */
3571   {
3572     PetscSection coordSection, subCoordSection;
3573     Vec          coordinates, subCoordinates;
3574     PetscScalar *coords, *subCoords;
3575     PetscInt     cdim, numComp, coordSize, v;
3576     const char  *name;
3577 
3578     PetscCall(DMGetCoordinateDim(dm, &cdim));
3579     PetscCall(DMGetCoordinateSection(dm, &coordSection));
3580     PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
3581     PetscCall(DMGetCoordinateSection(subdm, &subCoordSection));
3582     PetscCall(PetscSectionSetNumFields(subCoordSection, 1));
3583     PetscCall(PetscSectionGetFieldComponents(coordSection, 0, &numComp));
3584     PetscCall(PetscSectionSetFieldComponents(subCoordSection, 0, numComp));
3585     PetscCall(PetscSectionSetChart(subCoordSection, firstSubVertex, firstSubVertex + numSubVertices));
3586     for (v = 0; v < numSubVertices; ++v) {
3587       const PetscInt vertex    = subVertices[v];
3588       const PetscInt subvertex = firstSubVertex + v;
3589       PetscInt       dof;
3590 
3591       PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3592       PetscCall(PetscSectionSetDof(subCoordSection, subvertex, dof));
3593       PetscCall(PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof));
3594     }
3595     PetscCall(PetscSectionSetUp(subCoordSection));
3596     PetscCall(PetscSectionGetStorageSize(subCoordSection, &coordSize));
3597     PetscCall(VecCreate(PETSC_COMM_SELF, &subCoordinates));
3598     PetscCall(PetscObjectGetName((PetscObject)coordinates, &name));
3599     PetscCall(PetscObjectSetName((PetscObject)subCoordinates, name));
3600     PetscCall(VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE));
3601     PetscCall(VecSetBlockSize(subCoordinates, cdim));
3602     PetscCall(VecSetType(subCoordinates, VECSTANDARD));
3603     PetscCall(VecGetArray(coordinates, &coords));
3604     PetscCall(VecGetArray(subCoordinates, &subCoords));
3605     for (v = 0; v < numSubVertices; ++v) {
3606       const PetscInt vertex    = subVertices[v];
3607       const PetscInt subvertex = firstSubVertex + v;
3608       PetscInt       dof, off, sdof, soff, d;
3609 
3610       PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3611       PetscCall(PetscSectionGetOffset(coordSection, vertex, &off));
3612       PetscCall(PetscSectionGetDof(subCoordSection, subvertex, &sdof));
3613       PetscCall(PetscSectionGetOffset(subCoordSection, subvertex, &soff));
3614       PetscCheck(dof == sdof, comm, PETSC_ERR_PLIB, "Coordinate dimension %" PetscInt_FMT " on subvertex %" PetscInt_FMT ", vertex %" PetscInt_FMT " should be %" PetscInt_FMT, sdof, subvertex, vertex, dof);
3615       for (d = 0; d < dof; ++d) subCoords[soff + d] = coords[off + d];
3616     }
3617     PetscCall(VecRestoreArray(coordinates, &coords));
3618     PetscCall(VecRestoreArray(subCoordinates, &subCoords));
3619     PetscCall(DMSetCoordinatesLocal(subdm, subCoordinates));
3620     PetscCall(VecDestroy(&subCoordinates));
3621   }
3622   /* Build SF */
3623   CHKMEMQ;
3624   {
3625     PetscSF            sfPoint, sfPointSub;
3626     const PetscSFNode *remotePoints;
3627     PetscSFNode       *sremotePoints, *newLocalPoints, *newOwners;
3628     const PetscInt    *localPoints;
3629     PetscInt          *slocalPoints;
3630     PetscInt           numRoots, numLeaves, numSubRoots = numSubCells + numSubFaces + numSubVertices, numSubLeaves = 0, l, sl, ll, pStart, pEnd, p, vStart, vEnd;
3631     PetscMPIInt        rank;
3632 
3633     PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
3634     PetscCall(DMGetPointSF(dm, &sfPoint));
3635     PetscCall(DMGetPointSF(subdm, &sfPointSub));
3636     PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
3637     PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
3638     PetscCall(PetscSFGetGraph(sfPoint, &numRoots, &numLeaves, &localPoints, &remotePoints));
3639     if (numRoots >= 0) {
3640       /* Only vertices should be shared */
3641       PetscCall(PetscMalloc2(pEnd - pStart, &newLocalPoints, numRoots, &newOwners));
3642       for (p = 0; p < pEnd - pStart; ++p) {
3643         newLocalPoints[p].rank  = -2;
3644         newLocalPoints[p].index = -2;
3645       }
3646       /* Set subleaves */
3647       for (l = 0; l < numLeaves; ++l) {
3648         const PetscInt point    = localPoints[l];
3649         const PetscInt subPoint = DMPlexFilterPoint_Internal(point, firstSubVertex, numSubVertices, subVertices);
3650 
3651         PetscCheck(!(point < vStart) || !(point >= vEnd), PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Should not be mapping anything but vertices, %" PetscInt_FMT, point);
3652         if (subPoint < 0) continue;
3653         newLocalPoints[point - pStart].rank  = rank;
3654         newLocalPoints[point - pStart].index = subPoint;
3655         ++numSubLeaves;
3656       }
3657       /* Must put in owned subpoints */
3658       for (p = pStart; p < pEnd; ++p) {
3659         const PetscInt subPoint = DMPlexFilterPoint_Internal(p, firstSubVertex, numSubVertices, subVertices);
3660 
3661         if (subPoint < 0) {
3662           newOwners[p - pStart].rank  = -3;
3663           newOwners[p - pStart].index = -3;
3664         } else {
3665           newOwners[p - pStart].rank  = rank;
3666           newOwners[p - pStart].index = subPoint;
3667         }
3668       }
3669       PetscCall(PetscSFReduceBegin(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC));
3670       PetscCall(PetscSFReduceEnd(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC));
3671       PetscCall(PetscSFBcastBegin(sfPoint, MPIU_2INT, newOwners, newLocalPoints, MPI_REPLACE));
3672       PetscCall(PetscSFBcastEnd(sfPoint, MPIU_2INT, newOwners, newLocalPoints, MPI_REPLACE));
3673       PetscCall(PetscMalloc1(numSubLeaves, &slocalPoints));
3674       PetscCall(PetscMalloc1(numSubLeaves, &sremotePoints));
3675       for (l = 0, sl = 0, ll = 0; l < numLeaves; ++l) {
3676         const PetscInt point    = localPoints[l];
3677         const PetscInt subPoint = DMPlexFilterPoint_Internal(point, firstSubVertex, numSubVertices, subVertices);
3678 
3679         if (subPoint < 0) continue;
3680         if (newLocalPoints[point].rank == rank) {
3681           ++ll;
3682           continue;
3683         }
3684         slocalPoints[sl]        = subPoint;
3685         sremotePoints[sl].rank  = newLocalPoints[point].rank;
3686         sremotePoints[sl].index = newLocalPoints[point].index;
3687         PetscCheck(sremotePoints[sl].rank >= 0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote rank for local point %" PetscInt_FMT, point);
3688         PetscCheck(sremotePoints[sl].index >= 0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote subpoint for local point %" PetscInt_FMT, point);
3689         ++sl;
3690       }
3691       PetscCall(PetscFree2(newLocalPoints, newOwners));
3692       PetscCheck(sl + ll == numSubLeaves, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatch in number of subleaves %" PetscInt_FMT " + %" PetscInt_FMT " != %" PetscInt_FMT, sl, ll, numSubLeaves);
3693       PetscCall(PetscSFSetGraph(sfPointSub, numSubRoots, sl, slocalPoints, PETSC_OWN_POINTER, sremotePoints, PETSC_OWN_POINTER));
3694     }
3695   }
3696   CHKMEMQ;
3697   /* Cleanup */
3698   if (subvertexIS) PetscCall(ISRestoreIndices(subvertexIS, &subVertices));
3699   PetscCall(ISDestroy(&subvertexIS));
3700   PetscCall(PetscFree(subCells));
3701   PetscFunctionReturn(0);
3702 }
3703 
3704 static PetscErrorCode DMPlexCreateCohesiveSubmesh_Interpolated(DM dm, const char labelname[], PetscInt value, DM subdm) {
3705   DMLabel label = NULL;
3706 
3707   PetscFunctionBegin;
3708   if (labelname) PetscCall(DMGetLabel(dm, labelname, &label));
3709   PetscCall(DMPlexCreateSubmeshGeneric_Interpolated(dm, label, value, PETSC_FALSE, PETSC_TRUE, 1, subdm));
3710   PetscFunctionReturn(0);
3711 }
3712 
3713 /*@C
3714   DMPlexCreateCohesiveSubmesh - Extract from a mesh with cohesive cells the hypersurface defined by one face of the cells. Optionally, a Label can be given to restrict the cells.
3715 
3716   Input Parameters:
3717 + dm          - The original mesh
3718 . hasLagrange - The mesh has Lagrange unknowns in the cohesive cells
3719 . label       - A label name, or NULL
3720 - value  - A label value
3721 
3722   Output Parameter:
3723 . subdm - The surface mesh
3724 
3725   Note: This function produces a DMLabel mapping original points in the submesh to their depth. This can be obtained using DMPlexGetSubpointMap().
3726 
3727   Level: developer
3728 
3729 .seealso: `DMPlexGetSubpointMap()`, `DMPlexCreateSubmesh()`
3730 @*/
3731 PetscErrorCode DMPlexCreateCohesiveSubmesh(DM dm, PetscBool hasLagrange, const char label[], PetscInt value, DM *subdm) {
3732   PetscInt dim, cdim, depth;
3733 
3734   PetscFunctionBegin;
3735   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3736   PetscValidPointer(subdm, 5);
3737   PetscCall(DMGetDimension(dm, &dim));
3738   PetscCall(DMPlexGetDepth(dm, &depth));
3739   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), subdm));
3740   PetscCall(DMSetType(*subdm, DMPLEX));
3741   PetscCall(DMSetDimension(*subdm, dim - 1));
3742   PetscCall(DMGetCoordinateDim(dm, &cdim));
3743   PetscCall(DMSetCoordinateDim(*subdm, cdim));
3744   if (depth == dim) {
3745     PetscCall(DMPlexCreateCohesiveSubmesh_Interpolated(dm, label, value, *subdm));
3746   } else {
3747     PetscCall(DMPlexCreateCohesiveSubmesh_Uninterpolated(dm, hasLagrange, label, value, *subdm));
3748   }
3749   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_TRUE, *subdm));
3750   PetscFunctionReturn(0);
3751 }
3752 
3753 /*@
3754   DMPlexFilter - Extract a subset of mesh cells defined by a label as a separate mesh
3755 
3756   Input Parameters:
3757 + dm        - The original mesh
3758 . cellLabel - The DMLabel marking cells contained in the new mesh
3759 - value     - The label value to use
3760 
3761   Output Parameter:
3762 . subdm - The new mesh
3763 
3764   Note: This function produces a DMLabel mapping original points in the submesh to their depth. This can be obtained using DMPlexGetSubpointMap().
3765 
3766   Level: developer
3767 
3768 .seealso: `DMPlexGetSubpointMap()`, `DMGetLabel()`, `DMLabelSetValue()`
3769 @*/
3770 PetscErrorCode DMPlexFilter(DM dm, DMLabel cellLabel, PetscInt value, DM *subdm) {
3771   PetscInt dim;
3772 
3773   PetscFunctionBegin;
3774   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3775   PetscValidPointer(subdm, 4);
3776   PetscCall(DMGetDimension(dm, &dim));
3777   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), subdm));
3778   PetscCall(DMSetType(*subdm, DMPLEX));
3779   PetscCall(DMSetDimension(*subdm, dim));
3780   /* Extract submesh in place, could be empty on some procs, could have inconsistency if procs do not both extract a shared cell */
3781   PetscCall(DMPlexCreateSubmeshGeneric_Interpolated(dm, cellLabel, value, PETSC_FALSE, PETSC_FALSE, 0, *subdm));
3782   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_TRUE, *subdm));
3783   PetscFunctionReturn(0);
3784 }
3785 
3786 /*@
3787   DMPlexGetSubpointMap - Returns a DMLabel with point dimension as values
3788 
3789   Input Parameter:
3790 . dm - The submesh DM
3791 
3792   Output Parameter:
3793 . subpointMap - The DMLabel of all the points from the original mesh in this submesh, or NULL if this is not a submesh
3794 
3795   Level: developer
3796 
3797 .seealso: `DMPlexCreateSubmesh()`, `DMPlexGetSubpointIS()`
3798 @*/
3799 PetscErrorCode DMPlexGetSubpointMap(DM dm, DMLabel *subpointMap) {
3800   PetscFunctionBegin;
3801   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3802   PetscValidPointer(subpointMap, 2);
3803   *subpointMap = ((DM_Plex *)dm->data)->subpointMap;
3804   PetscFunctionReturn(0);
3805 }
3806 
3807 /*@
3808   DMPlexSetSubpointMap - Sets the DMLabel with point dimension as values
3809 
3810   Input Parameters:
3811 + dm - The submesh DM
3812 - subpointMap - The DMLabel of all the points from the original mesh in this submesh
3813 
3814   Note: Should normally not be called by the user, since it is set in DMPlexCreateSubmesh()
3815 
3816   Level: developer
3817 
3818 .seealso: `DMPlexCreateSubmesh()`, `DMPlexGetSubpointIS()`
3819 @*/
3820 PetscErrorCode DMPlexSetSubpointMap(DM dm, DMLabel subpointMap) {
3821   DM_Plex *mesh = (DM_Plex *)dm->data;
3822   DMLabel  tmp;
3823 
3824   PetscFunctionBegin;
3825   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3826   tmp               = mesh->subpointMap;
3827   mesh->subpointMap = subpointMap;
3828   PetscCall(PetscObjectReference((PetscObject)mesh->subpointMap));
3829   PetscCall(DMLabelDestroy(&tmp));
3830   PetscFunctionReturn(0);
3831 }
3832 
3833 static PetscErrorCode DMPlexCreateSubpointIS_Internal(DM dm, IS *subpointIS) {
3834   DMLabel  spmap;
3835   PetscInt depth, d;
3836 
3837   PetscFunctionBegin;
3838   PetscCall(DMPlexGetSubpointMap(dm, &spmap));
3839   PetscCall(DMPlexGetDepth(dm, &depth));
3840   if (spmap && depth >= 0) {
3841     DM_Plex  *mesh = (DM_Plex *)dm->data;
3842     PetscInt *points, *depths;
3843     PetscInt  pStart, pEnd, p, off;
3844 
3845     PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
3846     PetscCheck(!pStart, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Submeshes must start the point numbering at 0, not %" PetscInt_FMT, pStart);
3847     PetscCall(PetscMalloc1(pEnd, &points));
3848     PetscCall(DMGetWorkArray(dm, depth + 1, MPIU_INT, &depths));
3849     depths[0] = depth;
3850     depths[1] = 0;
3851     for (d = 2; d <= depth; ++d) depths[d] = depth + 1 - d;
3852     for (d = 0, off = 0; d <= depth; ++d) {
3853       const PetscInt dep = depths[d];
3854       PetscInt       depStart, depEnd, n;
3855 
3856       PetscCall(DMPlexGetDepthStratum(dm, dep, &depStart, &depEnd));
3857       PetscCall(DMLabelGetStratumSize(spmap, dep, &n));
3858       if (((d < 2) && (depth > 1)) || (d == 1)) { /* Only check vertices and cells for now since the map is broken for others */
3859         PetscCheck(n == depEnd - depStart, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "The number of mapped submesh points %" PetscInt_FMT " at depth %" PetscInt_FMT " should be %" PetscInt_FMT, n, dep, depEnd - depStart);
3860       } else {
3861         if (!n) {
3862           if (d == 0) {
3863             /* Missing cells */
3864             for (p = 0; p < depEnd - depStart; ++p, ++off) points[off] = -1;
3865           } else {
3866             /* Missing faces */
3867             for (p = 0; p < depEnd - depStart; ++p, ++off) points[off] = PETSC_MAX_INT;
3868           }
3869         }
3870       }
3871       if (n) {
3872         IS              is;
3873         const PetscInt *opoints;
3874 
3875         PetscCall(DMLabelGetStratumIS(spmap, dep, &is));
3876         PetscCall(ISGetIndices(is, &opoints));
3877         for (p = 0; p < n; ++p, ++off) points[off] = opoints[p];
3878         PetscCall(ISRestoreIndices(is, &opoints));
3879         PetscCall(ISDestroy(&is));
3880       }
3881     }
3882     PetscCall(DMRestoreWorkArray(dm, depth + 1, MPIU_INT, &depths));
3883     PetscCheck(off == pEnd, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "The number of mapped submesh points %" PetscInt_FMT " should be %" PetscInt_FMT, off, pEnd);
3884     PetscCall(ISCreateGeneral(PETSC_COMM_SELF, pEnd, points, PETSC_OWN_POINTER, subpointIS));
3885     PetscCall(PetscObjectStateGet((PetscObject)spmap, &mesh->subpointState));
3886   }
3887   PetscFunctionReturn(0);
3888 }
3889 
3890 /*@
3891   DMPlexGetSubpointIS - Returns an IS covering the entire subdm chart with the original points as data
3892 
3893   Input Parameter:
3894 . dm - The submesh DM
3895 
3896   Output Parameter:
3897 . subpointIS - The IS of all the points from the original mesh in this submesh, or NULL if this is not a submesh
3898 
3899   Note: This IS is guaranteed to be sorted by the construction of the submesh
3900 
3901   Level: developer
3902 
3903 .seealso: `DMPlexCreateSubmesh()`, `DMPlexGetSubpointMap()`
3904 @*/
3905 PetscErrorCode DMPlexGetSubpointIS(DM dm, IS *subpointIS) {
3906   DM_Plex         *mesh = (DM_Plex *)dm->data;
3907   DMLabel          spmap;
3908   PetscObjectState state;
3909 
3910   PetscFunctionBegin;
3911   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3912   PetscValidPointer(subpointIS, 2);
3913   PetscCall(DMPlexGetSubpointMap(dm, &spmap));
3914   PetscCall(PetscObjectStateGet((PetscObject)spmap, &state));
3915   if (state != mesh->subpointState || !mesh->subpointIS) PetscCall(DMPlexCreateSubpointIS_Internal(dm, &mesh->subpointIS));
3916   *subpointIS = mesh->subpointIS;
3917   PetscFunctionReturn(0);
3918 }
3919 
3920 /*@
3921   DMGetEnclosureRelation - Get the relationship between dmA and dmB
3922 
3923   Input Parameters:
3924 + dmA - The first DM
3925 - dmB - The second DM
3926 
3927   Output Parameter:
3928 . rel - The relation of dmA to dmB
3929 
3930   Level: intermediate
3931 
3932 .seealso: `DMGetEnclosurePoint()`
3933 @*/
3934 PetscErrorCode DMGetEnclosureRelation(DM dmA, DM dmB, DMEnclosureType *rel) {
3935   DM       plexA, plexB, sdm;
3936   DMLabel  spmap;
3937   PetscInt pStartA, pEndA, pStartB, pEndB, NpA, NpB;
3938 
3939   PetscFunctionBegin;
3940   PetscValidPointer(rel, 3);
3941   *rel = DM_ENC_NONE;
3942   if (!dmA || !dmB) PetscFunctionReturn(0);
3943   PetscValidHeaderSpecific(dmA, DM_CLASSID, 1);
3944   PetscValidHeaderSpecific(dmB, DM_CLASSID, 2);
3945   if (dmA == dmB) {
3946     *rel = DM_ENC_EQUALITY;
3947     PetscFunctionReturn(0);
3948   }
3949   PetscCall(DMConvert(dmA, DMPLEX, &plexA));
3950   PetscCall(DMConvert(dmB, DMPLEX, &plexB));
3951   PetscCall(DMPlexGetChart(plexA, &pStartA, &pEndA));
3952   PetscCall(DMPlexGetChart(plexB, &pStartB, &pEndB));
3953   /* Assumption 1: subDMs have smaller charts than the DMs that they originate from
3954     - The degenerate case of a subdomain which includes all of the domain on some process can be treated as equality */
3955   if ((pStartA == pStartB) && (pEndA == pEndB)) {
3956     *rel = DM_ENC_EQUALITY;
3957     goto end;
3958   }
3959   NpA = pEndA - pStartA;
3960   NpB = pEndB - pStartB;
3961   if (NpA == NpB) goto end;
3962   sdm = NpA > NpB ? plexB : plexA; /* The other is the original, enclosing dm */
3963   PetscCall(DMPlexGetSubpointMap(sdm, &spmap));
3964   if (!spmap) goto end;
3965   /* TODO Check the space mapped to by subpointMap is same size as dm */
3966   if (NpA > NpB) {
3967     *rel = DM_ENC_SUPERMESH;
3968   } else {
3969     *rel = DM_ENC_SUBMESH;
3970   }
3971 end:
3972   PetscCall(DMDestroy(&plexA));
3973   PetscCall(DMDestroy(&plexB));
3974   PetscFunctionReturn(0);
3975 }
3976 
3977 /*@
3978   DMGetEnclosurePoint - Get the point pA in dmA which corresponds to the point pB in dmB
3979 
3980   Input Parameters:
3981 + dmA   - The first DM
3982 . dmB   - The second DM
3983 . etype - The type of enclosure relation that dmA has to dmB
3984 - pB    - A point of dmB
3985 
3986   Output Parameter:
3987 . pA    - The corresponding point of dmA
3988 
3989   Level: intermediate
3990 
3991 .seealso: `DMGetEnclosureRelation()`
3992 @*/
3993 PetscErrorCode DMGetEnclosurePoint(DM dmA, DM dmB, DMEnclosureType etype, PetscInt pB, PetscInt *pA) {
3994   DM              sdm;
3995   IS              subpointIS;
3996   const PetscInt *subpoints;
3997   PetscInt        numSubpoints;
3998 
3999   PetscFunctionBegin;
4000   /* TODO Cache the IS, making it look like an index */
4001   switch (etype) {
4002   case DM_ENC_SUPERMESH:
4003     sdm = dmB;
4004     PetscCall(DMPlexGetSubpointIS(sdm, &subpointIS));
4005     PetscCall(ISGetIndices(subpointIS, &subpoints));
4006     *pA = subpoints[pB];
4007     PetscCall(ISRestoreIndices(subpointIS, &subpoints));
4008     break;
4009   case DM_ENC_SUBMESH:
4010     sdm = dmA;
4011     PetscCall(DMPlexGetSubpointIS(sdm, &subpointIS));
4012     PetscCall(ISGetLocalSize(subpointIS, &numSubpoints));
4013     PetscCall(ISGetIndices(subpointIS, &subpoints));
4014     PetscCall(PetscFindInt(pB, numSubpoints, subpoints, pA));
4015     if (*pA < 0) {
4016       PetscCall(DMViewFromOptions(dmA, NULL, "-dm_enc_A_view"));
4017       PetscCall(DMViewFromOptions(dmB, NULL, "-dm_enc_B_view"));
4018       SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Point %" PetscInt_FMT " not found in submesh", pB);
4019     }
4020     PetscCall(ISRestoreIndices(subpointIS, &subpoints));
4021     break;
4022   case DM_ENC_EQUALITY:
4023   case DM_ENC_NONE: *pA = pB; break;
4024   case DM_ENC_UNKNOWN: {
4025     DMEnclosureType enc;
4026 
4027     PetscCall(DMGetEnclosureRelation(dmA, dmB, &enc));
4028     PetscCall(DMGetEnclosurePoint(dmA, dmB, enc, pB, pA));
4029   } break;
4030   default: SETERRQ(PetscObjectComm((PetscObject)dmA), PETSC_ERR_ARG_OUTOFRANGE, "Invalid enclosure type %d", (int)etype);
4031   }
4032   PetscFunctionReturn(0);
4033 }
4034