xref: /petsc/src/dm/impls/plex/plexsubmesh.c (revision caf9e14d8ca99dc20b2119107fcf65b01f7b29cf)
1 #include <petsc/private/dmpleximpl.h>    /*I      "petscdmplex.h"    I*/
2 #include <petsc/private/dmlabelimpl.h>   /*I      "petscdmlabel.h"   I*/
3 #include <petscsf.h>
4 
5 static PetscErrorCode DMPlexCellIsHybrid_Internal(DM dm, PetscInt p, PetscBool *isHybrid)
6 {
7   DMPolytopeType ct;
8 
9   PetscFunctionBegin;
10   PetscCall(DMPlexGetCellType(dm, p, &ct));
11   switch (ct) {
12     case DM_POLYTOPE_POINT_PRISM_TENSOR:
13     case DM_POLYTOPE_SEG_PRISM_TENSOR:
14     case DM_POLYTOPE_TRI_PRISM_TENSOR:
15     case DM_POLYTOPE_QUAD_PRISM_TENSOR:
16       *isHybrid = PETSC_TRUE;
17     default: *isHybrid = PETSC_FALSE;
18   }
19   PetscFunctionReturn(0);
20 }
21 
22 static PetscErrorCode DMPlexGetTensorPrismBounds_Internal(DM dm, PetscInt dim, PetscInt *cStart, PetscInt *cEnd)
23 {
24   DMLabel        ctLabel;
25 
26   PetscFunctionBegin;
27   if (cStart) *cStart = -1;
28   if (cEnd)   *cEnd   = -1;
29   PetscCall(DMPlexGetCellTypeLabel(dm, &ctLabel));
30   switch (dim) {
31     case 1: PetscCall(DMLabelGetStratumBounds(ctLabel, DM_POLYTOPE_POINT_PRISM_TENSOR, cStart, cEnd));break;
32     case 2: PetscCall(DMLabelGetStratumBounds(ctLabel, DM_POLYTOPE_SEG_PRISM_TENSOR, cStart, cEnd));break;
33     case 3:
34       PetscCall(DMLabelGetStratumBounds(ctLabel, DM_POLYTOPE_TRI_PRISM_TENSOR, cStart, cEnd));
35       if (*cStart < 0) PetscCall(DMLabelGetStratumBounds(ctLabel, DM_POLYTOPE_QUAD_PRISM_TENSOR, cStart, cEnd));
36       break;
37     default: PetscFunctionReturn(0);
38   }
39   PetscFunctionReturn(0);
40 }
41 
42 static PetscErrorCode DMPlexMarkBoundaryFaces_Internal(DM dm, PetscInt val, PetscInt cellHeight, DMLabel label)
43 {
44   PetscInt       fStart, fEnd, f;
45 
46   PetscFunctionBegin;
47   PetscCall(DMPlexGetHeightStratum(dm, cellHeight+1, &fStart, &fEnd));
48   for (f = fStart; f < fEnd; ++f) {
49     PetscInt supportSize;
50 
51     PetscCall(DMPlexGetSupportSize(dm, f, &supportSize));
52     if (supportSize == 1) {
53       if (val < 0) {
54         PetscInt *closure = NULL;
55         PetscInt  clSize, cl, cval;
56 
57         PetscCall(DMPlexGetTransitiveClosure(dm, f, PETSC_TRUE, &clSize, &closure));
58         for (cl = 0; cl < clSize*2; cl += 2) {
59           PetscCall(DMLabelGetValue(label, closure[cl], &cval));
60           if (cval < 0) continue;
61           PetscCall(DMLabelSetValue(label, f, cval));
62           break;
63         }
64         if (cl == clSize*2) PetscCall(DMLabelSetValue(label, f, 1));
65         PetscCall(DMPlexRestoreTransitiveClosure(dm, f, PETSC_TRUE, &clSize, &closure));
66       } else {
67         PetscCall(DMLabelSetValue(label, f, val));
68       }
69     }
70   }
71   PetscFunctionReturn(0);
72 }
73 
74 /*@
75   DMPlexMarkBoundaryFaces - Mark all faces on the boundary
76 
77   Not Collective
78 
79   Input Parameters:
80 + dm - The original DM
81 - val - The marker value, or PETSC_DETERMINE to use some value in the closure (or 1 if none are found)
82 
83   Output Parameter:
84 . label - The DMLabel marking boundary faces with the given value
85 
86   Level: developer
87 
88 .seealso: `DMLabelCreate()`, `DMCreateLabel()`
89 @*/
90 PetscErrorCode DMPlexMarkBoundaryFaces(DM dm, PetscInt val, DMLabel label)
91 {
92   DMPlexInterpolatedFlag  flg;
93 
94   PetscFunctionBegin;
95   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
96   PetscCall(DMPlexIsInterpolated(dm, &flg));
97   PetscCheck(flg == DMPLEX_INTERPOLATED_FULL,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM is not fully interpolated on this rank");
98   PetscCall(DMPlexMarkBoundaryFaces_Internal(dm, val, 0, label));
99   PetscFunctionReturn(0);
100 }
101 
102 static PetscErrorCode DMPlexLabelComplete_Internal(DM dm, DMLabel label, PetscBool completeCells)
103 {
104   IS              valueIS;
105   PetscSF         sfPoint;
106   const PetscInt *values;
107   PetscInt        numValues, v, cStart, cEnd, nroots;
108 
109   PetscFunctionBegin;
110   PetscCall(DMLabelGetNumValues(label, &numValues));
111   PetscCall(DMLabelGetValueIS(label, &valueIS));
112   PetscCall(DMPlexGetHeightStratum(dm,0,&cStart,&cEnd));
113   PetscCall(ISGetIndices(valueIS, &values));
114   for (v = 0; v < numValues; ++v) {
115     IS              pointIS;
116     const PetscInt *points;
117     PetscInt        numPoints, p;
118 
119     PetscCall(DMLabelGetStratumSize(label, values[v], &numPoints));
120     PetscCall(DMLabelGetStratumIS(label, values[v], &pointIS));
121     PetscCall(ISGetIndices(pointIS, &points));
122     for (p = 0; p < numPoints; ++p) {
123       PetscInt  q = points[p];
124       PetscInt *closure = NULL;
125       PetscInt  closureSize, c;
126 
127       if (cStart <= q && q < cEnd && !completeCells) { /* skip cells */
128         continue;
129       }
130       PetscCall(DMPlexGetTransitiveClosure(dm, q, PETSC_TRUE, &closureSize, &closure));
131       for (c = 0; c < closureSize*2; c += 2) {
132         PetscCall(DMLabelSetValue(label, closure[c], values[v]));
133       }
134       PetscCall(DMPlexRestoreTransitiveClosure(dm, q, PETSC_TRUE, &closureSize, &closure));
135     }
136     PetscCall(ISRestoreIndices(pointIS, &points));
137     PetscCall(ISDestroy(&pointIS));
138   }
139   PetscCall(ISRestoreIndices(valueIS, &values));
140   PetscCall(ISDestroy(&valueIS));
141   PetscCall(DMGetPointSF(dm, &sfPoint));
142   PetscCall(PetscSFGetGraph(sfPoint, &nroots, NULL, NULL, NULL));
143   if (nroots >= 0) {
144     DMLabel         lblRoots, lblLeaves;
145     IS              valueIS, pointIS;
146     const PetscInt *values;
147     PetscInt        numValues, v;
148 
149     /* Pull point contributions from remote leaves into local roots */
150     PetscCall(DMLabelGather(label, sfPoint, &lblLeaves));
151     PetscCall(DMLabelGetValueIS(lblLeaves, &valueIS));
152     PetscCall(ISGetLocalSize(valueIS, &numValues));
153     PetscCall(ISGetIndices(valueIS, &values));
154     for (v = 0; v < numValues; ++v) {
155       const PetscInt value = values[v];
156 
157       PetscCall(DMLabelGetStratumIS(lblLeaves, value, &pointIS));
158       PetscCall(DMLabelInsertIS(label, pointIS, value));
159       PetscCall(ISDestroy(&pointIS));
160     }
161     PetscCall(ISRestoreIndices(valueIS, &values));
162     PetscCall(ISDestroy(&valueIS));
163     PetscCall(DMLabelDestroy(&lblLeaves));
164     /* Push point contributions from roots into remote leaves */
165     PetscCall(DMLabelDistribute(label, sfPoint, &lblRoots));
166     PetscCall(DMLabelGetValueIS(lblRoots, &valueIS));
167     PetscCall(ISGetLocalSize(valueIS, &numValues));
168     PetscCall(ISGetIndices(valueIS, &values));
169     for (v = 0; v < numValues; ++v) {
170       const PetscInt value = values[v];
171 
172       PetscCall(DMLabelGetStratumIS(lblRoots, value, &pointIS));
173       PetscCall(DMLabelInsertIS(label, pointIS, value));
174       PetscCall(ISDestroy(&pointIS));
175     }
176     PetscCall(ISRestoreIndices(valueIS, &values));
177     PetscCall(ISDestroy(&valueIS));
178     PetscCall(DMLabelDestroy(&lblRoots));
179   }
180   PetscFunctionReturn(0);
181 }
182 
183 /*@
184   DMPlexLabelComplete - Starting with a label marking points on a surface, we add the transitive closure to the surface
185 
186   Input Parameters:
187 + dm - The DM
188 - label - A DMLabel marking the surface points
189 
190   Output Parameter:
191 . label - A DMLabel marking all surface points in the transitive closure
192 
193   Level: developer
194 
195 .seealso: `DMPlexLabelCohesiveComplete()`
196 @*/
197 PetscErrorCode DMPlexLabelComplete(DM dm, DMLabel label)
198 {
199   PetscFunctionBegin;
200   PetscCall(DMPlexLabelComplete_Internal(dm, label, PETSC_TRUE));
201   PetscFunctionReturn(0);
202 }
203 
204 /*@
205   DMPlexLabelAddCells - Starting with a label marking points on a surface, we add a cell for each point
206 
207   Input Parameters:
208 + dm - The DM
209 - label - A DMLabel marking the surface points
210 
211   Output Parameter:
212 . label - A DMLabel incorporating cells
213 
214   Level: developer
215 
216   Note: The cells allow FEM boundary conditions to be applied using the cell geometry
217 
218 .seealso: `DMPlexLabelAddFaceCells()`, `DMPlexLabelComplete()`, `DMPlexLabelCohesiveComplete()`
219 @*/
220 PetscErrorCode DMPlexLabelAddCells(DM dm, DMLabel label)
221 {
222   IS              valueIS;
223   const PetscInt *values;
224   PetscInt        numValues, v, cStart, cEnd;
225 
226   PetscFunctionBegin;
227   PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd));
228   PetscCall(DMLabelGetNumValues(label, &numValues));
229   PetscCall(DMLabelGetValueIS(label, &valueIS));
230   PetscCall(ISGetIndices(valueIS, &values));
231   for (v = 0; v < numValues; ++v) {
232     IS              pointIS;
233     const PetscInt *points;
234     PetscInt        numPoints, p;
235 
236     PetscCall(DMLabelGetStratumSize(label, values[v], &numPoints));
237     PetscCall(DMLabelGetStratumIS(label, values[v], &pointIS));
238     PetscCall(ISGetIndices(pointIS, &points));
239     for (p = 0; p < numPoints; ++p) {
240       PetscInt *closure = NULL;
241       PetscInt  closureSize, cl;
242 
243       PetscCall(DMPlexGetTransitiveClosure(dm, points[p], PETSC_FALSE, &closureSize, &closure));
244       for (cl = closureSize-1; cl > 0; --cl) {
245         const PetscInt cell = closure[cl*2];
246         if ((cell >= cStart) && (cell < cEnd)) {PetscCall(DMLabelSetValue(label, cell, values[v])); break;}
247       }
248       PetscCall(DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_FALSE, &closureSize, &closure));
249     }
250     PetscCall(ISRestoreIndices(pointIS, &points));
251     PetscCall(ISDestroy(&pointIS));
252   }
253   PetscCall(ISRestoreIndices(valueIS, &values));
254   PetscCall(ISDestroy(&valueIS));
255   PetscFunctionReturn(0);
256 }
257 
258 /*@
259   DMPlexLabelAddFaceCells - Starting with a label marking faces on a surface, we add a cell for each face
260 
261   Input Parameters:
262 + dm - The DM
263 - label - A DMLabel marking the surface points
264 
265   Output Parameter:
266 . label - A DMLabel incorporating cells
267 
268   Level: developer
269 
270   Note: The cells allow FEM boundary conditions to be applied using the cell geometry
271 
272 .seealso: `DMPlexLabelAddCells()`, `DMPlexLabelComplete()`, `DMPlexLabelCohesiveComplete()`
273 @*/
274 PetscErrorCode DMPlexLabelAddFaceCells(DM dm, DMLabel label)
275 {
276   IS              valueIS;
277   const PetscInt *values;
278   PetscInt        numValues, v, cStart, cEnd, fStart, fEnd;
279 
280   PetscFunctionBegin;
281   PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd));
282   PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
283   PetscCall(DMLabelGetNumValues(label, &numValues));
284   PetscCall(DMLabelGetValueIS(label, &valueIS));
285   PetscCall(ISGetIndices(valueIS, &values));
286   for (v = 0; v < numValues; ++v) {
287     IS              pointIS;
288     const PetscInt *points;
289     PetscInt        numPoints, p;
290 
291     PetscCall(DMLabelGetStratumSize(label, values[v], &numPoints));
292     PetscCall(DMLabelGetStratumIS(label, values[v], &pointIS));
293     PetscCall(ISGetIndices(pointIS, &points));
294     for (p = 0; p < numPoints; ++p) {
295       const PetscInt face = points[p];
296       PetscInt      *closure = NULL;
297       PetscInt       closureSize, cl;
298 
299       if ((face < fStart) || (face >= fEnd)) continue;
300       PetscCall(DMPlexGetTransitiveClosure(dm, face, PETSC_FALSE, &closureSize, &closure));
301       for (cl = closureSize-1; cl > 0; --cl) {
302         const PetscInt cell = closure[cl*2];
303         if ((cell >= cStart) && (cell < cEnd)) {PetscCall(DMLabelSetValue(label, cell, values[v])); break;}
304       }
305       PetscCall(DMPlexRestoreTransitiveClosure(dm, face, PETSC_FALSE, &closureSize, &closure));
306     }
307     PetscCall(ISRestoreIndices(pointIS, &points));
308     PetscCall(ISDestroy(&pointIS));
309   }
310   PetscCall(ISRestoreIndices(valueIS, &values));
311   PetscCall(ISDestroy(&valueIS));
312   PetscFunctionReturn(0);
313 }
314 
315 /*@
316   DMPlexLabelClearCells - Remove cells from a label
317 
318   Input Parameters:
319 + dm - The DM
320 - label - A DMLabel marking surface points and their adjacent cells
321 
322   Output Parameter:
323 . label - A DMLabel without cells
324 
325   Level: developer
326 
327   Note: This undoes DMPlexLabelAddCells() or DMPlexLabelAddFaceCells()
328 
329 .seealso: `DMPlexLabelComplete()`, `DMPlexLabelCohesiveComplete()`, `DMPlexLabelAddCells()`
330 @*/
331 PetscErrorCode DMPlexLabelClearCells(DM dm, DMLabel label)
332 {
333   IS              valueIS;
334   const PetscInt *values;
335   PetscInt        numValues, v, cStart, cEnd;
336 
337   PetscFunctionBegin;
338   PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd));
339   PetscCall(DMLabelGetNumValues(label, &numValues));
340   PetscCall(DMLabelGetValueIS(label, &valueIS));
341   PetscCall(ISGetIndices(valueIS, &values));
342   for (v = 0; v < numValues; ++v) {
343     IS              pointIS;
344     const PetscInt *points;
345     PetscInt        numPoints, p;
346 
347     PetscCall(DMLabelGetStratumSize(label, values[v], &numPoints));
348     PetscCall(DMLabelGetStratumIS(label, values[v], &pointIS));
349     PetscCall(ISGetIndices(pointIS, &points));
350     for (p = 0; p < numPoints; ++p) {
351       PetscInt point = points[p];
352 
353       if (point >= cStart && point < cEnd) {
354         PetscCall(DMLabelClearValue(label,point,values[v]));
355       }
356     }
357     PetscCall(ISRestoreIndices(pointIS, &points));
358     PetscCall(ISDestroy(&pointIS));
359   }
360   PetscCall(ISRestoreIndices(valueIS, &values));
361   PetscCall(ISDestroy(&valueIS));
362   PetscFunctionReturn(0);
363 }
364 
365 /* take (oldEnd, added) pairs, ordered by height and convert them to (oldstart, newstart) pairs, ordered by ascending
366  * index (skipping first, which is (0,0)) */
367 static inline PetscErrorCode DMPlexShiftPointSetUp_Internal(PetscInt depth, PetscInt depthShift[])
368 {
369   PetscInt d, off = 0;
370 
371   PetscFunctionBegin;
372   /* sort by (oldend): yes this is an O(n^2) sort, we expect depth <= 3 */
373   for (d = 0; d < depth; d++) {
374     PetscInt firstd = d;
375     PetscInt firstStart = depthShift[2*d];
376     PetscInt e;
377 
378     for (e = d+1; e <= depth; e++) {
379       if (depthShift[2*e] < firstStart) {
380         firstd = e;
381         firstStart = depthShift[2*d];
382       }
383     }
384     if (firstd != d) {
385       PetscInt swap[2];
386 
387       e = firstd;
388       swap[0] = depthShift[2*d];
389       swap[1] = depthShift[2*d+1];
390       depthShift[2*d]   = depthShift[2*e];
391       depthShift[2*d+1] = depthShift[2*e+1];
392       depthShift[2*e]   = swap[0];
393       depthShift[2*e+1] = swap[1];
394     }
395   }
396   /* convert (oldstart, added) to (oldstart, newstart) */
397   for (d = 0; d <= depth; d++) {
398     off += depthShift[2*d+1];
399     depthShift[2*d+1] = depthShift[2*d] + off;
400   }
401   PetscFunctionReturn(0);
402 }
403 
404 /* depthShift is a list of (old, new) pairs */
405 static inline PetscInt DMPlexShiftPoint_Internal(PetscInt p, PetscInt depth, PetscInt depthShift[])
406 {
407   PetscInt d;
408   PetscInt newOff = 0;
409 
410   for (d = 0; d <= depth; d++) {
411     if (p < depthShift[2*d]) return p + newOff;
412     else newOff = depthShift[2*d+1] - depthShift[2*d];
413   }
414   return p + newOff;
415 }
416 
417 /* depthShift is a list of (old, new) pairs */
418 static inline PetscInt DMPlexShiftPointInverse_Internal(PetscInt p, PetscInt depth, PetscInt depthShift[])
419 {
420   PetscInt d;
421   PetscInt newOff = 0;
422 
423   for (d = 0; d <= depth; d++) {
424     if (p < depthShift[2*d+1]) return p + newOff;
425     else newOff = depthShift[2*d] - depthShift[2*d+1];
426   }
427   return p + newOff;
428 }
429 
430 static PetscErrorCode DMPlexShiftSizes_Internal(DM dm, PetscInt depthShift[], DM dmNew)
431 {
432   PetscInt       depth = 0, d, pStart, pEnd, p;
433   DMLabel        depthLabel;
434 
435   PetscFunctionBegin;
436   PetscCall(DMPlexGetDepth(dm, &depth));
437   if (depth < 0) PetscFunctionReturn(0);
438   /* Step 1: Expand chart */
439   PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
440   pEnd = DMPlexShiftPoint_Internal(pEnd,depth,depthShift);
441   PetscCall(DMPlexSetChart(dmNew, pStart, pEnd));
442   PetscCall(DMCreateLabel(dmNew,"depth"));
443   PetscCall(DMPlexGetDepthLabel(dmNew,&depthLabel));
444   PetscCall(DMCreateLabel(dmNew, "celltype"));
445   /* Step 2: Set cone and support sizes */
446   for (d = 0; d <= depth; ++d) {
447     PetscInt pStartNew, pEndNew;
448     IS pIS;
449 
450     PetscCall(DMPlexGetDepthStratum(dm, d, &pStart, &pEnd));
451     pStartNew = DMPlexShiftPoint_Internal(pStart, depth, depthShift);
452     pEndNew = DMPlexShiftPoint_Internal(pEnd, depth, depthShift);
453     PetscCall(ISCreateStride(PETSC_COMM_SELF, pEndNew - pStartNew, pStartNew, 1, &pIS));
454     PetscCall(DMLabelSetStratumIS(depthLabel, d, pIS));
455     PetscCall(ISDestroy(&pIS));
456     for (p = pStart; p < pEnd; ++p) {
457       PetscInt       newp = DMPlexShiftPoint_Internal(p, depth, depthShift);
458       PetscInt       size;
459       DMPolytopeType ct;
460 
461       PetscCall(DMPlexGetConeSize(dm, p, &size));
462       PetscCall(DMPlexSetConeSize(dmNew, newp, size));
463       PetscCall(DMPlexGetSupportSize(dm, p, &size));
464       PetscCall(DMPlexSetSupportSize(dmNew, newp, size));
465       PetscCall(DMPlexGetCellType(dm, p, &ct));
466       PetscCall(DMPlexSetCellType(dmNew, newp, ct));
467     }
468   }
469   PetscFunctionReturn(0);
470 }
471 
472 static PetscErrorCode DMPlexShiftPoints_Internal(DM dm, PetscInt depthShift[], DM dmNew)
473 {
474   PetscInt      *newpoints;
475   PetscInt       depth = 0, maxConeSize, maxSupportSize, maxConeSizeNew, maxSupportSizeNew, pStart, pEnd, p;
476 
477   PetscFunctionBegin;
478   PetscCall(DMPlexGetDepth(dm, &depth));
479   if (depth < 0) PetscFunctionReturn(0);
480   PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize));
481   PetscCall(DMPlexGetMaxSizes(dmNew, &maxConeSizeNew, &maxSupportSizeNew));
482   PetscCall(PetscMalloc1(PetscMax(PetscMax(maxConeSize, maxSupportSize), PetscMax(maxConeSizeNew, maxSupportSizeNew)),&newpoints));
483   /* Step 5: Set cones and supports */
484   PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
485   for (p = pStart; p < pEnd; ++p) {
486     const PetscInt *points = NULL, *orientations = NULL;
487     PetscInt        size,sizeNew, i, newp = DMPlexShiftPoint_Internal(p, depth, depthShift);
488 
489     PetscCall(DMPlexGetConeSize(dm, p, &size));
490     PetscCall(DMPlexGetCone(dm, p, &points));
491     PetscCall(DMPlexGetConeOrientation(dm, p, &orientations));
492     for (i = 0; i < size; ++i) {
493       newpoints[i] = DMPlexShiftPoint_Internal(points[i], depth, depthShift);
494     }
495     PetscCall(DMPlexSetCone(dmNew, newp, newpoints));
496     PetscCall(DMPlexSetConeOrientation(dmNew, newp, orientations));
497     PetscCall(DMPlexGetSupportSize(dm, p, &size));
498     PetscCall(DMPlexGetSupportSize(dmNew, newp, &sizeNew));
499     PetscCall(DMPlexGetSupport(dm, p, &points));
500     for (i = 0; i < size; ++i) {
501       newpoints[i] = DMPlexShiftPoint_Internal(points[i], depth, depthShift);
502     }
503     for (i = size; i < sizeNew; ++i) newpoints[i] = 0;
504     PetscCall(DMPlexSetSupport(dmNew, newp, newpoints));
505   }
506   PetscCall(PetscFree(newpoints));
507   PetscFunctionReturn(0);
508 }
509 
510 static PetscErrorCode DMPlexShiftCoordinates_Internal(DM dm, PetscInt depthShift[], DM dmNew)
511 {
512   PetscSection   coordSection, newCoordSection;
513   Vec            coordinates, newCoordinates;
514   PetscScalar   *coords, *newCoords;
515   PetscInt       coordSize, sStart, sEnd;
516   PetscInt       dim, depth = 0, cStart, cEnd, cStartNew, cEndNew, c, vStart, vEnd, vStartNew, vEndNew, v;
517   PetscBool      hasCells;
518 
519   PetscFunctionBegin;
520   PetscCall(DMGetCoordinateDim(dm, &dim));
521   PetscCall(DMSetCoordinateDim(dmNew, dim));
522   PetscCall(DMPlexGetDepth(dm, &depth));
523   /* Step 8: Convert coordinates */
524   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
525   PetscCall(DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd));
526   PetscCall(DMPlexGetDepthStratum(dmNew, 0, &vStartNew, &vEndNew));
527   PetscCall(DMPlexGetHeightStratum(dmNew, 0, &cStartNew, &cEndNew));
528   PetscCall(DMGetCoordinateSection(dm, &coordSection));
529   PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dm), &newCoordSection));
530   PetscCall(PetscSectionSetNumFields(newCoordSection, 1));
531   PetscCall(PetscSectionSetFieldComponents(newCoordSection, 0, dim));
532   PetscCall(PetscSectionGetChart(coordSection, &sStart, &sEnd));
533   hasCells = sStart == cStart ? PETSC_TRUE : PETSC_FALSE;
534   PetscCall(PetscSectionSetChart(newCoordSection, hasCells ? cStartNew : vStartNew, vEndNew));
535   if (hasCells) {
536     for (c = cStart; c < cEnd; ++c) {
537       PetscInt cNew = DMPlexShiftPoint_Internal(c, depth, depthShift), dof;
538 
539       PetscCall(PetscSectionGetDof(coordSection, c, &dof));
540       PetscCall(PetscSectionSetDof(newCoordSection, cNew, dof));
541       PetscCall(PetscSectionSetFieldDof(newCoordSection, cNew, 0, dof));
542     }
543   }
544   for (v = vStartNew; v < vEndNew; ++v) {
545     PetscCall(PetscSectionSetDof(newCoordSection, v, dim));
546     PetscCall(PetscSectionSetFieldDof(newCoordSection, v, 0, dim));
547   }
548   PetscCall(PetscSectionSetUp(newCoordSection));
549   PetscCall(DMSetCoordinateSection(dmNew, PETSC_DETERMINE, newCoordSection));
550   PetscCall(PetscSectionGetStorageSize(newCoordSection, &coordSize));
551   PetscCall(VecCreate(PETSC_COMM_SELF, &newCoordinates));
552   PetscCall(PetscObjectSetName((PetscObject) newCoordinates, "coordinates"));
553   PetscCall(VecSetSizes(newCoordinates, coordSize, PETSC_DETERMINE));
554   PetscCall(VecSetBlockSize(newCoordinates, dim));
555   PetscCall(VecSetType(newCoordinates,VECSTANDARD));
556   PetscCall(DMSetCoordinatesLocal(dmNew, newCoordinates));
557   PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
558   PetscCall(VecGetArray(coordinates, &coords));
559   PetscCall(VecGetArray(newCoordinates, &newCoords));
560   if (hasCells) {
561     for (c = cStart; c < cEnd; ++c) {
562       PetscInt cNew = DMPlexShiftPoint_Internal(c, depth, depthShift), dof, off, noff, d;
563 
564       PetscCall(PetscSectionGetDof(coordSection, c, &dof));
565       PetscCall(PetscSectionGetOffset(coordSection, c, &off));
566       PetscCall(PetscSectionGetOffset(newCoordSection, cNew, &noff));
567       for (d = 0; d < dof; ++d) newCoords[noff+d] = coords[off+d];
568     }
569   }
570   for (v = vStart; v < vEnd; ++v) {
571     PetscInt dof, off, noff, d;
572 
573     PetscCall(PetscSectionGetDof(coordSection, v, &dof));
574     PetscCall(PetscSectionGetOffset(coordSection, v, &off));
575     PetscCall(PetscSectionGetOffset(newCoordSection, DMPlexShiftPoint_Internal(v, depth, depthShift), &noff));
576     for (d = 0; d < dof; ++d) newCoords[noff+d] = coords[off+d];
577   }
578   PetscCall(VecRestoreArray(coordinates, &coords));
579   PetscCall(VecRestoreArray(newCoordinates, &newCoords));
580   PetscCall(VecDestroy(&newCoordinates));
581   PetscCall(PetscSectionDestroy(&newCoordSection));
582   PetscFunctionReturn(0);
583 }
584 
585 static PetscErrorCode DMPlexShiftSF_Single(DM dm, PetscInt depthShift[], PetscSF sf, PetscSF sfNew)
586 {
587   const PetscSFNode *remotePoints;
588   PetscSFNode       *gremotePoints;
589   const PetscInt    *localPoints;
590   PetscInt          *glocalPoints, *newLocation, *newRemoteLocation;
591   PetscInt           numRoots, numLeaves, l, pStart, pEnd, depth = 0, totShift = 0;
592 
593   PetscFunctionBegin;
594   PetscCall(DMPlexGetDepth(dm, &depth));
595   PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
596   PetscCall(PetscSFGetGraph(sf, &numRoots, &numLeaves, &localPoints, &remotePoints));
597   totShift = DMPlexShiftPoint_Internal(pEnd, depth, depthShift) - pEnd;
598   if (numRoots >= 0) {
599     PetscCall(PetscMalloc2(numRoots, &newLocation, pEnd-pStart, &newRemoteLocation));
600     for (l = 0; l < numRoots; ++l) newLocation[l] = DMPlexShiftPoint_Internal(l, depth, depthShift);
601     PetscCall(PetscSFBcastBegin(sf, MPIU_INT, newLocation, newRemoteLocation, MPI_REPLACE));
602     PetscCall(PetscSFBcastEnd(sf, MPIU_INT, newLocation, newRemoteLocation, MPI_REPLACE));
603     PetscCall(PetscMalloc1(numLeaves, &glocalPoints));
604     PetscCall(PetscMalloc1(numLeaves, &gremotePoints));
605     for (l = 0; l < numLeaves; ++l) {
606       glocalPoints[l]        = DMPlexShiftPoint_Internal(localPoints[l], depth, depthShift);
607       gremotePoints[l].rank  = remotePoints[l].rank;
608       gremotePoints[l].index = newRemoteLocation[localPoints[l]];
609     }
610     PetscCall(PetscFree2(newLocation, newRemoteLocation));
611     PetscCall(PetscSFSetGraph(sfNew, numRoots + totShift, numLeaves, glocalPoints, PETSC_OWN_POINTER, gremotePoints, PETSC_OWN_POINTER));
612   }
613   PetscFunctionReturn(0);
614 }
615 
616 static PetscErrorCode DMPlexShiftSF_Internal(DM dm, PetscInt depthShift[], DM dmNew)
617 {
618   PetscSF        sfPoint, sfPointNew;
619   PetscBool      useNatural;
620 
621   PetscFunctionBegin;
622   /* Step 9: Convert pointSF */
623   PetscCall(DMGetPointSF(dm, &sfPoint));
624   PetscCall(DMGetPointSF(dmNew, &sfPointNew));
625   PetscCall(DMPlexShiftSF_Single(dm, depthShift, sfPoint, sfPointNew));
626   /* Step 9b: Convert naturalSF */
627   PetscCall(DMGetUseNatural(dm, &useNatural));
628   if (useNatural) {
629     PetscSF sfNat, sfNatNew;
630 
631     PetscCall(DMSetUseNatural(dmNew, useNatural));
632     PetscCall(DMGetNaturalSF(dm, &sfNat));
633     PetscCall(DMGetNaturalSF(dmNew, &sfNatNew));
634     PetscCall(DMPlexShiftSF_Single(dm, depthShift, sfNat, sfNatNew));
635   }
636   PetscFunctionReturn(0);
637 }
638 
639 static PetscErrorCode DMPlexShiftLabels_Internal(DM dm, PetscInt depthShift[], DM dmNew)
640 {
641   PetscInt           depth = 0, numLabels, l;
642 
643   PetscFunctionBegin;
644   PetscCall(DMPlexGetDepth(dm, &depth));
645   /* Step 10: Convert labels */
646   PetscCall(DMGetNumLabels(dm, &numLabels));
647   for (l = 0; l < numLabels; ++l) {
648     DMLabel         label, newlabel;
649     const char     *lname;
650     PetscBool       isDepth, isDim;
651     IS              valueIS;
652     const PetscInt *values;
653     PetscInt        numValues, val;
654 
655     PetscCall(DMGetLabelName(dm, l, &lname));
656     PetscCall(PetscStrcmp(lname, "depth", &isDepth));
657     if (isDepth) continue;
658     PetscCall(PetscStrcmp(lname, "dim", &isDim));
659     if (isDim) continue;
660     PetscCall(DMCreateLabel(dmNew, lname));
661     PetscCall(DMGetLabel(dm, lname, &label));
662     PetscCall(DMGetLabel(dmNew, lname, &newlabel));
663     PetscCall(DMLabelGetDefaultValue(label,&val));
664     PetscCall(DMLabelSetDefaultValue(newlabel,val));
665     PetscCall(DMLabelGetValueIS(label, &valueIS));
666     PetscCall(ISGetLocalSize(valueIS, &numValues));
667     PetscCall(ISGetIndices(valueIS, &values));
668     for (val = 0; val < numValues; ++val) {
669       IS              pointIS;
670       const PetscInt *points;
671       PetscInt        numPoints, p;
672 
673       PetscCall(DMLabelGetStratumIS(label, values[val], &pointIS));
674       PetscCall(ISGetLocalSize(pointIS, &numPoints));
675       PetscCall(ISGetIndices(pointIS, &points));
676       for (p = 0; p < numPoints; ++p) {
677         const PetscInt newpoint = DMPlexShiftPoint_Internal(points[p], depth, depthShift);
678 
679         PetscCall(DMLabelSetValue(newlabel, newpoint, values[val]));
680       }
681       PetscCall(ISRestoreIndices(pointIS, &points));
682       PetscCall(ISDestroy(&pointIS));
683     }
684     PetscCall(ISRestoreIndices(valueIS, &values));
685     PetscCall(ISDestroy(&valueIS));
686   }
687   PetscFunctionReturn(0);
688 }
689 
690 static PetscErrorCode DMPlexCreateVTKLabel_Internal(DM dm, PetscBool createGhostLabel, DM dmNew)
691 {
692   PetscSF            sfPoint;
693   DMLabel            vtkLabel, ghostLabel = NULL;
694   const PetscSFNode *leafRemote;
695   const PetscInt    *leafLocal;
696   PetscInt           cellHeight, cStart, cEnd, c, fStart, fEnd, f, numLeaves, l;
697   PetscMPIInt        rank;
698 
699   PetscFunctionBegin;
700    /* Step 11: Make label for output (vtk) and to mark ghost points (ghost) */
701   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
702   PetscCall(DMGetPointSF(dm, &sfPoint));
703   PetscCall(DMPlexGetVTKCellHeight(dmNew, &cellHeight));
704   PetscCall(DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd));
705   PetscCall(PetscSFGetGraph(sfPoint, NULL, &numLeaves, &leafLocal, &leafRemote));
706   PetscCall(DMCreateLabel(dmNew, "vtk"));
707   PetscCall(DMGetLabel(dmNew, "vtk", &vtkLabel));
708   if (createGhostLabel) {
709     PetscCall(DMCreateLabel(dmNew, "ghost"));
710     PetscCall(DMGetLabel(dmNew, "ghost", &ghostLabel));
711   }
712   for (l = 0, c = cStart; l < numLeaves && c < cEnd; ++l, ++c) {
713     for (; c < leafLocal[l] && c < cEnd; ++c) {
714       PetscCall(DMLabelSetValue(vtkLabel, c, 1));
715     }
716     if (leafLocal[l] >= cEnd) break;
717     if (leafRemote[l].rank == rank) {
718       PetscCall(DMLabelSetValue(vtkLabel, c, 1));
719     } else if (ghostLabel) {
720       PetscCall(DMLabelSetValue(ghostLabel, c, 2));
721     }
722   }
723   for (; c < cEnd; ++c) {
724     PetscCall(DMLabelSetValue(vtkLabel, c, 1));
725   }
726   if (ghostLabel) {
727     PetscCall(DMPlexGetHeightStratum(dmNew, 1, &fStart, &fEnd));
728     for (f = fStart; f < fEnd; ++f) {
729       PetscInt numCells;
730 
731       PetscCall(DMPlexGetSupportSize(dmNew, f, &numCells));
732       if (numCells < 2) {
733         PetscCall(DMLabelSetValue(ghostLabel, f, 1));
734       } else {
735         const PetscInt *cells = NULL;
736         PetscInt        vA, vB;
737 
738         PetscCall(DMPlexGetSupport(dmNew, f, &cells));
739         PetscCall(DMLabelGetValue(vtkLabel, cells[0], &vA));
740         PetscCall(DMLabelGetValue(vtkLabel, cells[1], &vB));
741         if (vA != 1 && vB != 1) PetscCall(DMLabelSetValue(ghostLabel, f, 1));
742       }
743     }
744   }
745   PetscFunctionReturn(0);
746 }
747 
748 static PetscErrorCode DMPlexShiftTree_Internal(DM dm, PetscInt depthShift[], DM dmNew)
749 {
750   DM             refTree;
751   PetscSection   pSec;
752   PetscInt       *parents, *childIDs;
753 
754   PetscFunctionBegin;
755   PetscCall(DMPlexGetReferenceTree(dm,&refTree));
756   PetscCall(DMPlexSetReferenceTree(dmNew,refTree));
757   PetscCall(DMPlexGetTree(dm,&pSec,&parents,&childIDs,NULL,NULL));
758   if (pSec) {
759     PetscInt p, pStart, pEnd, *parentsShifted, pStartShifted, pEndShifted, depth;
760     PetscInt *childIDsShifted;
761     PetscSection pSecShifted;
762 
763     PetscCall(PetscSectionGetChart(pSec,&pStart,&pEnd));
764     PetscCall(DMPlexGetDepth(dm,&depth));
765     pStartShifted = DMPlexShiftPoint_Internal(pStart,depth,depthShift);
766     pEndShifted   = DMPlexShiftPoint_Internal(pEnd,depth,depthShift);
767     PetscCall(PetscMalloc2(pEndShifted - pStartShifted,&parentsShifted,pEndShifted-pStartShifted,&childIDsShifted));
768     PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dmNew),&pSecShifted));
769     PetscCall(PetscSectionSetChart(pSecShifted,pStartShifted,pEndShifted));
770     for (p = pStartShifted; p < pEndShifted; p++) {
771       /* start off assuming no children */
772       PetscCall(PetscSectionSetDof(pSecShifted,p,0));
773     }
774     for (p = pStart; p < pEnd; p++) {
775       PetscInt dof;
776       PetscInt pNew = DMPlexShiftPoint_Internal(p,depth,depthShift);
777 
778       PetscCall(PetscSectionGetDof(pSec,p,&dof));
779       PetscCall(PetscSectionSetDof(pSecShifted,pNew,dof));
780     }
781     PetscCall(PetscSectionSetUp(pSecShifted));
782     for (p = pStart; p < pEnd; p++) {
783       PetscInt dof;
784       PetscInt pNew = DMPlexShiftPoint_Internal(p,depth,depthShift);
785 
786       PetscCall(PetscSectionGetDof(pSec,p,&dof));
787       if (dof) {
788         PetscInt off, offNew;
789 
790         PetscCall(PetscSectionGetOffset(pSec,p,&off));
791         PetscCall(PetscSectionGetOffset(pSecShifted,pNew,&offNew));
792         parentsShifted[offNew] = DMPlexShiftPoint_Internal(parents[off],depth,depthShift);
793         childIDsShifted[offNew] = childIDs[off];
794       }
795     }
796     PetscCall(DMPlexSetTree(dmNew,pSecShifted,parentsShifted,childIDsShifted));
797     PetscCall(PetscFree2(parentsShifted,childIDsShifted));
798     PetscCall(PetscSectionDestroy(&pSecShifted));
799   }
800   PetscFunctionReturn(0);
801 }
802 
803 static PetscErrorCode DMPlexConstructGhostCells_Internal(DM dm, DMLabel label, PetscInt *numGhostCells, DM gdm)
804 {
805   PetscSF               sf;
806   IS                    valueIS;
807   const PetscInt       *values, *leaves;
808   PetscInt             *depthShift;
809   PetscInt              d, depth = 0, nleaves, loc, Ng, numFS, fs, fStart, fEnd, ghostCell, cEnd, c;
810   PetscBool             isper;
811   const PetscReal      *maxCell, *L;
812   const DMBoundaryType *bd;
813 
814   PetscFunctionBegin;
815   PetscCall(DMGetPointSF(dm, &sf));
816   PetscCall(PetscSFGetGraph(sf, NULL, &nleaves, &leaves, NULL));
817   nleaves = PetscMax(0, nleaves);
818   PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
819   /* Count ghost cells */
820   PetscCall(DMLabelGetValueIS(label, &valueIS));
821   PetscCall(ISGetLocalSize(valueIS, &numFS));
822   PetscCall(ISGetIndices(valueIS, &values));
823   Ng   = 0;
824   for (fs = 0; fs < numFS; ++fs) {
825     IS              faceIS;
826     const PetscInt *faces;
827     PetscInt        numFaces, f, numBdFaces = 0;
828 
829     PetscCall(DMLabelGetStratumIS(label, values[fs], &faceIS));
830     PetscCall(ISGetLocalSize(faceIS, &numFaces));
831     PetscCall(ISGetIndices(faceIS, &faces));
832     for (f = 0; f < numFaces; ++f) {
833       PetscInt numChildren;
834 
835       PetscCall(PetscFindInt(faces[f], nleaves, leaves, &loc));
836       PetscCall(DMPlexGetTreeChildren(dm,faces[f],&numChildren,NULL));
837       /* non-local and ancestors points don't get to register ghosts */
838       if (loc >= 0 || numChildren) continue;
839       if ((faces[f] >= fStart) && (faces[f] < fEnd)) ++numBdFaces;
840     }
841     Ng += numBdFaces;
842     PetscCall(ISRestoreIndices(faceIS, &faces));
843     PetscCall(ISDestroy(&faceIS));
844   }
845   PetscCall(DMPlexGetDepth(dm, &depth));
846   PetscCall(PetscMalloc1(2*(depth+1), &depthShift));
847   for (d = 0; d <= depth; d++) {
848     PetscInt dEnd;
849 
850     PetscCall(DMPlexGetDepthStratum(dm,d,NULL,&dEnd));
851     depthShift[2*d]   = dEnd;
852     depthShift[2*d+1] = 0;
853   }
854   if (depth >= 0) depthShift[2*depth+1] = Ng;
855   PetscCall(DMPlexShiftPointSetUp_Internal(depth,depthShift));
856   PetscCall(DMPlexShiftSizes_Internal(dm, depthShift, gdm));
857   /* Step 3: Set cone/support sizes for new points */
858   PetscCall(DMPlexGetHeightStratum(dm, 0, NULL, &cEnd));
859   for (c = cEnd; c < cEnd + Ng; ++c) {
860     PetscCall(DMPlexSetConeSize(gdm, c, 1));
861   }
862   for (fs = 0; fs < numFS; ++fs) {
863     IS              faceIS;
864     const PetscInt *faces;
865     PetscInt        numFaces, f;
866 
867     PetscCall(DMLabelGetStratumIS(label, values[fs], &faceIS));
868     PetscCall(ISGetLocalSize(faceIS, &numFaces));
869     PetscCall(ISGetIndices(faceIS, &faces));
870     for (f = 0; f < numFaces; ++f) {
871       PetscInt size, numChildren;
872 
873       PetscCall(PetscFindInt(faces[f], nleaves, leaves, &loc));
874       PetscCall(DMPlexGetTreeChildren(dm,faces[f],&numChildren,NULL));
875       if (loc >= 0 || numChildren) continue;
876       if ((faces[f] < fStart) || (faces[f] >= fEnd)) continue;
877       PetscCall(DMPlexGetSupportSize(dm, faces[f], &size));
878       PetscCheck(size == 1,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM has boundary face %" PetscInt_FMT " with %" PetscInt_FMT " support cells", faces[f], size);
879       PetscCall(DMPlexSetSupportSize(gdm, faces[f] + Ng, 2));
880     }
881     PetscCall(ISRestoreIndices(faceIS, &faces));
882     PetscCall(ISDestroy(&faceIS));
883   }
884   /* Step 4: Setup ghosted DM */
885   PetscCall(DMSetUp(gdm));
886   PetscCall(DMPlexShiftPoints_Internal(dm, depthShift, gdm));
887   /* Step 6: Set cones and supports for new points */
888   ghostCell = cEnd;
889   for (fs = 0; fs < numFS; ++fs) {
890     IS              faceIS;
891     const PetscInt *faces;
892     PetscInt        numFaces, f;
893 
894     PetscCall(DMLabelGetStratumIS(label, values[fs], &faceIS));
895     PetscCall(ISGetLocalSize(faceIS, &numFaces));
896     PetscCall(ISGetIndices(faceIS, &faces));
897     for (f = 0; f < numFaces; ++f) {
898       PetscInt newFace = faces[f] + Ng, numChildren;
899 
900       PetscCall(PetscFindInt(faces[f], nleaves, leaves, &loc));
901       PetscCall(DMPlexGetTreeChildren(dm,faces[f],&numChildren,NULL));
902       if (loc >= 0 || numChildren) continue;
903       if ((faces[f] < fStart) || (faces[f] >= fEnd)) continue;
904       PetscCall(DMPlexSetCone(gdm, ghostCell, &newFace));
905       PetscCall(DMPlexInsertSupport(gdm, newFace, 1, ghostCell));
906       ++ghostCell;
907     }
908     PetscCall(ISRestoreIndices(faceIS, &faces));
909     PetscCall(ISDestroy(&faceIS));
910   }
911   PetscCall(ISRestoreIndices(valueIS, &values));
912   PetscCall(ISDestroy(&valueIS));
913   PetscCall(DMPlexShiftCoordinates_Internal(dm, depthShift, gdm));
914   PetscCall(DMPlexShiftSF_Internal(dm, depthShift, gdm));
915   PetscCall(DMPlexShiftLabels_Internal(dm, depthShift, gdm));
916   PetscCall(DMPlexCreateVTKLabel_Internal(dm, PETSC_TRUE, gdm));
917   PetscCall(DMPlexShiftTree_Internal(dm, depthShift, gdm));
918   PetscCall(PetscFree(depthShift));
919   for (c = cEnd; c < cEnd + Ng; ++c) {
920     PetscCall(DMPlexSetCellType(gdm, c, DM_POLYTOPE_FV_GHOST));
921   }
922   /* Step 7: Periodicity */
923   PetscCall(DMGetPeriodicity(dm, &isper, &maxCell, &L, &bd));
924   PetscCall(DMSetPeriodicity(gdm, isper, maxCell,  L,  bd));
925   if (numGhostCells) *numGhostCells = Ng;
926   PetscFunctionReturn(0);
927 }
928 
929 /*@C
930   DMPlexConstructGhostCells - Construct ghost cells which connect to every boundary face
931 
932   Collective on dm
933 
934   Input Parameters:
935 + dm - The original DM
936 - labelName - The label specifying the boundary faces, or "Face Sets" if this is NULL
937 
938   Output Parameters:
939 + numGhostCells - The number of ghost cells added to the DM
940 - dmGhosted - The new DM
941 
942   Note: If no label exists of that name, one will be created marking all boundary faces
943 
944   Level: developer
945 
946 .seealso: `DMCreate()`
947 @*/
948 PetscErrorCode DMPlexConstructGhostCells(DM dm, const char labelName[], PetscInt *numGhostCells, DM *dmGhosted)
949 {
950   DM             gdm;
951   DMLabel        label;
952   const char    *name = labelName ? labelName : "Face Sets";
953   PetscInt       dim, Ng = 0;
954   PetscBool      useCone, useClosure;
955 
956   PetscFunctionBegin;
957   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
958   if (numGhostCells) PetscValidIntPointer(numGhostCells, 3);
959   PetscValidPointer(dmGhosted, 4);
960   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &gdm));
961   PetscCall(DMSetType(gdm, DMPLEX));
962   PetscCall(DMGetDimension(dm, &dim));
963   PetscCall(DMSetDimension(gdm, dim));
964   PetscCall(DMGetBasicAdjacency(dm, &useCone, &useClosure));
965   PetscCall(DMSetBasicAdjacency(gdm, useCone,  useClosure));
966   PetscCall(DMGetLabel(dm, name, &label));
967   if (!label) {
968     /* Get label for boundary faces */
969     PetscCall(DMCreateLabel(dm, name));
970     PetscCall(DMGetLabel(dm, name, &label));
971     PetscCall(DMPlexMarkBoundaryFaces(dm, 1, label));
972   }
973   PetscCall(DMPlexConstructGhostCells_Internal(dm, label, &Ng, gdm));
974   PetscCall(DMCopyDisc(dm, gdm));
975   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, gdm));
976   gdm->setfromoptionscalled = dm->setfromoptionscalled;
977   if (numGhostCells) *numGhostCells = Ng;
978   *dmGhosted = gdm;
979   PetscFunctionReturn(0);
980 }
981 
982 static PetscErrorCode DivideCells_Private(DM dm, DMLabel label, DMPlexPointQueue queue)
983 {
984   PetscInt dim, d, shift = 100, *pStart, *pEnd;
985 
986   PetscFunctionBegin;
987   PetscCall(DMGetDimension(dm, &dim));
988   PetscCall(PetscMalloc2(dim, &pStart, dim, &pEnd));
989   for (d = 0; d < dim; ++d) {
990     PetscCall(DMPlexGetDepthStratum(dm, d, &pStart[d], &pEnd[d]));
991   }
992   while (!DMPlexPointQueueEmpty(queue)) {
993     PetscInt  cell    = -1;
994     PetscInt *closure = NULL;
995     PetscInt  closureSize, cl, cval;
996 
997     PetscCall(DMPlexPointQueueDequeue(queue, &cell));
998     PetscCall(DMLabelGetValue(label, cell, &cval));
999     PetscCall(DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
1000     /* Mark points in the cell closure that touch the fault */
1001     for (d = 0; d < dim; ++d) {
1002       for (cl = 0; cl < closureSize*2; cl += 2) {
1003         const PetscInt clp = closure[cl];
1004         PetscInt       clval;
1005 
1006         if ((clp < pStart[d]) || (clp >= pEnd[d])) continue;
1007         PetscCall(DMLabelGetValue(label, clp, &clval));
1008         if (clval == -1) {
1009           const PetscInt *cone;
1010           PetscInt        coneSize, c;
1011 
1012           /* If a cone point touches the fault, then this point touches the fault */
1013           PetscCall(DMPlexGetCone(dm, clp, &cone));
1014           PetscCall(DMPlexGetConeSize(dm, clp, &coneSize));
1015           for (c = 0; c < coneSize; ++c) {
1016             PetscInt cpval;
1017 
1018             PetscCall(DMLabelGetValue(label, cone[c], &cpval));
1019             if (cpval != -1) {
1020               PetscInt dep;
1021 
1022               PetscCall(DMPlexGetPointDepth(dm, clp, &dep));
1023               clval = cval < 0 ? -(shift+dep) : shift+dep;
1024               PetscCall(DMLabelSetValue(label, clp, clval));
1025               break;
1026             }
1027           }
1028         }
1029         /* Mark neighbor cells through marked faces (these cells must also touch the fault) */
1030         if (d == dim-1 && clval != -1) {
1031           const PetscInt *support;
1032           PetscInt        supportSize, s, nval;
1033 
1034           PetscCall(DMPlexGetSupport(dm, clp, &support));
1035           PetscCall(DMPlexGetSupportSize(dm, clp, &supportSize));
1036           for (s = 0; s < supportSize; ++s) {
1037             PetscCall(DMLabelGetValue(label, support[s], &nval));
1038             if (nval == -1) {
1039               PetscCall(DMLabelSetValue(label, support[s], clval < 0 ? clval-1 : clval+1));
1040               PetscCall(DMPlexPointQueueEnqueue(queue, support[s]));
1041             }
1042           }
1043         }
1044       }
1045     }
1046     PetscCall(DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
1047   }
1048   PetscCall(PetscFree2(pStart, pEnd));
1049   PetscFunctionReturn(0);
1050 }
1051 
1052 typedef struct {
1053   DM               dm;
1054   DMPlexPointQueue queue;
1055 } PointDivision;
1056 
1057 static PetscErrorCode divideCell(DMLabel label, PetscInt p, PetscInt val, void *ctx)
1058 {
1059   PointDivision  *div  = (PointDivision *) ctx;
1060   PetscInt        cval = val < 0 ? val-1 : val+1;
1061   const PetscInt *support;
1062   PetscInt        supportSize, s;
1063 
1064   PetscFunctionBegin;
1065   PetscCall(DMPlexGetSupport(div->dm, p, &support));
1066   PetscCall(DMPlexGetSupportSize(div->dm, p, &supportSize));
1067   for (s = 0; s < supportSize; ++s) {
1068     PetscCall(DMLabelSetValue(label, support[s], cval));
1069     PetscCall(DMPlexPointQueueEnqueue(div->queue, support[s]));
1070   }
1071   PetscFunctionReturn(0);
1072 }
1073 
1074 /* Mark cells by label propagation */
1075 static PetscErrorCode DMPlexLabelFaultHalo(DM dm, DMLabel faultLabel)
1076 {
1077   DMPlexPointQueue queue = NULL;
1078   PointDivision    div;
1079   PetscSF          pointSF;
1080   IS               pointIS;
1081   const PetscInt  *points;
1082   PetscBool        empty;
1083   PetscInt         dim, shift = 100, n, i;
1084 
1085   PetscFunctionBegin;
1086   PetscCall(DMGetDimension(dm, &dim));
1087   PetscCall(DMPlexPointQueueCreate(1024, &queue));
1088   div.dm    = dm;
1089   div.queue = queue;
1090   /* Enqueue cells on fault */
1091   PetscCall(DMLabelGetStratumIS(faultLabel, shift+dim, &pointIS));
1092   if (pointIS) {
1093     PetscCall(ISGetLocalSize(pointIS, &n));
1094     PetscCall(ISGetIndices(pointIS, &points));
1095     for (i = 0; i < n; ++i) {PetscCall(DMPlexPointQueueEnqueue(queue, points[i]));}
1096     PetscCall(ISRestoreIndices(pointIS, &points));
1097     PetscCall(ISDestroy(&pointIS));
1098   }
1099   PetscCall(DMLabelGetStratumIS(faultLabel, -(shift+dim), &pointIS));
1100   if (pointIS) {
1101     PetscCall(ISGetLocalSize(pointIS, &n));
1102     PetscCall(ISGetIndices(pointIS, &points));
1103     for (i = 0; i < n; ++i) {PetscCall(DMPlexPointQueueEnqueue(queue, points[i]));}
1104     PetscCall(ISRestoreIndices(pointIS, &points));
1105     PetscCall(ISDestroy(&pointIS));
1106   }
1107 
1108   PetscCall(DMGetPointSF(dm, &pointSF));
1109   PetscCall(DMLabelPropagateBegin(faultLabel, pointSF));
1110   /* While edge queue is not empty: */
1111   PetscCall(DMPlexPointQueueEmptyCollective((PetscObject) dm, queue, &empty));
1112   while (!empty) {
1113     PetscCall(DivideCells_Private(dm, faultLabel, queue));
1114     PetscCall(DMLabelPropagatePush(faultLabel, pointSF, divideCell, &div));
1115     PetscCall(DMPlexPointQueueEmptyCollective((PetscObject) dm, queue, &empty));
1116   }
1117   PetscCall(DMLabelPropagateEnd(faultLabel, pointSF));
1118   PetscCall(DMPlexPointQueueDestroy(&queue));
1119   PetscFunctionReturn(0);
1120 }
1121 
1122 /*
1123   We are adding three kinds of points here:
1124     Replicated:     Copies of points which exist in the mesh, such as vertices identified across a fault
1125     Non-replicated: Points which exist on the fault, but are not replicated
1126     Ghost:          These are shared fault faces which are not owned by this process. These do not produce hybrid cells and do not replicate
1127     Hybrid:         Entirely new points, such as cohesive cells
1128 
1129   When creating subsequent cohesive cells, we shift the old hybrid cells to the end of the numbering at
1130   each depth so that the new split/hybrid points can be inserted as a block.
1131 */
1132 static PetscErrorCode DMPlexConstructCohesiveCells_Internal(DM dm, DMLabel label, DMLabel splitLabel, DM sdm)
1133 {
1134   MPI_Comm         comm;
1135   IS               valueIS;
1136   PetscInt         numSP = 0;          /* The number of depths for which we have replicated points */
1137   const PetscInt  *values;             /* List of depths for which we have replicated points */
1138   IS              *splitIS;
1139   IS              *unsplitIS;
1140   IS               ghostIS;
1141   PetscInt        *numSplitPoints;     /* The number of replicated points at each depth */
1142   PetscInt        *numUnsplitPoints;   /* The number of non-replicated points at each depth which still give rise to hybrid points */
1143   PetscInt        *numHybridPoints;    /* The number of new hybrid points at each depth */
1144   PetscInt        *numHybridPointsOld; /* The number of existing hybrid points at each depth */
1145   PetscInt         numGhostPoints;     /* The number of unowned, shared fault faces */
1146   const PetscInt **splitPoints;        /* Replicated points for each depth */
1147   const PetscInt **unsplitPoints;      /* Non-replicated points for each depth */
1148   const PetscInt  *ghostPoints;        /* Ghost fault faces */
1149   PetscSection     coordSection;
1150   Vec              coordinates;
1151   PetscScalar     *coords;
1152   PetscInt        *depthMax;           /* The first hybrid point at each depth in the original mesh */
1153   PetscInt        *depthEnd;           /* The point limit at each depth in the original mesh */
1154   PetscInt        *depthShift;         /* Number of replicated+hybrid points at each depth */
1155   PetscInt        *pMaxNew;            /* The first replicated point at each depth in the new mesh, hybrids come after this */
1156   PetscInt        *coneNew, *coneONew, *supportNew;
1157   PetscInt         shift = 100, shift2 = 200, depth = 0, dep, dim, d, sp, maxConeSize, maxSupportSize, maxConeSizeNew, maxSupportSizeNew, numLabels, vStart, vEnd, pEnd, p, v;
1158 
1159   PetscFunctionBegin;
1160   PetscCall(PetscObjectGetComm((PetscObject)dm,&comm));
1161   PetscCall(DMGetDimension(dm, &dim));
1162   PetscCall(DMPlexGetDepth(dm, &depth));
1163   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
1164   /* We do not want this label automatically computed, instead we compute it here */
1165   PetscCall(DMCreateLabel(sdm, "celltype"));
1166   /* Count split points and add cohesive cells */
1167   PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize));
1168   PetscCall(PetscMalloc5(depth+1,&depthMax,depth+1,&depthEnd,2*(depth+1),&depthShift,depth+1,&pMaxNew,depth+1,&numHybridPointsOld));
1169   PetscCall(PetscMalloc7(depth+1,&splitIS,depth+1,&unsplitIS,depth+1,&numSplitPoints,depth+1,&numUnsplitPoints,depth+1,&numHybridPoints,depth+1,&splitPoints,depth+1,&unsplitPoints));
1170   for (d = 0; d <= depth; ++d) {
1171     PetscCall(DMPlexGetDepthStratum(dm, d, NULL, &pMaxNew[d]));
1172     PetscCall(DMPlexGetTensorPrismBounds_Internal(dm, d, &depthMax[d], NULL));
1173     depthEnd[d]           = pMaxNew[d];
1174     depthMax[d]           = depthMax[d] < 0 ? depthEnd[d] : depthMax[d];
1175     numSplitPoints[d]     = 0;
1176     numUnsplitPoints[d]   = 0;
1177     numHybridPoints[d]    = 0;
1178     numHybridPointsOld[d] = depthMax[d] < 0 ? 0 : depthEnd[d] - depthMax[d];
1179     splitPoints[d]        = NULL;
1180     unsplitPoints[d]      = NULL;
1181     splitIS[d]            = NULL;
1182     unsplitIS[d]          = NULL;
1183     /* we are shifting the existing hybrid points with the stratum behind them, so
1184      * the split comes at the end of the normal points, i.e., at depthMax[d] */
1185     depthShift[2*d]       = depthMax[d];
1186     depthShift[2*d+1]     = 0;
1187   }
1188   numGhostPoints = 0;
1189   ghostPoints    = NULL;
1190   if (label) {
1191     PetscCall(DMLabelGetValueIS(label, &valueIS));
1192     PetscCall(ISGetLocalSize(valueIS, &numSP));
1193     PetscCall(ISGetIndices(valueIS, &values));
1194   }
1195   for (sp = 0; sp < numSP; ++sp) {
1196     const PetscInt dep = values[sp];
1197 
1198     if ((dep < 0) || (dep > depth)) continue;
1199     PetscCall(DMLabelGetStratumIS(label, dep, &splitIS[dep]));
1200     if (splitIS[dep]) {
1201       PetscCall(ISGetLocalSize(splitIS[dep], &numSplitPoints[dep]));
1202       PetscCall(ISGetIndices(splitIS[dep], &splitPoints[dep]));
1203     }
1204     PetscCall(DMLabelGetStratumIS(label, shift2+dep, &unsplitIS[dep]));
1205     if (unsplitIS[dep]) {
1206       PetscCall(ISGetLocalSize(unsplitIS[dep], &numUnsplitPoints[dep]));
1207       PetscCall(ISGetIndices(unsplitIS[dep], &unsplitPoints[dep]));
1208     }
1209   }
1210   PetscCall(DMLabelGetStratumIS(label, shift2+dim-1, &ghostIS));
1211   if (ghostIS) {
1212     PetscCall(ISGetLocalSize(ghostIS, &numGhostPoints));
1213     PetscCall(ISGetIndices(ghostIS, &ghostPoints));
1214   }
1215   /* Calculate number of hybrid points */
1216   for (d = 1; d <= depth; ++d) numHybridPoints[d]     = numSplitPoints[d-1] + numUnsplitPoints[d-1]; /* There is a hybrid cell/face/edge for every split face/edge/vertex   */
1217   for (d = 0; d <= depth; ++d) depthShift[2*d+1]      = numSplitPoints[d] + numHybridPoints[d];
1218   PetscCall(DMPlexShiftPointSetUp_Internal(depth,depthShift));
1219   /* the end of the points in this stratum that come before the new points:
1220    * shifting pMaxNew[d] gets the new start of the next stratum, then count back the old hybrid points and the newly
1221    * added points */
1222   for (d = 0; d <= depth; ++d) pMaxNew[d]             = DMPlexShiftPoint_Internal(pMaxNew[d],depth,depthShift) - (numHybridPointsOld[d] + numSplitPoints[d] + numHybridPoints[d]);
1223   PetscCall(DMPlexShiftSizes_Internal(dm, depthShift, sdm));
1224   /* Step 3: Set cone/support sizes for new points */
1225   for (dep = 0; dep <= depth; ++dep) {
1226     for (p = 0; p < numSplitPoints[dep]; ++p) {
1227       const PetscInt  oldp   = splitPoints[dep][p];
1228       const PetscInt  newp   = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*oldp + depthOffset[dep]*/;
1229       const PetscInt  splitp = p    + pMaxNew[dep];
1230       const PetscInt *support;
1231       DMPolytopeType  ct;
1232       PetscInt        coneSize, supportSize, qf, qn, qp, e;
1233 
1234       PetscCall(DMPlexGetConeSize(dm, oldp, &coneSize));
1235       PetscCall(DMPlexSetConeSize(sdm, splitp, coneSize));
1236       PetscCall(DMPlexGetSupportSize(dm, oldp, &supportSize));
1237       PetscCall(DMPlexSetSupportSize(sdm, splitp, supportSize));
1238       PetscCall(DMPlexGetCellType(dm, oldp, &ct));
1239       PetscCall(DMPlexSetCellType(sdm, splitp, ct));
1240       if (dep == depth-1) {
1241         const PetscInt hybcell = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1242 
1243         /* Add cohesive cells, they are prisms */
1244         PetscCall(DMPlexSetConeSize(sdm, hybcell, 2 + coneSize));
1245         switch (coneSize) {
1246           case 2: PetscCall(DMPlexSetCellType(sdm, hybcell, DM_POLYTOPE_SEG_PRISM_TENSOR));break;
1247           case 3: PetscCall(DMPlexSetCellType(sdm, hybcell, DM_POLYTOPE_TRI_PRISM_TENSOR));break;
1248           case 4: PetscCall(DMPlexSetCellType(sdm, hybcell, DM_POLYTOPE_QUAD_PRISM_TENSOR));break;
1249         }
1250         /* Shared fault faces with only one support cell now have two with the cohesive cell */
1251         /*   TODO Check thaat oldp has rootdegree == 1 */
1252         if (supportSize == 1) {
1253           const PetscInt *support;
1254           PetscInt        val;
1255 
1256           PetscCall(DMPlexGetSupport(dm, oldp, &support));
1257           PetscCall(DMLabelGetValue(label, support[0], &val));
1258           if (val < 0) PetscCall(DMPlexSetSupportSize(sdm, splitp, 2));
1259           else         PetscCall(DMPlexSetSupportSize(sdm, newp,   2));
1260         }
1261       } else if (dep == 0) {
1262         const PetscInt hybedge = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1263 
1264         PetscCall(DMPlexGetSupport(dm, oldp, &support));
1265         for (e = 0, qn = 0, qp = 0, qf = 0; e < supportSize; ++e) {
1266           PetscInt val;
1267 
1268           PetscCall(DMLabelGetValue(label, support[e], &val));
1269           if (val == 1) ++qf;
1270           if ((val == 1) || (val ==  (shift + 1))) ++qn;
1271           if ((val == 1) || (val == -(shift + 1))) ++qp;
1272         }
1273         /* Split old vertex: Edges into original vertex and new cohesive edge */
1274         PetscCall(DMPlexSetSupportSize(sdm, newp, qn+1));
1275         /* Split new vertex: Edges into split vertex and new cohesive edge */
1276         PetscCall(DMPlexSetSupportSize(sdm, splitp, qp+1));
1277         /* Add hybrid edge */
1278         PetscCall(DMPlexSetConeSize(sdm, hybedge, 2));
1279         PetscCall(DMPlexSetSupportSize(sdm, hybedge, qf));
1280         PetscCall(DMPlexSetCellType(sdm, hybedge, DM_POLYTOPE_POINT_PRISM_TENSOR));
1281       } else if (dep == dim-2) {
1282         const PetscInt hybface = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1283 
1284         PetscCall(DMPlexGetSupport(dm, oldp, &support));
1285         for (e = 0, qn = 0, qp = 0, qf = 0; e < supportSize; ++e) {
1286           PetscInt val;
1287 
1288           PetscCall(DMLabelGetValue(label, support[e], &val));
1289           if (val == dim-1) ++qf;
1290           if ((val == dim-1) || (val ==  (shift + dim-1))) ++qn;
1291           if ((val == dim-1) || (val == -(shift + dim-1))) ++qp;
1292         }
1293         /* Split old edge: Faces into original edge and cohesive face (positive side?) */
1294         PetscCall(DMPlexSetSupportSize(sdm, newp, qn+1));
1295         /* Split new edge: Faces into split edge and cohesive face (negative side?) */
1296         PetscCall(DMPlexSetSupportSize(sdm, splitp, qp+1));
1297         /* Add hybrid face */
1298         PetscCall(DMPlexSetConeSize(sdm, hybface, 4));
1299         PetscCall(DMPlexSetSupportSize(sdm, hybface, qf));
1300         PetscCall(DMPlexSetCellType(sdm, hybface, DM_POLYTOPE_SEG_PRISM_TENSOR));
1301       }
1302     }
1303   }
1304   for (dep = 0; dep <= depth; ++dep) {
1305     for (p = 0; p < numUnsplitPoints[dep]; ++p) {
1306       const PetscInt  oldp   = unsplitPoints[dep][p];
1307       const PetscInt  newp   = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*oldp + depthOffset[dep]*/;
1308       const PetscInt *support;
1309       PetscInt        coneSize, supportSize, qf, e, s;
1310 
1311       PetscCall(DMPlexGetConeSize(dm, oldp, &coneSize));
1312       PetscCall(DMPlexGetSupportSize(dm, oldp, &supportSize));
1313       PetscCall(DMPlexGetSupport(dm, oldp, &support));
1314       if (dep == 0) {
1315         const PetscInt hybedge = p + pMaxNew[dep+1] + numSplitPoints[dep+1] + numSplitPoints[dep];
1316 
1317         /* Unsplit vertex: Edges into original vertex, split edges, and new cohesive edge twice */
1318         for (s = 0, qf = 0; s < supportSize; ++s, ++qf) {
1319           PetscCall(PetscFindInt(support[s], numSplitPoints[dep+1], splitPoints[dep+1], &e));
1320           if (e >= 0) ++qf;
1321         }
1322         PetscCall(DMPlexSetSupportSize(sdm, newp, qf+2));
1323         /* Add hybrid edge */
1324         PetscCall(DMPlexSetConeSize(sdm, hybedge, 2));
1325         for (e = 0, qf = 0; e < supportSize; ++e) {
1326           PetscInt val;
1327 
1328           PetscCall(DMLabelGetValue(label, support[e], &val));
1329           /* Split and unsplit edges produce hybrid faces */
1330           if (val == 1) ++qf;
1331           if (val == (shift2 + 1)) ++qf;
1332         }
1333         PetscCall(DMPlexSetSupportSize(sdm, hybedge, qf));
1334         PetscCall(DMPlexSetCellType(sdm, hybedge, DM_POLYTOPE_POINT_PRISM_TENSOR));
1335       } else if (dep == dim-2) {
1336         const PetscInt hybface = p + pMaxNew[dep+1] + numSplitPoints[dep+1] + numSplitPoints[dep];
1337         PetscInt       val;
1338 
1339         for (e = 0, qf = 0; e < supportSize; ++e) {
1340           PetscCall(DMLabelGetValue(label, support[e], &val));
1341           if (val == dim-1) qf += 2;
1342           else              ++qf;
1343         }
1344         /* Unsplit edge: Faces into original edge, split face, and cohesive face twice */
1345         PetscCall(DMPlexSetSupportSize(sdm, newp, qf+2));
1346         /* Add hybrid face */
1347         for (e = 0, qf = 0; e < supportSize; ++e) {
1348           PetscCall(DMLabelGetValue(label, support[e], &val));
1349           if (val == dim-1) ++qf;
1350         }
1351         PetscCall(DMPlexSetConeSize(sdm, hybface, 4));
1352         PetscCall(DMPlexSetSupportSize(sdm, hybface, qf));
1353         PetscCall(DMPlexSetCellType(sdm, hybface, DM_POLYTOPE_SEG_PRISM_TENSOR));
1354       }
1355     }
1356   }
1357   /* Step 4: Setup split DM */
1358   PetscCall(DMSetUp(sdm));
1359   PetscCall(DMPlexShiftPoints_Internal(dm, depthShift, sdm));
1360   PetscCall(DMPlexGetMaxSizes(sdm, &maxConeSizeNew, &maxSupportSizeNew));
1361   PetscCall(PetscMalloc3(PetscMax(maxConeSize, maxConeSizeNew)*3,&coneNew,PetscMax(maxConeSize, maxConeSizeNew)*3,&coneONew,PetscMax(maxSupportSize, maxSupportSizeNew),&supportNew));
1362   /* Step 6: Set cones and supports for new points */
1363   for (dep = 0; dep <= depth; ++dep) {
1364     for (p = 0; p < numSplitPoints[dep]; ++p) {
1365       const PetscInt  oldp   = splitPoints[dep][p];
1366       const PetscInt  newp   = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*oldp + depthOffset[dep]*/;
1367       const PetscInt  splitp = p    + pMaxNew[dep];
1368       const PetscInt *cone, *support, *ornt;
1369       DMPolytopeType  ct;
1370       PetscInt        coneSize, supportSize, q, qf, qn, qp, v, e, s;
1371 
1372       PetscCall(DMPlexGetCellType(dm, oldp, &ct));
1373       PetscCall(DMPlexGetConeSize(dm, oldp, &coneSize));
1374       PetscCall(DMPlexGetCone(dm, oldp, &cone));
1375       PetscCall(DMPlexGetConeOrientation(dm, oldp, &ornt));
1376       PetscCall(DMPlexGetSupportSize(dm, oldp, &supportSize));
1377       PetscCall(DMPlexGetSupport(dm, oldp, &support));
1378       if (dep == depth-1) {
1379         PetscBool       hasUnsplit = PETSC_FALSE;
1380         const PetscInt  hybcell    = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1381         const PetscInt *supportF;
1382 
1383         coneONew[0] = coneONew[1] = -1000;
1384         /* Split face:       copy in old face to new face to start */
1385         PetscCall(DMPlexGetSupport(sdm, newp,  &supportF));
1386         PetscCall(DMPlexSetSupport(sdm, splitp, supportF));
1387         /* Split old face:   old vertices/edges in cone so no change */
1388         /* Split new face:   new vertices/edges in cone */
1389         for (q = 0; q < coneSize; ++q) {
1390           PetscCall(PetscFindInt(cone[q], numSplitPoints[dep-1], splitPoints[dep-1], &v));
1391           if (v < 0) {
1392             PetscCall(PetscFindInt(cone[q], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &v));
1393             PetscCheck(v >= 0,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %" PetscInt_FMT " in split or unsplit points of depth %" PetscInt_FMT, cone[q], dep-1);
1394             coneNew[2+q] = DMPlexShiftPoint_Internal(cone[q], depth, depthShift) /*cone[q] + depthOffset[dep-1]*/;
1395             hasUnsplit   = PETSC_TRUE;
1396           } else {
1397             coneNew[2+q] = v + pMaxNew[dep-1];
1398             if (dep > 1) {
1399               const PetscInt *econe;
1400               PetscInt        econeSize, r, vs, vu;
1401 
1402               PetscCall(DMPlexGetConeSize(dm, cone[q], &econeSize));
1403               PetscCall(DMPlexGetCone(dm, cone[q], &econe));
1404               for (r = 0; r < econeSize; ++r) {
1405                 PetscCall(PetscFindInt(econe[r], numSplitPoints[dep-2],   splitPoints[dep-2],   &vs));
1406                 PetscCall(PetscFindInt(econe[r], numUnsplitPoints[dep-2], unsplitPoints[dep-2], &vu));
1407                 if (vs >= 0) continue;
1408                 PetscCheck(vu >= 0,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %" PetscInt_FMT " in split or unsplit points of depth %" PetscInt_FMT, econe[r], dep-2);
1409                 hasUnsplit   = PETSC_TRUE;
1410               }
1411             }
1412           }
1413         }
1414         PetscCall(DMPlexSetCone(sdm, splitp, &coneNew[2]));
1415         PetscCall(DMPlexSetConeOrientation(sdm, splitp, ornt));
1416         /* Face support */
1417         PetscInt vals[2];
1418 
1419         PetscCall(DMLabelGetValue(label, support[0], &vals[0]));
1420         if (supportSize > 1) PetscCall(DMLabelGetValue(label, support[1], &vals[1]));
1421         else                 vals[1] = -vals[0];
1422         PetscCheck(vals[0]*vals[1] < 0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid support labels %" PetscInt_FMT " %" PetscInt_FMT, vals[0], vals[1]);
1423 
1424         for (s = 0; s < 2; ++s) {
1425           if (s >= supportSize) {
1426             if (vals[s] < 0) {
1427               /* Ghost old face:   Replace negative side cell with cohesive cell */
1428               PetscCall(DMPlexInsertSupport(sdm, newp,   s, hybcell));
1429             } else {
1430               /* Ghost new face:   Replace positive side cell with cohesive cell */
1431               PetscCall(DMPlexInsertSupport(sdm, splitp, s, hybcell));
1432             }
1433           } else {
1434             if (vals[s] < 0) {
1435               /* Split old face:   Replace negative side cell with cohesive cell */
1436               PetscCall(DMPlexInsertSupport(sdm, newp,   s, hybcell));
1437             } else {
1438               /* Split new face:   Replace positive side cell with cohesive cell */
1439               PetscCall(DMPlexInsertSupport(sdm, splitp, s, hybcell));
1440             }
1441           }
1442         }
1443         /* Get orientation for cohesive face using the positive side cell */
1444         {
1445           const PetscInt *ncone, *nconeO;
1446           PetscInt        nconeSize, nc, ocell;
1447           PetscBool       flip = PETSC_FALSE;
1448 
1449           if (supportSize > 1) {ocell = vals[0] < 0 ? support[1] : support[0];}
1450           else                 {ocell = support[0]; flip = vals[0] < 0 ? PETSC_TRUE : PETSC_FALSE;}
1451           PetscCall(DMPlexGetConeSize(dm, ocell, &nconeSize));
1452           PetscCall(DMPlexGetCone(dm, ocell, &ncone));
1453           PetscCall(DMPlexGetConeOrientation(dm, support[s], &nconeO));
1454           for (nc = 0; nc < nconeSize; ++nc) {
1455             if (ncone[nc] == oldp) {
1456               coneONew[0] = flip ? -(nconeO[nc]+1) : nconeO[nc];
1457               break;
1458             }
1459           }
1460           PetscCheck(nc < nconeSize, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate face %" PetscInt_FMT " in neighboring cell %" PetscInt_FMT, oldp, ocell);
1461         }
1462         /* Cohesive cell:    Old and new split face, then new cohesive faces */
1463         {
1464           const PetscInt No = DMPolytopeTypeGetNumArrangments(ct)/2;
1465           PetscCheck((coneONew[0] >= -No) && (coneONew[0] < No), PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid %s orientation %" PetscInt_FMT, DMPolytopeTypes[ct], coneONew[0]);
1466         }
1467         const PetscInt *arr = DMPolytopeTypeGetArrangment(ct, coneONew[0]);
1468 
1469         coneNew[0]  = newp;   /* Extracted negative side orientation above */
1470         coneNew[1]  = splitp;
1471         coneONew[1] = coneONew[0];
1472         for (q = 0; q < coneSize; ++q) {
1473           /* Hybrid faces must follow order from oriented end face */
1474           const PetscInt qa = arr[q*2+0];
1475           const PetscInt qo = arr[q*2+1];
1476           DMPolytopeType ft = dep == 2 ? DM_POLYTOPE_SEGMENT : DM_POLYTOPE_POINT;
1477 
1478           PetscCall(PetscFindInt(cone[qa], numSplitPoints[dep-1], splitPoints[dep-1], &v));
1479           if (v < 0) {
1480             PetscCall(PetscFindInt(cone[qa], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &v));
1481             coneNew[2+q]  = v + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep-1];
1482           } else {
1483             coneNew[2+q]  = v + pMaxNew[dep] + numSplitPoints[dep];
1484           }
1485           coneONew[2+q] = DMPolytopeTypeComposeOrientation(ft, qo, ornt[qa]);
1486         }
1487         PetscCall(DMPlexSetCone(sdm, hybcell, coneNew));
1488         PetscCall(DMPlexSetConeOrientation(sdm, hybcell, coneONew));
1489         /* Label the hybrid cells on the boundary of the split */
1490         if (hasUnsplit) PetscCall(DMLabelSetValue(label, -hybcell, dim));
1491       } else if (dep == 0) {
1492         const PetscInt hybedge = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1493 
1494         /* Split old vertex: Edges in old split faces and new cohesive edge */
1495         for (e = 0, qn = 0; e < supportSize; ++e) {
1496           PetscInt val;
1497 
1498           PetscCall(DMLabelGetValue(label, support[e], &val));
1499           if ((val == 1) || (val == (shift + 1))) {
1500             supportNew[qn++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1501           }
1502         }
1503         supportNew[qn] = hybedge;
1504         PetscCall(DMPlexSetSupport(sdm, newp, supportNew));
1505         /* Split new vertex: Edges in new split faces and new cohesive edge */
1506         for (e = 0, qp = 0; e < supportSize; ++e) {
1507           PetscInt val, edge;
1508 
1509           PetscCall(DMLabelGetValue(label, support[e], &val));
1510           if (val == 1) {
1511             PetscCall(PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &edge));
1512             PetscCheck(edge >= 0,comm, PETSC_ERR_ARG_WRONG, "Edge %" PetscInt_FMT " is not a split edge", support[e]);
1513             supportNew[qp++] = edge + pMaxNew[dep+1];
1514           } else if (val == -(shift + 1)) {
1515             supportNew[qp++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1516           }
1517         }
1518         supportNew[qp] = hybedge;
1519         PetscCall(DMPlexSetSupport(sdm, splitp, supportNew));
1520         /* Hybrid edge:    Old and new split vertex */
1521         coneNew[0] = newp;
1522         coneNew[1] = splitp;
1523         PetscCall(DMPlexSetCone(sdm, hybedge, coneNew));
1524         for (e = 0, qf = 0; e < supportSize; ++e) {
1525           PetscInt val, edge;
1526 
1527           PetscCall(DMLabelGetValue(label, support[e], &val));
1528           if (val == 1) {
1529             PetscCall(PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &edge));
1530             PetscCheck(edge >= 0,comm, PETSC_ERR_ARG_WRONG, "Edge %" PetscInt_FMT " is not a split edge", support[e]);
1531             supportNew[qf++] = edge + pMaxNew[dep+2] + numSplitPoints[dep+2];
1532           }
1533         }
1534         PetscCall(DMPlexSetSupport(sdm, hybedge, supportNew));
1535       } else if (dep == dim-2) {
1536         const PetscInt hybface = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1537 
1538         /* Split old edge:   old vertices in cone so no change */
1539         /* Split new edge:   new vertices in cone */
1540         for (q = 0; q < coneSize; ++q) {
1541           PetscCall(PetscFindInt(cone[q], numSplitPoints[dep-1], splitPoints[dep-1], &v));
1542           if (v < 0) {
1543             PetscCall(PetscFindInt(cone[q], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &v));
1544             PetscCheck(v >= 0,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %" PetscInt_FMT " in split or unsplit points of depth %" PetscInt_FMT, cone[q], dep-1);
1545             coneNew[q] = DMPlexShiftPoint_Internal(cone[q], depth, depthShift) /*cone[q] + depthOffset[dep-1]*/;
1546           } else {
1547             coneNew[q] = v + pMaxNew[dep-1];
1548           }
1549         }
1550         PetscCall(DMPlexSetCone(sdm, splitp, coneNew));
1551         /* Split old edge: Faces in positive side cells and old split faces */
1552         for (e = 0, q = 0; e < supportSize; ++e) {
1553           PetscInt val;
1554 
1555           PetscCall(DMLabelGetValue(label, support[e], &val));
1556           if (val == dim-1) {
1557             supportNew[q++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1558           } else if (val == (shift + dim-1)) {
1559             supportNew[q++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1560           }
1561         }
1562         supportNew[q++] = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1563         PetscCall(DMPlexSetSupport(sdm, newp, supportNew));
1564         /* Split new edge: Faces in negative side cells and new split faces */
1565         for (e = 0, q = 0; e < supportSize; ++e) {
1566           PetscInt val, face;
1567 
1568           PetscCall(DMLabelGetValue(label, support[e], &val));
1569           if (val == dim-1) {
1570             PetscCall(PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &face));
1571             PetscCheck(face >= 0,comm, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " is not a split face", support[e]);
1572             supportNew[q++] = face + pMaxNew[dep+1];
1573           } else if (val == -(shift + dim-1)) {
1574             supportNew[q++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1575           }
1576         }
1577         supportNew[q++] = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1578         PetscCall(DMPlexSetSupport(sdm, splitp, supportNew));
1579         /* Hybrid face */
1580         coneNew[0] = newp;
1581         coneNew[1] = splitp;
1582         for (v = 0; v < coneSize; ++v) {
1583           PetscInt vertex;
1584           PetscCall(PetscFindInt(cone[v], numSplitPoints[dep-1], splitPoints[dep-1], &vertex));
1585           if (vertex < 0) {
1586             PetscCall(PetscFindInt(cone[v], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &vertex));
1587             PetscCheck(vertex >= 0,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %" PetscInt_FMT " in split or unsplit points of depth %" PetscInt_FMT, cone[v], dep-1);
1588             coneNew[2+v] = vertex + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep-1];
1589           } else {
1590             coneNew[2+v] = vertex + pMaxNew[dep] + numSplitPoints[dep];
1591           }
1592         }
1593         PetscCall(DMPlexSetCone(sdm, hybface, coneNew));
1594         for (e = 0, qf = 0; e < supportSize; ++e) {
1595           PetscInt val, face;
1596 
1597           PetscCall(DMLabelGetValue(label, support[e], &val));
1598           if (val == dim-1) {
1599             PetscCall(PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &face));
1600             PetscCheck(face >= 0,comm, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " is not a split face", support[e]);
1601             supportNew[qf++] = face + pMaxNew[dep+2] + numSplitPoints[dep+2];
1602           }
1603         }
1604         PetscCall(DMPlexSetSupport(sdm, hybface, supportNew));
1605       }
1606     }
1607   }
1608   for (dep = 0; dep <= depth; ++dep) {
1609     for (p = 0; p < numUnsplitPoints[dep]; ++p) {
1610       const PetscInt  oldp   = unsplitPoints[dep][p];
1611       const PetscInt  newp   = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*oldp + depthOffset[dep]*/;
1612       const PetscInt *cone, *support;
1613       PetscInt        coneSize, supportSize, supportSizeNew, q, qf, e, f, s;
1614 
1615       PetscCall(DMPlexGetConeSize(dm, oldp, &coneSize));
1616       PetscCall(DMPlexGetCone(dm, oldp, &cone));
1617       PetscCall(DMPlexGetSupportSize(dm, oldp, &supportSize));
1618       PetscCall(DMPlexGetSupport(dm, oldp, &support));
1619       if (dep == 0) {
1620         const PetscInt hybedge = p + pMaxNew[dep+1] + numSplitPoints[dep+1] + numSplitPoints[dep];
1621 
1622         /* Unsplit vertex */
1623         PetscCall(DMPlexGetSupportSize(sdm, newp, &supportSizeNew));
1624         for (s = 0, q = 0; s < supportSize; ++s) {
1625           supportNew[q++] = DMPlexShiftPoint_Internal(support[s], depth, depthShift) /*support[s] + depthOffset[dep+1]*/;
1626           PetscCall(PetscFindInt(support[s], numSplitPoints[dep+1], splitPoints[dep+1], &e));
1627           if (e >= 0) {
1628             supportNew[q++] = e + pMaxNew[dep+1];
1629           }
1630         }
1631         supportNew[q++] = hybedge;
1632         supportNew[q++] = hybedge;
1633         PetscCheck(q == supportSizeNew,comm, PETSC_ERR_ARG_WRONG, "Support size %" PetscInt_FMT " != %" PetscInt_FMT " for vertex %" PetscInt_FMT, q, supportSizeNew, newp);
1634         PetscCall(DMPlexSetSupport(sdm, newp, supportNew));
1635         /* Hybrid edge */
1636         coneNew[0] = newp;
1637         coneNew[1] = newp;
1638         PetscCall(DMPlexSetCone(sdm, hybedge, coneNew));
1639         for (e = 0, qf = 0; e < supportSize; ++e) {
1640           PetscInt val, edge;
1641 
1642           PetscCall(DMLabelGetValue(label, support[e], &val));
1643           if (val == 1) {
1644             PetscCall(PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &edge));
1645             PetscCheck(edge >= 0,comm, PETSC_ERR_ARG_WRONG, "Edge %" PetscInt_FMT " is not a split edge", support[e]);
1646             supportNew[qf++] = edge + pMaxNew[dep+2] + numSplitPoints[dep+2];
1647           } else if  (val ==  (shift2 + 1)) {
1648             PetscCall(PetscFindInt(support[e], numUnsplitPoints[dep+1], unsplitPoints[dep+1], &edge));
1649             PetscCheck(edge >= 0,comm, PETSC_ERR_ARG_WRONG, "Edge %" PetscInt_FMT " is not a unsplit edge", support[e]);
1650             supportNew[qf++] = edge + pMaxNew[dep+2] + numSplitPoints[dep+2] + numSplitPoints[dep+1];
1651           }
1652         }
1653         PetscCall(DMPlexSetSupport(sdm, hybedge, supportNew));
1654       } else if (dep == dim-2) {
1655         const PetscInt hybface = p + pMaxNew[dep+1] + numSplitPoints[dep+1] + numSplitPoints[dep];
1656 
1657         /* Unsplit edge: Faces into original edge, split face, and hybrid face twice */
1658         for (f = 0, qf = 0; f < supportSize; ++f) {
1659           PetscInt val, face;
1660 
1661           PetscCall(DMLabelGetValue(label, support[f], &val));
1662           if (val == dim-1) {
1663             PetscCall(PetscFindInt(support[f], numSplitPoints[dep+1], splitPoints[dep+1], &face));
1664             PetscCheck(face >= 0,comm, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " is not a split face", support[f]);
1665             supportNew[qf++] = DMPlexShiftPoint_Internal(support[f], depth, depthShift) /*support[f] + depthOffset[dep+1]*/;
1666             supportNew[qf++] = face + pMaxNew[dep+1];
1667           } else {
1668             supportNew[qf++] = DMPlexShiftPoint_Internal(support[f], depth, depthShift) /*support[f] + depthOffset[dep+1]*/;
1669           }
1670         }
1671         supportNew[qf++] = hybface;
1672         supportNew[qf++] = hybface;
1673         PetscCall(DMPlexGetSupportSize(sdm, newp, &supportSizeNew));
1674         PetscCheck(qf == supportSizeNew,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Support size for unsplit edge %" PetscInt_FMT " is %" PetscInt_FMT " != %" PetscInt_FMT, newp, qf, supportSizeNew);
1675         PetscCall(DMPlexSetSupport(sdm, newp, supportNew));
1676         /* Add hybrid face */
1677         coneNew[0] = newp;
1678         coneNew[1] = newp;
1679         PetscCall(PetscFindInt(cone[0], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &v));
1680         PetscCheck(v >= 0,comm, PETSC_ERR_ARG_WRONG, "Vertex %" PetscInt_FMT " is not an unsplit vertex", cone[0]);
1681         coneNew[2] = v + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep-1];
1682         PetscCall(PetscFindInt(cone[1], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &v));
1683         PetscCheck(v >= 0,comm, PETSC_ERR_ARG_WRONG, "Vertex %" PetscInt_FMT " is not an unsplit vertex", cone[1]);
1684         coneNew[3] = v + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep-1];
1685         PetscCall(DMPlexSetCone(sdm, hybface, coneNew));
1686         for (f = 0, qf = 0; f < supportSize; ++f) {
1687           PetscInt val, face;
1688 
1689           PetscCall(DMLabelGetValue(label, support[f], &val));
1690           if (val == dim-1) {
1691             PetscCall(PetscFindInt(support[f], numSplitPoints[dep+1], splitPoints[dep+1], &face));
1692             supportNew[qf++] = face + pMaxNew[dep+2] + numSplitPoints[dep+2];
1693           }
1694         }
1695         PetscCall(DMPlexGetSupportSize(sdm, hybface, &supportSizeNew));
1696         PetscCheck(qf == supportSizeNew,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Support size for hybrid face %" PetscInt_FMT " is %" PetscInt_FMT " != %" PetscInt_FMT, hybface, qf, supportSizeNew);
1697         PetscCall(DMPlexSetSupport(sdm, hybface, supportNew));
1698       }
1699     }
1700   }
1701   /* Step 6b: Replace split points in negative side cones */
1702   for (sp = 0; sp < numSP; ++sp) {
1703     PetscInt        dep = values[sp];
1704     IS              pIS;
1705     PetscInt        numPoints;
1706     const PetscInt *points;
1707 
1708     if (dep >= 0) continue;
1709     PetscCall(DMLabelGetStratumIS(label, dep, &pIS));
1710     if (!pIS) continue;
1711     dep  = -dep - shift;
1712     PetscCall(ISGetLocalSize(pIS, &numPoints));
1713     PetscCall(ISGetIndices(pIS, &points));
1714     for (p = 0; p < numPoints; ++p) {
1715       const PetscInt  oldp = points[p];
1716       const PetscInt  newp = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*depthOffset[dep] + oldp*/;
1717       const PetscInt *cone;
1718       PetscInt        coneSize, c;
1719       /* PetscBool       replaced = PETSC_FALSE; */
1720 
1721       /* Negative edge: replace split vertex */
1722       /* Negative cell: replace split face */
1723       PetscCall(DMPlexGetConeSize(sdm, newp, &coneSize));
1724       PetscCall(DMPlexGetCone(sdm, newp, &cone));
1725       for (c = 0; c < coneSize; ++c) {
1726         const PetscInt coldp = DMPlexShiftPointInverse_Internal(cone[c],depth,depthShift);
1727         PetscInt       csplitp, cp, val;
1728 
1729         PetscCall(DMLabelGetValue(label, coldp, &val));
1730         if (val == dep-1) {
1731           PetscCall(PetscFindInt(coldp, numSplitPoints[dep-1], splitPoints[dep-1], &cp));
1732           PetscCheck(cp >= 0,comm, PETSC_ERR_ARG_WRONG, "Point %" PetscInt_FMT " is not a split point of dimension %" PetscInt_FMT, oldp, dep-1);
1733           csplitp  = pMaxNew[dep-1] + cp;
1734           PetscCall(DMPlexInsertCone(sdm, newp, c, csplitp));
1735           /* replaced = PETSC_TRUE; */
1736         }
1737       }
1738       /* Cells with only a vertex or edge on the submesh have no replacement */
1739       /* PetscCheck(replaced,comm, PETSC_ERR_ARG_WRONG, "The cone of point %d does not contain split points", oldp); */
1740     }
1741     PetscCall(ISRestoreIndices(pIS, &points));
1742     PetscCall(ISDestroy(&pIS));
1743   }
1744   /* Step 7: Coordinates */
1745   PetscCall(DMPlexShiftCoordinates_Internal(dm, depthShift, sdm));
1746   PetscCall(DMGetCoordinateSection(sdm, &coordSection));
1747   PetscCall(DMGetCoordinatesLocal(sdm, &coordinates));
1748   PetscCall(VecGetArray(coordinates, &coords));
1749   for (v = 0; v < (numSplitPoints ? numSplitPoints[0] : 0); ++v) {
1750     const PetscInt newp   = DMPlexShiftPoint_Internal(splitPoints[0][v], depth, depthShift) /*depthOffset[0] + splitPoints[0][v]*/;
1751     const PetscInt splitp = pMaxNew[0] + v;
1752     PetscInt       dof, off, soff, d;
1753 
1754     PetscCall(PetscSectionGetDof(coordSection, newp, &dof));
1755     PetscCall(PetscSectionGetOffset(coordSection, newp, &off));
1756     PetscCall(PetscSectionGetOffset(coordSection, splitp, &soff));
1757     for (d = 0; d < dof; ++d) coords[soff+d] = coords[off+d];
1758   }
1759   PetscCall(VecRestoreArray(coordinates, &coords));
1760   /* Step 8: SF, if I can figure this out we can split the mesh in parallel */
1761   PetscCall(DMPlexShiftSF_Internal(dm, depthShift, sdm));
1762   /*   TODO We need to associate the ghost points with the correct replica */
1763   /* Step 9: Labels */
1764   PetscCall(DMPlexShiftLabels_Internal(dm, depthShift, sdm));
1765   PetscCall(DMPlexCreateVTKLabel_Internal(dm, PETSC_FALSE, sdm));
1766   PetscCall(DMGetNumLabels(sdm, &numLabels));
1767   for (dep = 0; dep <= depth; ++dep) {
1768     for (p = 0; p < numSplitPoints[dep]; ++p) {
1769       const PetscInt newp   = DMPlexShiftPoint_Internal(splitPoints[dep][p], depth, depthShift) /*depthOffset[dep] + splitPoints[dep][p]*/;
1770       const PetscInt splitp = pMaxNew[dep] + p;
1771       PetscInt       l;
1772 
1773       if (splitLabel) {
1774         const PetscInt val = 100 + dep;
1775 
1776         PetscCall(DMLabelSetValue(splitLabel, newp,    val));
1777         PetscCall(DMLabelSetValue(splitLabel, splitp, -val));
1778       }
1779       for (l = 0; l < numLabels; ++l) {
1780         DMLabel     mlabel;
1781         const char *lname;
1782         PetscInt    val;
1783         PetscBool   isDepth;
1784 
1785         PetscCall(DMGetLabelName(sdm, l, &lname));
1786         PetscCall(PetscStrcmp(lname, "depth", &isDepth));
1787         if (isDepth) continue;
1788         PetscCall(DMGetLabel(sdm, lname, &mlabel));
1789         PetscCall(DMLabelGetValue(mlabel, newp, &val));
1790         if (val >= 0) {
1791           PetscCall(DMLabelSetValue(mlabel, splitp, val));
1792         }
1793       }
1794     }
1795   }
1796   for (sp = 0; sp < numSP; ++sp) {
1797     const PetscInt dep = values[sp];
1798 
1799     if ((dep < 0) || (dep > depth)) continue;
1800     if (splitIS[dep]) PetscCall(ISRestoreIndices(splitIS[dep], &splitPoints[dep]));
1801     PetscCall(ISDestroy(&splitIS[dep]));
1802     if (unsplitIS[dep]) PetscCall(ISRestoreIndices(unsplitIS[dep], &unsplitPoints[dep]));
1803     PetscCall(ISDestroy(&unsplitIS[dep]));
1804   }
1805   if (ghostIS) PetscCall(ISRestoreIndices(ghostIS, &ghostPoints));
1806   PetscCall(ISDestroy(&ghostIS));
1807   if (label) {
1808     PetscCall(ISRestoreIndices(valueIS, &values));
1809     PetscCall(ISDestroy(&valueIS));
1810   }
1811   for (d = 0; d <= depth; ++d) {
1812     PetscCall(DMPlexGetDepthStratum(sdm, d, NULL, &pEnd));
1813     pMaxNew[d] = pEnd - numHybridPoints[d] - numHybridPointsOld[d];
1814   }
1815   PetscCall(PetscFree3(coneNew, coneONew, supportNew));
1816   PetscCall(PetscFree5(depthMax, depthEnd, depthShift, pMaxNew, numHybridPointsOld));
1817   PetscCall(PetscFree7(splitIS, unsplitIS, numSplitPoints, numUnsplitPoints, numHybridPoints, splitPoints, unsplitPoints));
1818   PetscFunctionReturn(0);
1819 }
1820 
1821 /*@C
1822   DMPlexConstructCohesiveCells - Construct cohesive cells which split the face along an internal interface
1823 
1824   Collective on dm
1825 
1826   Input Parameters:
1827 + dm - The original DM
1828 - label - The label specifying the boundary faces (this could be auto-generated)
1829 
1830   Output Parameters:
1831 + splitLabel - The label containing the split points, or NULL if no output is desired
1832 - dmSplit - The new DM
1833 
1834   Level: developer
1835 
1836 .seealso: `DMCreate()`, `DMPlexLabelCohesiveComplete()`
1837 @*/
1838 PetscErrorCode DMPlexConstructCohesiveCells(DM dm, DMLabel label, DMLabel splitLabel, DM *dmSplit)
1839 {
1840   DM             sdm;
1841   PetscInt       dim;
1842 
1843   PetscFunctionBegin;
1844   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1845   PetscValidPointer(dmSplit, 4);
1846   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &sdm));
1847   PetscCall(DMSetType(sdm, DMPLEX));
1848   PetscCall(DMGetDimension(dm, &dim));
1849   PetscCall(DMSetDimension(sdm, dim));
1850   switch (dim) {
1851   case 2:
1852   case 3:
1853     PetscCall(DMPlexConstructCohesiveCells_Internal(dm, label, splitLabel, sdm));
1854     break;
1855   default:
1856     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cannot construct cohesive cells for dimension %" PetscInt_FMT, dim);
1857   }
1858   *dmSplit = sdm;
1859   PetscFunctionReturn(0);
1860 }
1861 
1862 /* Returns the side of the surface for a given cell with a face on the surface */
1863 static PetscErrorCode GetSurfaceSide_Static(DM dm, DM subdm, PetscInt numSubpoints, const PetscInt *subpoints, PetscInt cell, PetscInt face, PetscBool *pos)
1864 {
1865   const PetscInt *cone, *ornt;
1866   PetscInt        dim, coneSize, c;
1867 
1868   PetscFunctionBegin;
1869   *pos = PETSC_TRUE;
1870   PetscCall(DMGetDimension(dm, &dim));
1871   PetscCall(DMPlexGetConeSize(dm, cell, &coneSize));
1872   PetscCall(DMPlexGetCone(dm, cell, &cone));
1873   PetscCall(DMPlexGetConeOrientation(dm, cell, &ornt));
1874   for (c = 0; c < coneSize; ++c) {
1875     if (cone[c] == face) {
1876       PetscInt o = ornt[c];
1877 
1878       if (subdm) {
1879         const PetscInt *subcone, *subornt;
1880         PetscInt        subpoint, subface, subconeSize, sc;
1881 
1882         PetscCall(PetscFindInt(cell, numSubpoints, subpoints, &subpoint));
1883         PetscCall(PetscFindInt(face, numSubpoints, subpoints, &subface));
1884         PetscCall(DMPlexGetConeSize(subdm, subpoint, &subconeSize));
1885         PetscCall(DMPlexGetCone(subdm, subpoint, &subcone));
1886         PetscCall(DMPlexGetConeOrientation(subdm, subpoint, &subornt));
1887         for (sc = 0; sc < subconeSize; ++sc) {
1888           if (subcone[sc] == subface) {
1889             o = subornt[0];
1890             break;
1891           }
1892         }
1893         PetscCheck(sc < subconeSize,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not find subpoint %" PetscInt_FMT " (%" PetscInt_FMT ") in cone for subpoint %" PetscInt_FMT " (%" PetscInt_FMT ")", subface, face, subpoint, cell);
1894       }
1895       if (o >= 0) *pos = PETSC_TRUE;
1896       else        *pos = PETSC_FALSE;
1897       break;
1898     }
1899   }
1900   PetscCheck(c != coneSize,PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Cell %" PetscInt_FMT " in split face %" PetscInt_FMT " support does not have it in the cone", cell, face);
1901   PetscFunctionReturn(0);
1902 }
1903 
1904 static PetscErrorCode CheckFaultEdge_Private(DM dm, DMLabel label)
1905 {
1906   IS              facePosIS, faceNegIS, dimIS;
1907   const PetscInt *points;
1908   PetscInt        dim, numPoints, p, shift = 100, shift2 = 200;
1909 
1910   PetscFunctionBegin;
1911   PetscCall(DMGetDimension(dm, &dim));
1912   /* If any faces touching the fault divide cells on either side, split them */
1913   PetscCall(DMLabelGetStratumIS(label,   shift+dim-1,  &facePosIS));
1914   PetscCall(DMLabelGetStratumIS(label, -(shift+dim-1), &faceNegIS));
1915   if (!facePosIS || !faceNegIS) {
1916     PetscCall(ISDestroy(&facePosIS));
1917     PetscCall(ISDestroy(&faceNegIS));
1918     PetscFunctionReturn(0);
1919   }
1920   PetscCall(ISExpand(facePosIS, faceNegIS, &dimIS));
1921   PetscCall(ISDestroy(&facePosIS));
1922   PetscCall(ISDestroy(&faceNegIS));
1923   PetscCall(ISGetLocalSize(dimIS, &numPoints));
1924   PetscCall(ISGetIndices(dimIS, &points));
1925   for (p = 0; p < numPoints; ++p) {
1926     const PetscInt  point = points[p];
1927     const PetscInt *support;
1928     PetscInt        supportSize, valA, valB;
1929 
1930     PetscCall(DMPlexGetSupportSize(dm, point, &supportSize));
1931     if (supportSize != 2) continue;
1932     PetscCall(DMPlexGetSupport(dm, point, &support));
1933     PetscCall(DMLabelGetValue(label, support[0], &valA));
1934     PetscCall(DMLabelGetValue(label, support[1], &valB));
1935     if ((valA == -1) || (valB == -1)) continue;
1936     if (valA*valB > 0) continue;
1937     /* Split the face */
1938     PetscCall(DMLabelGetValue(label, point, &valA));
1939     PetscCall(DMLabelClearValue(label, point, valA));
1940     PetscCall(DMLabelSetValue(label, point, dim-1));
1941     /* Label its closure:
1942       unmarked: label as unsplit
1943       incident: relabel as split
1944       split:    do nothing
1945     */
1946     {
1947       PetscInt *closure = NULL;
1948       PetscInt  closureSize, cl, dep;
1949 
1950       PetscCall(DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
1951       for (cl = 0; cl < closureSize*2; cl += 2) {
1952         PetscCall(DMLabelGetValue(label, closure[cl], &valA));
1953         if (valA == -1) { /* Mark as unsplit */
1954           PetscCall(DMPlexGetPointDepth(dm, closure[cl], &dep));
1955           PetscCall(DMLabelSetValue(label, closure[cl], shift2+dep));
1956         } else if (((valA >= shift) && (valA < shift2)) || ((valA <= -shift) && (valA > -shift2))) {
1957           PetscCall(DMPlexGetPointDepth(dm, closure[cl], &dep));
1958           PetscCall(DMLabelClearValue(label, closure[cl], valA));
1959           PetscCall(DMLabelSetValue(label, closure[cl], dep));
1960         }
1961       }
1962       PetscCall(DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
1963     }
1964   }
1965   PetscCall(ISRestoreIndices(dimIS, &points));
1966   PetscCall(ISDestroy(&dimIS));
1967   PetscFunctionReturn(0);
1968 }
1969 
1970 /*@
1971   DMPlexLabelCohesiveComplete - Starting with a label marking points on an internal surface, we add all other mesh pieces
1972   to complete the surface
1973 
1974   Input Parameters:
1975 + dm     - The DM
1976 . label  - A DMLabel marking the surface
1977 . blabel - A DMLabel marking the vertices on the boundary which will not be duplicated, or NULL to find them automatically
1978 . bvalue - Value of DMLabel marking the vertices on the boundary
1979 . flip   - Flag to flip the submesh normal and replace points on the other side
1980 - subdm  - The subDM associated with the label, or NULL
1981 
1982   Output Parameter:
1983 . label - A DMLabel marking all surface points
1984 
1985   Note: The vertices in blabel are called "unsplit" in the terminology from hybrid cell creation.
1986 
1987   Level: developer
1988 
1989 .seealso: `DMPlexConstructCohesiveCells()`, `DMPlexLabelComplete()`
1990 @*/
1991 PetscErrorCode DMPlexLabelCohesiveComplete(DM dm, DMLabel label, DMLabel blabel, PetscInt bvalue, PetscBool flip, DM subdm)
1992 {
1993   DMLabel         depthLabel;
1994   IS              dimIS, subpointIS = NULL;
1995   const PetscInt *points, *subpoints;
1996   const PetscInt  rev   = flip ? -1 : 1;
1997   PetscInt        shift = 100, shift2 = 200, shift3 = 300, dim, depth, numPoints, numSubpoints, p, val;
1998 
1999   PetscFunctionBegin;
2000   PetscCall(DMPlexGetDepth(dm, &depth));
2001   PetscCall(DMGetDimension(dm, &dim));
2002   PetscCall(DMPlexGetDepthLabel(dm, &depthLabel));
2003   if (subdm) {
2004     PetscCall(DMPlexGetSubpointIS(subdm, &subpointIS));
2005     if (subpointIS) {
2006       PetscCall(ISGetLocalSize(subpointIS, &numSubpoints));
2007       PetscCall(ISGetIndices(subpointIS, &subpoints));
2008     }
2009   }
2010   /* Mark cell on the fault, and its faces which touch the fault: cell orientation for face gives the side of the fault */
2011   PetscCall(DMLabelGetStratumIS(label, dim-1, &dimIS));
2012   if (!dimIS) goto divide;
2013   PetscCall(ISGetLocalSize(dimIS, &numPoints));
2014   PetscCall(ISGetIndices(dimIS, &points));
2015   for (p = 0; p < numPoints; ++p) { /* Loop over fault faces */
2016     const PetscInt *support;
2017     PetscInt        supportSize, s;
2018 
2019     PetscCall(DMPlexGetSupportSize(dm, points[p], &supportSize));
2020 #if 0
2021     if (supportSize != 2) {
2022       const PetscInt *lp;
2023       PetscInt        Nlp, pind;
2024 
2025       /* Check that for a cell with a single support face, that face is in the SF */
2026       /*   THis check only works for the remote side. We would need root side information */
2027       PetscCall(PetscSFGetGraph(dm->sf, NULL, &Nlp, &lp, NULL));
2028       PetscCall(PetscFindInt(points[p], Nlp, lp, &pind));
2029       PetscCheck(pind >= 0,PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Split face %" PetscInt_FMT " has %" PetscInt_FMT " != 2 supports, and the face is not shared with another process", points[p], supportSize);
2030     }
2031 #endif
2032     PetscCall(DMPlexGetSupport(dm, points[p], &support));
2033     for (s = 0; s < supportSize; ++s) {
2034       const PetscInt *cone;
2035       PetscInt        coneSize, c;
2036       PetscBool       pos;
2037 
2038       PetscCall(GetSurfaceSide_Static(dm, subdm, numSubpoints, subpoints, support[s], points[p], &pos));
2039       if (pos) PetscCall(DMLabelSetValue(label, support[s],  rev*(shift+dim)));
2040       else     PetscCall(DMLabelSetValue(label, support[s], -rev*(shift+dim)));
2041       if (rev < 0) pos = !pos ? PETSC_TRUE : PETSC_FALSE;
2042       /* Put faces touching the fault in the label */
2043       PetscCall(DMPlexGetConeSize(dm, support[s], &coneSize));
2044       PetscCall(DMPlexGetCone(dm, support[s], &cone));
2045       for (c = 0; c < coneSize; ++c) {
2046         const PetscInt point = cone[c];
2047 
2048         PetscCall(DMLabelGetValue(label, point, &val));
2049         if (val == -1) {
2050           PetscInt *closure = NULL;
2051           PetscInt  closureSize, cl;
2052 
2053           PetscCall(DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
2054           for (cl = 0; cl < closureSize*2; cl += 2) {
2055             const PetscInt clp  = closure[cl];
2056             PetscInt       bval = -1;
2057 
2058             PetscCall(DMLabelGetValue(label, clp, &val));
2059             if (blabel) PetscCall(DMLabelGetValue(blabel, clp, &bval));
2060             if ((val >= 0) && (val < dim-1) && (bval < 0)) {
2061               PetscCall(DMLabelSetValue(label, point, pos == PETSC_TRUE ? shift+dim-1 : -(shift+dim-1)));
2062               break;
2063             }
2064           }
2065           PetscCall(DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
2066         }
2067       }
2068     }
2069   }
2070   PetscCall(ISRestoreIndices(dimIS, &points));
2071   PetscCall(ISDestroy(&dimIS));
2072   /* Mark boundary points as unsplit */
2073   if (blabel) {
2074     IS bdIS;
2075 
2076     PetscCall(DMLabelGetStratumIS(blabel, bvalue, &bdIS));
2077     PetscCall(ISGetLocalSize(bdIS, &numPoints));
2078     PetscCall(ISGetIndices(bdIS, &points));
2079     for (p = 0; p < numPoints; ++p) {
2080       const PetscInt point = points[p];
2081       PetscInt       val, bval;
2082 
2083       PetscCall(DMLabelGetValue(blabel, point, &bval));
2084       if (bval >= 0) {
2085         PetscCall(DMLabelGetValue(label, point, &val));
2086         if ((val < 0) || (val > dim)) {
2087           /* This could be a point added from splitting a vertex on an adjacent fault, otherwise its just wrong */
2088           PetscCall(DMLabelClearValue(blabel, point, bval));
2089         }
2090       }
2091     }
2092     for (p = 0; p < numPoints; ++p) {
2093       const PetscInt point = points[p];
2094       PetscInt       val, bval;
2095 
2096       PetscCall(DMLabelGetValue(blabel, point, &bval));
2097       if (bval >= 0) {
2098         const PetscInt *cone,    *support;
2099         PetscInt        coneSize, supportSize, s, valA, valB, valE;
2100 
2101         /* Mark as unsplit */
2102         PetscCall(DMLabelGetValue(label, point, &val));
2103         PetscCheck(!(val < 0) && !(val > dim),PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %" PetscInt_FMT " has label value %" PetscInt_FMT ", should be part of the fault", point, val);
2104         PetscCall(DMLabelClearValue(label, point, val));
2105         PetscCall(DMLabelSetValue(label, point, shift2+val));
2106         /* Check for cross-edge
2107              A cross-edge has endpoints which are both on the boundary of the surface, but the edge itself is not. */
2108         if (val != 0) continue;
2109         PetscCall(DMPlexGetSupport(dm, point, &support));
2110         PetscCall(DMPlexGetSupportSize(dm, point, &supportSize));
2111         for (s = 0; s < supportSize; ++s) {
2112           PetscCall(DMPlexGetCone(dm, support[s], &cone));
2113           PetscCall(DMPlexGetConeSize(dm, support[s], &coneSize));
2114           PetscCheck(coneSize == 2,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Edge %" PetscInt_FMT " has %" PetscInt_FMT " vertices != 2", support[s], coneSize);
2115           PetscCall(DMLabelGetValue(blabel, cone[0], &valA));
2116           PetscCall(DMLabelGetValue(blabel, cone[1], &valB));
2117           PetscCall(DMLabelGetValue(blabel, support[s], &valE));
2118           if ((valE < 0) && (valA >= 0) && (valB >= 0) && (cone[0] != cone[1])) PetscCall(DMLabelSetValue(blabel, support[s], 2));
2119         }
2120       }
2121     }
2122     PetscCall(ISRestoreIndices(bdIS, &points));
2123     PetscCall(ISDestroy(&bdIS));
2124   }
2125   /* Mark ghost fault cells */
2126   {
2127     PetscSF         sf;
2128     const PetscInt *leaves;
2129     PetscInt         Nl, l;
2130 
2131     PetscCall(DMGetPointSF(dm, &sf));
2132     PetscCall(PetscSFGetGraph(sf, NULL, &Nl, &leaves, NULL));
2133     PetscCall(DMLabelGetStratumIS(label, dim-1, &dimIS));
2134     if (!dimIS) goto divide;
2135     PetscCall(ISGetLocalSize(dimIS, &numPoints));
2136     PetscCall(ISGetIndices(dimIS, &points));
2137     if (Nl > 0) {
2138       for (p = 0; p < numPoints; ++p) {
2139         const PetscInt point = points[p];
2140         PetscInt       val;
2141 
2142         PetscCall(PetscFindInt(point, Nl, leaves, &l));
2143         if (l >= 0) {
2144           PetscInt *closure = NULL;
2145           PetscInt  closureSize, cl;
2146 
2147           PetscCall(DMLabelGetValue(label, point, &val));
2148           PetscCheck((val == dim-1) || (val == shift2+dim-1), PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %" PetscInt_FMT " has label value %" PetscInt_FMT ", should be a fault face", point, val);
2149           PetscCall(DMLabelClearValue(label, point, val));
2150           PetscCall(DMLabelSetValue(label, point, shift3+val));
2151           PetscCall(DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
2152           for (cl = 2; cl < closureSize*2; cl += 2) {
2153             const PetscInt clp  = closure[cl];
2154 
2155             PetscCall(DMLabelGetValue(label, clp, &val));
2156             PetscCheck(val != -1, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %" PetscInt_FMT " is missing from label, but is in the closure of a fault face", point);
2157             PetscCall(DMLabelClearValue(label, clp, val));
2158             PetscCall(DMLabelSetValue(label, clp, shift3+val));
2159           }
2160           PetscCall(DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
2161         }
2162       }
2163     }
2164     PetscCall(ISRestoreIndices(dimIS, &points));
2165     PetscCall(ISDestroy(&dimIS));
2166   }
2167   divide:
2168   if (subpointIS) PetscCall(ISRestoreIndices(subpointIS, &subpoints));
2169   PetscCall(DMPlexLabelFaultHalo(dm, label));
2170   PetscCall(CheckFaultEdge_Private(dm, label));
2171   PetscFunctionReturn(0);
2172 }
2173 
2174 /* Check that no cell have all vertices on the fault */
2175 PetscErrorCode DMPlexCheckValidSubmesh_Private(DM dm, DMLabel label, DM subdm)
2176 {
2177   IS              subpointIS;
2178   const PetscInt *dmpoints;
2179   PetscInt        defaultValue, cStart, cEnd, c, vStart, vEnd;
2180 
2181   PetscFunctionBegin;
2182   if (!label) PetscFunctionReturn(0);
2183   PetscCall(DMLabelGetDefaultValue(label, &defaultValue));
2184   PetscCall(DMPlexGetSubpointIS(subdm, &subpointIS));
2185   if (!subpointIS) PetscFunctionReturn(0);
2186   PetscCall(DMPlexGetHeightStratum(subdm, 0, &cStart, &cEnd));
2187   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
2188   PetscCall(ISGetIndices(subpointIS, &dmpoints));
2189   for (c = cStart; c < cEnd; ++c) {
2190     PetscBool invalidCell = PETSC_TRUE;
2191     PetscInt *closure     = NULL;
2192     PetscInt  closureSize, cl;
2193 
2194     PetscCall(DMPlexGetTransitiveClosure(dm, dmpoints[c], PETSC_TRUE, &closureSize, &closure));
2195     for (cl = 0; cl < closureSize*2; cl += 2) {
2196       PetscInt value = 0;
2197 
2198       if ((closure[cl] < vStart) || (closure[cl] >= vEnd)) continue;
2199       PetscCall(DMLabelGetValue(label, closure[cl], &value));
2200       if (value == defaultValue) {invalidCell = PETSC_FALSE; break;}
2201     }
2202     PetscCall(DMPlexRestoreTransitiveClosure(dm, dmpoints[c], PETSC_TRUE, &closureSize, &closure));
2203     if (invalidCell) {
2204       PetscCall(ISRestoreIndices(subpointIS, &dmpoints));
2205       PetscCall(ISDestroy(&subpointIS));
2206       PetscCall(DMDestroy(&subdm));
2207       SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Ambiguous submesh. Cell %" PetscInt_FMT " has all of its vertices on the submesh.", dmpoints[c]);
2208     }
2209   }
2210   PetscCall(ISRestoreIndices(subpointIS, &dmpoints));
2211   PetscFunctionReturn(0);
2212 }
2213 
2214 /*@
2215   DMPlexCreateHybridMesh - Create a mesh with hybrid cells along an internal interface
2216 
2217   Collective on dm
2218 
2219   Input Parameters:
2220 + dm - The original DM
2221 . label - The label specifying the interface vertices
2222 . bdlabel - The optional label specifying the interface boundary vertices
2223 - bdvalue - Value of optional label specifying the interface boundary vertices
2224 
2225   Output Parameters:
2226 + hybridLabel - The label fully marking the interface, or NULL if no output is desired
2227 . splitLabel - The label containing the split points, or NULL if no output is desired
2228 . dmInterface - The new interface DM, or NULL
2229 - dmHybrid - The new DM with cohesive cells
2230 
2231   Note: The hybridLabel indicates what parts of the original mesh impinged on the on division surface. For points
2232   directly on the division surface, they are labeled with their dimension, so an edge 7 on the division surface would be
2233   7 (1) in hybridLabel. For points that impinge from the positive side, they are labeled with 100+dim, so an edge 6 with
2234   one vertex 3 on the surface would be 6 (101) and 3 (0) in hybridLabel. If an edge 9 from the negative side of the
2235   surface also hits vertex 3, it would be 9 (-101) in hybridLabel.
2236 
2237   The splitLabel indicates what points in the new hybrid mesh were the result of splitting points in the original
2238   mesh. The label value is +=100+dim for each point. For example, if two edges 10 and 14 in the hybrid resulting from
2239   splitting an edge in the original mesh, you would have 10 (101) and 14 (-101) in the splitLabel.
2240 
2241   The dmInterface is a DM built from the original division surface. It has a label which can be retrieved using
2242   DMPlexGetSubpointMap() which maps each point back to the point in the surface of the original mesh.
2243 
2244   Level: developer
2245 
2246 .seealso: `DMPlexConstructCohesiveCells()`, `DMPlexLabelCohesiveComplete()`, `DMPlexGetSubpointMap()`, `DMCreate()`
2247 @*/
2248 PetscErrorCode DMPlexCreateHybridMesh(DM dm, DMLabel label, DMLabel bdlabel, PetscInt bdvalue, DMLabel *hybridLabel, DMLabel *splitLabel, DM *dmInterface, DM *dmHybrid)
2249 {
2250   DM             idm;
2251   DMLabel        subpointMap, hlabel, slabel = NULL;
2252   PetscInt       dim;
2253 
2254   PetscFunctionBegin;
2255   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2256   if (bdlabel) PetscValidPointer(bdlabel, 3);
2257   if (hybridLabel) PetscValidPointer(hybridLabel, 4);
2258   if (splitLabel)  PetscValidPointer(splitLabel, 5);
2259   if (dmInterface) PetscValidPointer(dmInterface, 6);
2260   PetscValidPointer(dmHybrid, 7);
2261   PetscCall(DMGetDimension(dm, &dim));
2262   PetscCall(DMPlexCreateSubmesh(dm, label, 1, PETSC_FALSE, &idm));
2263   PetscCall(DMPlexCheckValidSubmesh_Private(dm, label, idm));
2264   PetscCall(DMPlexOrient(idm));
2265   PetscCall(DMPlexGetSubpointMap(idm, &subpointMap));
2266   PetscCall(DMLabelDuplicate(subpointMap, &hlabel));
2267   PetscCall(DMLabelClearStratum(hlabel, dim));
2268   if (splitLabel) {
2269     const char *name;
2270     char        sname[PETSC_MAX_PATH_LEN];
2271 
2272     PetscCall(PetscObjectGetName((PetscObject) hlabel, &name));
2273     PetscCall(PetscStrncpy(sname, name, PETSC_MAX_PATH_LEN));
2274     PetscCall(PetscStrcat(sname, " split"));
2275     PetscCall(DMLabelCreate(PETSC_COMM_SELF, sname, &slabel));
2276   }
2277   PetscCall(DMPlexLabelCohesiveComplete(dm, hlabel, bdlabel, bdvalue, PETSC_FALSE, idm));
2278   if (dmInterface) {*dmInterface = idm;}
2279   else             PetscCall(DMDestroy(&idm));
2280   PetscCall(DMPlexConstructCohesiveCells(dm, hlabel, slabel, dmHybrid));
2281   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, *dmHybrid));
2282   if (hybridLabel) *hybridLabel = hlabel;
2283   else             PetscCall(DMLabelDestroy(&hlabel));
2284   if (splitLabel)  *splitLabel  = slabel;
2285   {
2286     DM      cdm;
2287     DMLabel ctLabel;
2288 
2289     /* We need to somehow share the celltype label with the coordinate dm */
2290     PetscCall(DMGetCoordinateDM(*dmHybrid, &cdm));
2291     PetscCall(DMPlexGetCellTypeLabel(*dmHybrid, &ctLabel));
2292     PetscCall(DMSetLabel(cdm, ctLabel));
2293   }
2294   PetscFunctionReturn(0);
2295 }
2296 
2297 /* Here we need the explicit assumption that:
2298 
2299      For any marked cell, the marked vertices constitute a single face
2300 */
2301 static PetscErrorCode DMPlexMarkSubmesh_Uninterpolated(DM dm, DMLabel vertexLabel, PetscInt value, DMLabel subpointMap, PetscInt *numFaces, PetscInt *nFV, DM subdm)
2302 {
2303   IS               subvertexIS = NULL;
2304   const PetscInt  *subvertices;
2305   PetscInt        *pStart, *pEnd, pSize;
2306   PetscInt         depth, dim, d, numSubVerticesInitial = 0, v;
2307 
2308   PetscFunctionBegin;
2309   *numFaces = 0;
2310   *nFV      = 0;
2311   PetscCall(DMPlexGetDepth(dm, &depth));
2312   PetscCall(DMGetDimension(dm, &dim));
2313   pSize = PetscMax(depth, dim) + 1;
2314   PetscCall(PetscMalloc2(pSize, &pStart, pSize, &pEnd));
2315   for (d = 0; d <= depth; ++d) {
2316     PetscCall(DMPlexGetSimplexOrBoxCells(dm, depth-d, &pStart[d], &pEnd[d]));
2317   }
2318   /* Loop over initial vertices and mark all faces in the collective star() */
2319   if (vertexLabel) PetscCall(DMLabelGetStratumIS(vertexLabel, value, &subvertexIS));
2320   if (subvertexIS) {
2321     PetscCall(ISGetSize(subvertexIS, &numSubVerticesInitial));
2322     PetscCall(ISGetIndices(subvertexIS, &subvertices));
2323   }
2324   for (v = 0; v < numSubVerticesInitial; ++v) {
2325     const PetscInt vertex = subvertices[v];
2326     PetscInt      *star   = NULL;
2327     PetscInt       starSize, s, numCells = 0, c;
2328 
2329     PetscCall(DMPlexGetTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star));
2330     for (s = 0; s < starSize*2; s += 2) {
2331       const PetscInt point = star[s];
2332       if ((point >= pStart[depth]) && (point < pEnd[depth])) star[numCells++] = point;
2333     }
2334     for (c = 0; c < numCells; ++c) {
2335       const PetscInt cell    = star[c];
2336       PetscInt      *closure = NULL;
2337       PetscInt       closureSize, cl;
2338       PetscInt       cellLoc, numCorners = 0, faceSize = 0;
2339 
2340       PetscCall(DMLabelGetValue(subpointMap, cell, &cellLoc));
2341       if (cellLoc == 2) continue;
2342       PetscCheck(cellLoc < 0,PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Cell %" PetscInt_FMT " has dimension %" PetscInt_FMT " in the surface label", cell, cellLoc);
2343       PetscCall(DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
2344       for (cl = 0; cl < closureSize*2; cl += 2) {
2345         const PetscInt point = closure[cl];
2346         PetscInt       vertexLoc;
2347 
2348         if ((point >= pStart[0]) && (point < pEnd[0])) {
2349           ++numCorners;
2350           PetscCall(DMLabelGetValue(vertexLabel, point, &vertexLoc));
2351           if (vertexLoc == value) closure[faceSize++] = point;
2352         }
2353       }
2354       if (!(*nFV)) PetscCall(DMPlexGetNumFaceVertices(dm, dim, numCorners, nFV));
2355       PetscCheck(faceSize <= *nFV,PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Invalid submesh: Too many vertices %" PetscInt_FMT " of an element on the surface", faceSize);
2356       if (faceSize == *nFV) {
2357         const PetscInt *cells = NULL;
2358         PetscInt        numCells, nc;
2359 
2360         ++(*numFaces);
2361         for (cl = 0; cl < faceSize; ++cl) {
2362           PetscCall(DMLabelSetValue(subpointMap, closure[cl], 0));
2363         }
2364         PetscCall(DMPlexGetJoin(dm, faceSize, closure, &numCells, &cells));
2365         for (nc = 0; nc < numCells; ++nc) {
2366           PetscCall(DMLabelSetValue(subpointMap, cells[nc], 2));
2367         }
2368         PetscCall(DMPlexRestoreJoin(dm, faceSize, closure, &numCells, &cells));
2369       }
2370       PetscCall(DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
2371     }
2372     PetscCall(DMPlexRestoreTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star));
2373   }
2374   if (subvertexIS) {
2375     PetscCall(ISRestoreIndices(subvertexIS, &subvertices));
2376   }
2377   PetscCall(ISDestroy(&subvertexIS));
2378   PetscCall(PetscFree2(pStart, pEnd));
2379   PetscFunctionReturn(0);
2380 }
2381 
2382 static PetscErrorCode DMPlexMarkSubmesh_Interpolated(DM dm, DMLabel vertexLabel, PetscInt value, PetscBool markedFaces, DMLabel subpointMap, DM subdm)
2383 {
2384   IS               subvertexIS = NULL;
2385   const PetscInt  *subvertices;
2386   PetscInt        *pStart, *pEnd;
2387   PetscInt         dim, d, numSubVerticesInitial = 0, v;
2388 
2389   PetscFunctionBegin;
2390   PetscCall(DMGetDimension(dm, &dim));
2391   PetscCall(PetscMalloc2(dim+1, &pStart, dim+1, &pEnd));
2392   for (d = 0; d <= dim; ++d) {
2393     PetscCall(DMPlexGetSimplexOrBoxCells(dm, dim-d, &pStart[d], &pEnd[d]));
2394   }
2395   /* Loop over initial vertices and mark all faces in the collective star() */
2396   if (vertexLabel) {
2397     PetscCall(DMLabelGetStratumIS(vertexLabel, value, &subvertexIS));
2398     if (subvertexIS) {
2399       PetscCall(ISGetSize(subvertexIS, &numSubVerticesInitial));
2400       PetscCall(ISGetIndices(subvertexIS, &subvertices));
2401     }
2402   }
2403   for (v = 0; v < numSubVerticesInitial; ++v) {
2404     const PetscInt vertex = subvertices[v];
2405     PetscInt      *star   = NULL;
2406     PetscInt       starSize, s, numFaces = 0, f;
2407 
2408     PetscCall(DMPlexGetTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star));
2409     for (s = 0; s < starSize*2; s += 2) {
2410       const PetscInt point = star[s];
2411       PetscInt       faceLoc;
2412 
2413       if ((point >= pStart[dim-1]) && (point < pEnd[dim-1])) {
2414         if (markedFaces) {
2415           PetscCall(DMLabelGetValue(vertexLabel, point, &faceLoc));
2416           if (faceLoc < 0) continue;
2417         }
2418         star[numFaces++] = point;
2419       }
2420     }
2421     for (f = 0; f < numFaces; ++f) {
2422       const PetscInt face    = star[f];
2423       PetscInt      *closure = NULL;
2424       PetscInt       closureSize, c;
2425       PetscInt       faceLoc;
2426 
2427       PetscCall(DMLabelGetValue(subpointMap, face, &faceLoc));
2428       if (faceLoc == dim-1) continue;
2429       PetscCheck(faceLoc < 0,PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Face %" PetscInt_FMT " has dimension %" PetscInt_FMT " in the surface label", face, faceLoc);
2430       PetscCall(DMPlexGetTransitiveClosure(dm, face, PETSC_TRUE, &closureSize, &closure));
2431       for (c = 0; c < closureSize*2; c += 2) {
2432         const PetscInt point = closure[c];
2433         PetscInt       vertexLoc;
2434 
2435         if ((point >= pStart[0]) && (point < pEnd[0])) {
2436           PetscCall(DMLabelGetValue(vertexLabel, point, &vertexLoc));
2437           if (vertexLoc != value) break;
2438         }
2439       }
2440       if (c == closureSize*2) {
2441         const PetscInt *support;
2442         PetscInt        supportSize, s;
2443 
2444         for (c = 0; c < closureSize*2; c += 2) {
2445           const PetscInt point = closure[c];
2446 
2447           for (d = 0; d < dim; ++d) {
2448             if ((point >= pStart[d]) && (point < pEnd[d])) {
2449               PetscCall(DMLabelSetValue(subpointMap, point, d));
2450               break;
2451             }
2452           }
2453         }
2454         PetscCall(DMPlexGetSupportSize(dm, face, &supportSize));
2455         PetscCall(DMPlexGetSupport(dm, face, &support));
2456         for (s = 0; s < supportSize; ++s) {
2457           PetscCall(DMLabelSetValue(subpointMap, support[s], dim));
2458         }
2459       }
2460       PetscCall(DMPlexRestoreTransitiveClosure(dm, face, PETSC_TRUE, &closureSize, &closure));
2461     }
2462     PetscCall(DMPlexRestoreTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star));
2463   }
2464   if (subvertexIS) PetscCall(ISRestoreIndices(subvertexIS, &subvertices));
2465   PetscCall(ISDestroy(&subvertexIS));
2466   PetscCall(PetscFree2(pStart, pEnd));
2467   PetscFunctionReturn(0);
2468 }
2469 
2470 static PetscErrorCode DMPlexMarkCohesiveSubmesh_Uninterpolated(DM dm, PetscBool hasLagrange, const char labelname[], PetscInt value, DMLabel subpointMap, PetscInt *numFaces, PetscInt *nFV, PetscInt *subCells[], DM subdm)
2471 {
2472   DMLabel         label = NULL;
2473   const PetscInt *cone;
2474   PetscInt        dim, cMax, cEnd, c, subc = 0, p, coneSize = -1;
2475 
2476   PetscFunctionBegin;
2477   *numFaces = 0;
2478   *nFV = 0;
2479   if (labelname) PetscCall(DMGetLabel(dm, labelname, &label));
2480   *subCells = NULL;
2481   PetscCall(DMGetDimension(dm, &dim));
2482   PetscCall(DMPlexGetTensorPrismBounds_Internal(dm, dim, &cMax, &cEnd));
2483   if (cMax < 0) PetscFunctionReturn(0);
2484   if (label) {
2485     for (c = cMax; c < cEnd; ++c) {
2486       PetscInt val;
2487 
2488       PetscCall(DMLabelGetValue(label, c, &val));
2489       if (val == value) {
2490         ++(*numFaces);
2491         PetscCall(DMPlexGetConeSize(dm, c, &coneSize));
2492       }
2493     }
2494   } else {
2495     *numFaces = cEnd - cMax;
2496     PetscCall(DMPlexGetConeSize(dm, cMax, &coneSize));
2497   }
2498   PetscCall(PetscMalloc1(*numFaces *2, subCells));
2499   if (!(*numFaces)) PetscFunctionReturn(0);
2500   *nFV = hasLagrange ? coneSize/3 : coneSize/2;
2501   for (c = cMax; c < cEnd; ++c) {
2502     const PetscInt *cells;
2503     PetscInt        numCells;
2504 
2505     if (label) {
2506       PetscInt val;
2507 
2508       PetscCall(DMLabelGetValue(label, c, &val));
2509       if (val != value) continue;
2510     }
2511     PetscCall(DMPlexGetCone(dm, c, &cone));
2512     for (p = 0; p < *nFV; ++p) {
2513       PetscCall(DMLabelSetValue(subpointMap, cone[p], 0));
2514     }
2515     /* Negative face */
2516     PetscCall(DMPlexGetJoin(dm, *nFV, cone, &numCells, &cells));
2517     /* Not true in parallel
2518     PetscCheck(numCells == 2,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive cells should separate two cells"); */
2519     for (p = 0; p < numCells; ++p) {
2520       PetscCall(DMLabelSetValue(subpointMap, cells[p], 2));
2521       (*subCells)[subc++] = cells[p];
2522     }
2523     PetscCall(DMPlexRestoreJoin(dm, *nFV, cone, &numCells, &cells));
2524     /* Positive face is not included */
2525   }
2526   PetscFunctionReturn(0);
2527 }
2528 
2529 static PetscErrorCode DMPlexMarkCohesiveSubmesh_Interpolated(DM dm, DMLabel label, PetscInt value, DMLabel subpointMap, DM subdm)
2530 {
2531   PetscInt      *pStart, *pEnd;
2532   PetscInt       dim, cMax, cEnd, c, d;
2533 
2534   PetscFunctionBegin;
2535   PetscCall(DMGetDimension(dm, &dim));
2536   PetscCall(DMPlexGetTensorPrismBounds_Internal(dm, dim, &cMax, &cEnd));
2537   if (cMax < 0) PetscFunctionReturn(0);
2538   PetscCall(PetscMalloc2(dim+1,&pStart,dim+1,&pEnd));
2539   for (d = 0; d <= dim; ++d) PetscCall(DMPlexGetDepthStratum(dm, d, &pStart[d], &pEnd[d]));
2540   for (c = cMax; c < cEnd; ++c) {
2541     const PetscInt *cone;
2542     PetscInt       *closure = NULL;
2543     PetscInt        fconeSize, coneSize, closureSize, cl, val;
2544 
2545     if (label) {
2546       PetscCall(DMLabelGetValue(label, c, &val));
2547       if (val != value) continue;
2548     }
2549     PetscCall(DMPlexGetConeSize(dm, c, &coneSize));
2550     PetscCall(DMPlexGetCone(dm, c, &cone));
2551     PetscCall(DMPlexGetConeSize(dm, cone[0], &fconeSize));
2552     PetscCheck(coneSize == (fconeSize ? fconeSize : 1) + 2,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive cells should separate two cells");
2553     /* Negative face */
2554     PetscCall(DMPlexGetTransitiveClosure(dm, cone[0], PETSC_TRUE, &closureSize, &closure));
2555     for (cl = 0; cl < closureSize*2; cl += 2) {
2556       const PetscInt point = closure[cl];
2557 
2558       for (d = 0; d <= dim; ++d) {
2559         if ((point >= pStart[d]) && (point < pEnd[d])) {
2560           PetscCall(DMLabelSetValue(subpointMap, point, d));
2561           break;
2562         }
2563       }
2564     }
2565     PetscCall(DMPlexRestoreTransitiveClosure(dm, cone[0], PETSC_TRUE, &closureSize, &closure));
2566     /* Cells -- positive face is not included */
2567     for (cl = 0; cl < 1; ++cl) {
2568       const PetscInt *support;
2569       PetscInt        supportSize, s;
2570 
2571       PetscCall(DMPlexGetSupportSize(dm, cone[cl], &supportSize));
2572       /* PetscCheck(supportSize == 2,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive faces should separate two cells"); */
2573       PetscCall(DMPlexGetSupport(dm, cone[cl], &support));
2574       for (s = 0; s < supportSize; ++s) {
2575         PetscCall(DMLabelSetValue(subpointMap, support[s], dim));
2576       }
2577     }
2578   }
2579   PetscCall(PetscFree2(pStart, pEnd));
2580   PetscFunctionReturn(0);
2581 }
2582 
2583 static PetscErrorCode DMPlexGetFaceOrientation(DM dm, PetscInt cell, PetscInt numCorners, PetscInt indices[], PetscInt oppositeVertex, PetscInt origVertices[], PetscInt faceVertices[], PetscBool *posOriented)
2584 {
2585   MPI_Comm       comm;
2586   PetscBool      posOrient = PETSC_FALSE;
2587   const PetscInt debug     = 0;
2588   PetscInt       cellDim, faceSize, f;
2589 
2590   PetscFunctionBegin;
2591   PetscCall(PetscObjectGetComm((PetscObject)dm,&comm));
2592   PetscCall(DMGetDimension(dm, &cellDim));
2593   if (debug) PetscCall(PetscPrintf(comm, "cellDim: %" PetscInt_FMT " numCorners: %" PetscInt_FMT "\n", cellDim, numCorners));
2594 
2595   if (cellDim == 1 && numCorners == 2) {
2596     /* Triangle */
2597     faceSize  = numCorners-1;
2598     posOrient = !(oppositeVertex%2) ? PETSC_TRUE : PETSC_FALSE;
2599   } else if (cellDim == 2 && numCorners == 3) {
2600     /* Triangle */
2601     faceSize  = numCorners-1;
2602     posOrient = !(oppositeVertex%2) ? PETSC_TRUE : PETSC_FALSE;
2603   } else if (cellDim == 3 && numCorners == 4) {
2604     /* Tetrahedron */
2605     faceSize  = numCorners-1;
2606     posOrient = (oppositeVertex%2) ? PETSC_TRUE : PETSC_FALSE;
2607   } else if (cellDim == 1 && numCorners == 3) {
2608     /* Quadratic line */
2609     faceSize  = 1;
2610     posOrient = PETSC_TRUE;
2611   } else if (cellDim == 2 && numCorners == 4) {
2612     /* Quads */
2613     faceSize = 2;
2614     if ((indices[1] > indices[0]) && (indices[1] - indices[0] == 1)) {
2615       posOrient = PETSC_TRUE;
2616     } else if ((indices[0] == 3) && (indices[1] == 0)) {
2617       posOrient = PETSC_TRUE;
2618     } else {
2619       if (((indices[0] > indices[1]) && (indices[0] - indices[1] == 1)) || ((indices[0] == 0) && (indices[1] == 3))) {
2620         posOrient = PETSC_FALSE;
2621       } else SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid quad crossedge");
2622     }
2623   } else if (cellDim == 2 && numCorners == 6) {
2624     /* Quadratic triangle (I hate this) */
2625     /* Edges are determined by the first 2 vertices (corners of edges) */
2626     const PetscInt faceSizeTri = 3;
2627     PetscInt       sortedIndices[3], i, iFace;
2628     PetscBool      found                    = PETSC_FALSE;
2629     PetscInt       faceVerticesTriSorted[9] = {
2630       0, 3,  4, /* bottom */
2631       1, 4,  5, /* right */
2632       2, 3,  5, /* left */
2633     };
2634     PetscInt       faceVerticesTri[9] = {
2635       0, 3,  4, /* bottom */
2636       1, 4,  5, /* right */
2637       2, 5,  3, /* left */
2638     };
2639 
2640     for (i = 0; i < faceSizeTri; ++i) sortedIndices[i] = indices[i];
2641     PetscCall(PetscSortInt(faceSizeTri, sortedIndices));
2642     for (iFace = 0; iFace < 3; ++iFace) {
2643       const PetscInt ii = iFace*faceSizeTri;
2644       PetscInt       fVertex, cVertex;
2645 
2646       if ((sortedIndices[0] == faceVerticesTriSorted[ii+0]) &&
2647           (sortedIndices[1] == faceVerticesTriSorted[ii+1])) {
2648         for (fVertex = 0; fVertex < faceSizeTri; ++fVertex) {
2649           for (cVertex = 0; cVertex < faceSizeTri; ++cVertex) {
2650             if (indices[cVertex] == faceVerticesTri[ii+fVertex]) {
2651               faceVertices[fVertex] = origVertices[cVertex];
2652               break;
2653             }
2654           }
2655         }
2656         found = PETSC_TRUE;
2657         break;
2658       }
2659     }
2660     PetscCheck(found,comm, PETSC_ERR_ARG_WRONG, "Invalid tri crossface");
2661     if (posOriented) *posOriented = PETSC_TRUE;
2662     PetscFunctionReturn(0);
2663   } else if (cellDim == 2 && numCorners == 9) {
2664     /* Quadratic quad (I hate this) */
2665     /* Edges are determined by the first 2 vertices (corners of edges) */
2666     const PetscInt faceSizeQuad = 3;
2667     PetscInt       sortedIndices[3], i, iFace;
2668     PetscBool      found                      = PETSC_FALSE;
2669     PetscInt       faceVerticesQuadSorted[12] = {
2670       0, 1,  4, /* bottom */
2671       1, 2,  5, /* right */
2672       2, 3,  6, /* top */
2673       0, 3,  7, /* left */
2674     };
2675     PetscInt       faceVerticesQuad[12] = {
2676       0, 1,  4, /* bottom */
2677       1, 2,  5, /* right */
2678       2, 3,  6, /* top */
2679       3, 0,  7, /* left */
2680     };
2681 
2682     for (i = 0; i < faceSizeQuad; ++i) sortedIndices[i] = indices[i];
2683     PetscCall(PetscSortInt(faceSizeQuad, sortedIndices));
2684     for (iFace = 0; iFace < 4; ++iFace) {
2685       const PetscInt ii = iFace*faceSizeQuad;
2686       PetscInt       fVertex, cVertex;
2687 
2688       if ((sortedIndices[0] == faceVerticesQuadSorted[ii+0]) &&
2689           (sortedIndices[1] == faceVerticesQuadSorted[ii+1])) {
2690         for (fVertex = 0; fVertex < faceSizeQuad; ++fVertex) {
2691           for (cVertex = 0; cVertex < faceSizeQuad; ++cVertex) {
2692             if (indices[cVertex] == faceVerticesQuad[ii+fVertex]) {
2693               faceVertices[fVertex] = origVertices[cVertex];
2694               break;
2695             }
2696           }
2697         }
2698         found = PETSC_TRUE;
2699         break;
2700       }
2701     }
2702     PetscCheck(found,comm, PETSC_ERR_ARG_WRONG, "Invalid quad crossface");
2703     if (posOriented) *posOriented = PETSC_TRUE;
2704     PetscFunctionReturn(0);
2705   } else if (cellDim == 3 && numCorners == 8) {
2706     /* Hexes
2707        A hex is two oriented quads with the normal of the first
2708        pointing up at the second.
2709 
2710           7---6
2711          /|  /|
2712         4---5 |
2713         | 1-|-2
2714         |/  |/
2715         0---3
2716 
2717         Faces are determined by the first 4 vertices (corners of faces) */
2718     const PetscInt faceSizeHex = 4;
2719     PetscInt       sortedIndices[4], i, iFace;
2720     PetscBool      found                     = PETSC_FALSE;
2721     PetscInt       faceVerticesHexSorted[24] = {
2722       0, 1, 2, 3,  /* bottom */
2723       4, 5, 6, 7,  /* top */
2724       0, 3, 4, 5,  /* front */
2725       2, 3, 5, 6,  /* right */
2726       1, 2, 6, 7,  /* back */
2727       0, 1, 4, 7,  /* left */
2728     };
2729     PetscInt       faceVerticesHex[24] = {
2730       1, 2, 3, 0,  /* bottom */
2731       4, 5, 6, 7,  /* top */
2732       0, 3, 5, 4,  /* front */
2733       3, 2, 6, 5,  /* right */
2734       2, 1, 7, 6,  /* back */
2735       1, 0, 4, 7,  /* left */
2736     };
2737 
2738     for (i = 0; i < faceSizeHex; ++i) sortedIndices[i] = indices[i];
2739     PetscCall(PetscSortInt(faceSizeHex, sortedIndices));
2740     for (iFace = 0; iFace < 6; ++iFace) {
2741       const PetscInt ii = iFace*faceSizeHex;
2742       PetscInt       fVertex, cVertex;
2743 
2744       if ((sortedIndices[0] == faceVerticesHexSorted[ii+0]) &&
2745           (sortedIndices[1] == faceVerticesHexSorted[ii+1]) &&
2746           (sortedIndices[2] == faceVerticesHexSorted[ii+2]) &&
2747           (sortedIndices[3] == faceVerticesHexSorted[ii+3])) {
2748         for (fVertex = 0; fVertex < faceSizeHex; ++fVertex) {
2749           for (cVertex = 0; cVertex < faceSizeHex; ++cVertex) {
2750             if (indices[cVertex] == faceVerticesHex[ii+fVertex]) {
2751               faceVertices[fVertex] = origVertices[cVertex];
2752               break;
2753             }
2754           }
2755         }
2756         found = PETSC_TRUE;
2757         break;
2758       }
2759     }
2760     PetscCheck(found,comm, PETSC_ERR_ARG_WRONG, "Invalid hex crossface");
2761     if (posOriented) *posOriented = PETSC_TRUE;
2762     PetscFunctionReturn(0);
2763   } else if (cellDim == 3 && numCorners == 10) {
2764     /* Quadratic tet */
2765     /* Faces are determined by the first 3 vertices (corners of faces) */
2766     const PetscInt faceSizeTet = 6;
2767     PetscInt       sortedIndices[6], i, iFace;
2768     PetscBool      found                     = PETSC_FALSE;
2769     PetscInt       faceVerticesTetSorted[24] = {
2770       0, 1, 2,  6, 7, 8, /* bottom */
2771       0, 3, 4,  6, 7, 9,  /* front */
2772       1, 4, 5,  7, 8, 9,  /* right */
2773       2, 3, 5,  6, 8, 9,  /* left */
2774     };
2775     PetscInt       faceVerticesTet[24] = {
2776       0, 1, 2,  6, 7, 8, /* bottom */
2777       0, 4, 3,  6, 7, 9,  /* front */
2778       1, 5, 4,  7, 8, 9,  /* right */
2779       2, 3, 5,  8, 6, 9,  /* left */
2780     };
2781 
2782     for (i = 0; i < faceSizeTet; ++i) sortedIndices[i] = indices[i];
2783     PetscCall(PetscSortInt(faceSizeTet, sortedIndices));
2784     for (iFace=0; iFace < 4; ++iFace) {
2785       const PetscInt ii = iFace*faceSizeTet;
2786       PetscInt       fVertex, cVertex;
2787 
2788       if ((sortedIndices[0] == faceVerticesTetSorted[ii+0]) &&
2789           (sortedIndices[1] == faceVerticesTetSorted[ii+1]) &&
2790           (sortedIndices[2] == faceVerticesTetSorted[ii+2]) &&
2791           (sortedIndices[3] == faceVerticesTetSorted[ii+3])) {
2792         for (fVertex = 0; fVertex < faceSizeTet; ++fVertex) {
2793           for (cVertex = 0; cVertex < faceSizeTet; ++cVertex) {
2794             if (indices[cVertex] == faceVerticesTet[ii+fVertex]) {
2795               faceVertices[fVertex] = origVertices[cVertex];
2796               break;
2797             }
2798           }
2799         }
2800         found = PETSC_TRUE;
2801         break;
2802       }
2803     }
2804     PetscCheck(found,comm, PETSC_ERR_ARG_WRONG, "Invalid tet crossface");
2805     if (posOriented) *posOriented = PETSC_TRUE;
2806     PetscFunctionReturn(0);
2807   } else if (cellDim == 3 && numCorners == 27) {
2808     /* Quadratic hexes (I hate this)
2809        A hex is two oriented quads with the normal of the first
2810        pointing up at the second.
2811 
2812          7---6
2813         /|  /|
2814        4---5 |
2815        | 3-|-2
2816        |/  |/
2817        0---1
2818 
2819        Faces are determined by the first 4 vertices (corners of faces) */
2820     const PetscInt faceSizeQuadHex = 9;
2821     PetscInt       sortedIndices[9], i, iFace;
2822     PetscBool      found                         = PETSC_FALSE;
2823     PetscInt       faceVerticesQuadHexSorted[54] = {
2824       0, 1, 2, 3,  8, 9, 10, 11,  24, /* bottom */
2825       4, 5, 6, 7,  12, 13, 14, 15,  25, /* top */
2826       0, 1, 4, 5,  8, 12, 16, 17,  22, /* front */
2827       1, 2, 5, 6,  9, 13, 17, 18,  21, /* right */
2828       2, 3, 6, 7,  10, 14, 18, 19,  23, /* back */
2829       0, 3, 4, 7,  11, 15, 16, 19,  20, /* left */
2830     };
2831     PetscInt       faceVerticesQuadHex[54] = {
2832       3, 2, 1, 0,  10, 9, 8, 11,  24, /* bottom */
2833       4, 5, 6, 7,  12, 13, 14, 15,  25, /* top */
2834       0, 1, 5, 4,  8, 17, 12, 16,  22, /* front */
2835       1, 2, 6, 5,  9, 18, 13, 17,  21, /* right */
2836       2, 3, 7, 6,  10, 19, 14, 18,  23, /* back */
2837       3, 0, 4, 7,  11, 16, 15, 19,  20 /* left */
2838     };
2839 
2840     for (i = 0; i < faceSizeQuadHex; ++i) sortedIndices[i] = indices[i];
2841     PetscCall(PetscSortInt(faceSizeQuadHex, sortedIndices));
2842     for (iFace = 0; iFace < 6; ++iFace) {
2843       const PetscInt ii = iFace*faceSizeQuadHex;
2844       PetscInt       fVertex, cVertex;
2845 
2846       if ((sortedIndices[0] == faceVerticesQuadHexSorted[ii+0]) &&
2847           (sortedIndices[1] == faceVerticesQuadHexSorted[ii+1]) &&
2848           (sortedIndices[2] == faceVerticesQuadHexSorted[ii+2]) &&
2849           (sortedIndices[3] == faceVerticesQuadHexSorted[ii+3])) {
2850         for (fVertex = 0; fVertex < faceSizeQuadHex; ++fVertex) {
2851           for (cVertex = 0; cVertex < faceSizeQuadHex; ++cVertex) {
2852             if (indices[cVertex] == faceVerticesQuadHex[ii+fVertex]) {
2853               faceVertices[fVertex] = origVertices[cVertex];
2854               break;
2855             }
2856           }
2857         }
2858         found = PETSC_TRUE;
2859         break;
2860       }
2861     }
2862     PetscCheck(found,comm, PETSC_ERR_ARG_WRONG, "Invalid hex crossface");
2863     if (posOriented) *posOriented = PETSC_TRUE;
2864     PetscFunctionReturn(0);
2865   } else SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Unknown cell type for faceOrientation().");
2866   if (!posOrient) {
2867     if (debug) PetscCall(PetscPrintf(comm, "  Reversing initial face orientation\n"));
2868     for (f = 0; f < faceSize; ++f) faceVertices[f] = origVertices[faceSize-1 - f];
2869   } else {
2870     if (debug) PetscCall(PetscPrintf(comm, "  Keeping initial face orientation\n"));
2871     for (f = 0; f < faceSize; ++f) faceVertices[f] = origVertices[f];
2872   }
2873   if (posOriented) *posOriented = posOrient;
2874   PetscFunctionReturn(0);
2875 }
2876 
2877 /*@
2878   DMPlexGetOrientedFace - Given a cell and a face, as a set of vertices, return the oriented face, as a set of vertices,
2879   in faceVertices. The orientation is such that the face normal points out of the cell
2880 
2881   Not collective
2882 
2883   Input Parameters:
2884 + dm           - The original mesh
2885 . cell         - The cell mesh point
2886 . faceSize     - The number of vertices on the face
2887 . face         - The face vertices
2888 . numCorners   - The number of vertices on the cell
2889 . indices      - Local numbering of face vertices in cell cone
2890 - origVertices - Original face vertices
2891 
2892   Output Parameters:
2893 + faceVertices - The face vertices properly oriented
2894 - posOriented  - PETSC_TRUE if the face was oriented with outward normal
2895 
2896   Level: developer
2897 
2898 .seealso: `DMPlexGetCone()`
2899 @*/
2900 PetscErrorCode DMPlexGetOrientedFace(DM dm, PetscInt cell, PetscInt faceSize, const PetscInt face[], PetscInt numCorners, PetscInt indices[], PetscInt origVertices[], PetscInt faceVertices[], PetscBool *posOriented)
2901 {
2902   const PetscInt *cone = NULL;
2903   PetscInt        coneSize, v, f, v2;
2904   PetscInt        oppositeVertex = -1;
2905 
2906   PetscFunctionBegin;
2907   PetscCall(DMPlexGetConeSize(dm, cell, &coneSize));
2908   PetscCall(DMPlexGetCone(dm, cell, &cone));
2909   for (v = 0, v2 = 0; v < coneSize; ++v) {
2910     PetscBool found = PETSC_FALSE;
2911 
2912     for (f = 0; f < faceSize; ++f) {
2913       if (face[f] == cone[v]) {
2914         found = PETSC_TRUE; break;
2915       }
2916     }
2917     if (found) {
2918       indices[v2]      = v;
2919       origVertices[v2] = cone[v];
2920       ++v2;
2921     } else {
2922       oppositeVertex = v;
2923     }
2924   }
2925   PetscCall(DMPlexGetFaceOrientation(dm, cell, numCorners, indices, oppositeVertex, origVertices, faceVertices, posOriented));
2926   PetscFunctionReturn(0);
2927 }
2928 
2929 /*
2930   DMPlexInsertFace_Internal - Puts a face into the mesh
2931 
2932   Not collective
2933 
2934   Input Parameters:
2935   + dm              - The DMPlex
2936   . numFaceVertex   - The number of vertices in the face
2937   . faceVertices    - The vertices in the face for dm
2938   . subfaceVertices - The vertices in the face for subdm
2939   . numCorners      - The number of vertices in the cell
2940   . cell            - A cell in dm containing the face
2941   . subcell         - A cell in subdm containing the face
2942   . firstFace       - First face in the mesh
2943   - newFacePoint    - Next face in the mesh
2944 
2945   Output Parameters:
2946   . newFacePoint - Contains next face point number on input, updated on output
2947 
2948   Level: developer
2949 */
2950 static PetscErrorCode DMPlexInsertFace_Internal(DM dm, DM subdm, PetscInt numFaceVertices, const PetscInt faceVertices[], const PetscInt subfaceVertices[], PetscInt numCorners, PetscInt cell, PetscInt subcell, PetscInt firstFace, PetscInt *newFacePoint)
2951 {
2952   MPI_Comm        comm;
2953   DM_Plex        *submesh = (DM_Plex*) subdm->data;
2954   const PetscInt *faces;
2955   PetscInt        numFaces, coneSize;
2956 
2957   PetscFunctionBegin;
2958   PetscCall(PetscObjectGetComm((PetscObject)dm,&comm));
2959   PetscCall(DMPlexGetConeSize(subdm, subcell, &coneSize));
2960   PetscCheck(coneSize == 1,comm, PETSC_ERR_ARG_OUTOFRANGE, "Cone size of cell %" PetscInt_FMT " is %" PetscInt_FMT " != 1", cell, coneSize);
2961 #if 0
2962   /* Cannot use this because support() has not been constructed yet */
2963   PetscCall(DMPlexGetJoin(subdm, numFaceVertices, subfaceVertices, &numFaces, &faces));
2964 #else
2965   {
2966     PetscInt f;
2967 
2968     numFaces = 0;
2969     PetscCall(DMGetWorkArray(subdm, 1, MPIU_INT, (void **) &faces));
2970     for (f = firstFace; f < *newFacePoint; ++f) {
2971       PetscInt dof, off, d;
2972 
2973       PetscCall(PetscSectionGetDof(submesh->coneSection, f, &dof));
2974       PetscCall(PetscSectionGetOffset(submesh->coneSection, f, &off));
2975       /* Yes, I know this is quadratic, but I expect the sizes to be <5 */
2976       for (d = 0; d < dof; ++d) {
2977         const PetscInt p = submesh->cones[off+d];
2978         PetscInt       v;
2979 
2980         for (v = 0; v < numFaceVertices; ++v) {
2981           if (subfaceVertices[v] == p) break;
2982         }
2983         if (v == numFaceVertices) break;
2984       }
2985       if (d == dof) {
2986         numFaces               = 1;
2987         ((PetscInt*) faces)[0] = f;
2988       }
2989     }
2990   }
2991 #endif
2992   PetscCheck(numFaces <= 1,comm, PETSC_ERR_ARG_WRONG, "Vertex set had %" PetscInt_FMT " faces, not one", numFaces);
2993   else if (numFaces == 1) {
2994     /* Add the other cell neighbor for this face */
2995     PetscCall(DMPlexSetCone(subdm, subcell, faces));
2996   } else {
2997     PetscInt *indices, *origVertices, *orientedVertices, *orientedSubVertices, v, ov;
2998     PetscBool posOriented;
2999 
3000     PetscCall(DMGetWorkArray(subdm, 4*numFaceVertices * sizeof(PetscInt), MPIU_INT, &orientedVertices));
3001     origVertices        = &orientedVertices[numFaceVertices];
3002     indices             = &orientedVertices[numFaceVertices*2];
3003     orientedSubVertices = &orientedVertices[numFaceVertices*3];
3004     PetscCall(DMPlexGetOrientedFace(dm, cell, numFaceVertices, faceVertices, numCorners, indices, origVertices, orientedVertices, &posOriented));
3005     /* TODO: I know that routine should return a permutation, not the indices */
3006     for (v = 0; v < numFaceVertices; ++v) {
3007       const PetscInt vertex = faceVertices[v], subvertex = subfaceVertices[v];
3008       for (ov = 0; ov < numFaceVertices; ++ov) {
3009         if (orientedVertices[ov] == vertex) {
3010           orientedSubVertices[ov] = subvertex;
3011           break;
3012         }
3013       }
3014       PetscCheck(ov != numFaceVertices,comm, PETSC_ERR_PLIB, "Could not find face vertex %" PetscInt_FMT " in orientated set", vertex);
3015     }
3016     PetscCall(DMPlexSetCone(subdm, *newFacePoint, orientedSubVertices));
3017     PetscCall(DMPlexSetCone(subdm, subcell, newFacePoint));
3018     PetscCall(DMRestoreWorkArray(subdm, 4*numFaceVertices * sizeof(PetscInt), MPIU_INT, &orientedVertices));
3019     ++(*newFacePoint);
3020   }
3021 #if 0
3022   PetscCall(DMPlexRestoreJoin(subdm, numFaceVertices, subfaceVertices, &numFaces, &faces));
3023 #else
3024   PetscCall(DMRestoreWorkArray(subdm, 1, MPIU_INT, (void **) &faces));
3025 #endif
3026   PetscFunctionReturn(0);
3027 }
3028 
3029 static PetscErrorCode DMPlexCreateSubmesh_Uninterpolated(DM dm, DMLabel vertexLabel, PetscInt value, DM subdm)
3030 {
3031   MPI_Comm        comm;
3032   DMLabel         subpointMap;
3033   IS              subvertexIS,  subcellIS;
3034   const PetscInt *subVertices, *subCells;
3035   PetscInt        numSubVertices, firstSubVertex, numSubCells;
3036   PetscInt       *subface, maxConeSize, numSubFaces = 0, firstSubFace, newFacePoint, nFV = 0;
3037   PetscInt        vStart, vEnd, c, f;
3038 
3039   PetscFunctionBegin;
3040   PetscCall(PetscObjectGetComm((PetscObject)dm,&comm));
3041   /* Create subpointMap which marks the submesh */
3042   PetscCall(DMLabelCreate(PETSC_COMM_SELF, "subpoint_map", &subpointMap));
3043   PetscCall(DMPlexSetSubpointMap(subdm, subpointMap));
3044   PetscCall(DMLabelDestroy(&subpointMap));
3045   if (vertexLabel) PetscCall(DMPlexMarkSubmesh_Uninterpolated(dm, vertexLabel, value, subpointMap, &numSubFaces, &nFV, subdm));
3046   /* Setup chart */
3047   PetscCall(DMLabelGetStratumSize(subpointMap, 0, &numSubVertices));
3048   PetscCall(DMLabelGetStratumSize(subpointMap, 2, &numSubCells));
3049   PetscCall(DMPlexSetChart(subdm, 0, numSubCells+numSubFaces+numSubVertices));
3050   PetscCall(DMPlexSetVTKCellHeight(subdm, 1));
3051   /* Set cone sizes */
3052   firstSubVertex = numSubCells;
3053   firstSubFace   = numSubCells+numSubVertices;
3054   newFacePoint   = firstSubFace;
3055   PetscCall(DMLabelGetStratumIS(subpointMap, 0, &subvertexIS));
3056   if (subvertexIS) PetscCall(ISGetIndices(subvertexIS, &subVertices));
3057   PetscCall(DMLabelGetStratumIS(subpointMap, 2, &subcellIS));
3058   if (subcellIS) PetscCall(ISGetIndices(subcellIS, &subCells));
3059   for (c = 0; c < numSubCells; ++c) {
3060     PetscCall(DMPlexSetConeSize(subdm, c, 1));
3061   }
3062   for (f = firstSubFace; f < firstSubFace+numSubFaces; ++f) {
3063     PetscCall(DMPlexSetConeSize(subdm, f, nFV));
3064   }
3065   PetscCall(DMSetUp(subdm));
3066   /* Create face cones */
3067   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
3068   PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, NULL));
3069   PetscCall(DMGetWorkArray(subdm, maxConeSize, MPIU_INT, (void**) &subface));
3070   for (c = 0; c < numSubCells; ++c) {
3071     const PetscInt cell    = subCells[c];
3072     const PetscInt subcell = c;
3073     PetscInt      *closure = NULL;
3074     PetscInt       closureSize, cl, numCorners = 0, faceSize = 0;
3075 
3076     PetscCall(DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
3077     for (cl = 0; cl < closureSize*2; cl += 2) {
3078       const PetscInt point = closure[cl];
3079       PetscInt       subVertex;
3080 
3081       if ((point >= vStart) && (point < vEnd)) {
3082         ++numCorners;
3083         PetscCall(PetscFindInt(point, numSubVertices, subVertices, &subVertex));
3084         if (subVertex >= 0) {
3085           closure[faceSize] = point;
3086           subface[faceSize] = firstSubVertex+subVertex;
3087           ++faceSize;
3088         }
3089       }
3090     }
3091     PetscCheck(faceSize <= nFV,comm, PETSC_ERR_ARG_WRONG, "Invalid submesh: Too many vertices %" PetscInt_FMT " of an element on the surface", faceSize);
3092     if (faceSize == nFV) {
3093       PetscCall(DMPlexInsertFace_Internal(dm, subdm, faceSize, closure, subface, numCorners, cell, subcell, firstSubFace, &newFacePoint));
3094     }
3095     PetscCall(DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
3096   }
3097   PetscCall(DMRestoreWorkArray(subdm, maxConeSize, MPIU_INT, (void**) &subface));
3098   PetscCall(DMPlexSymmetrize(subdm));
3099   PetscCall(DMPlexStratify(subdm));
3100   /* Build coordinates */
3101   {
3102     PetscSection coordSection, subCoordSection;
3103     Vec          coordinates, subCoordinates;
3104     PetscScalar *coords, *subCoords;
3105     PetscInt     numComp, coordSize, v;
3106     const char  *name;
3107 
3108     PetscCall(DMGetCoordinateSection(dm, &coordSection));
3109     PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
3110     PetscCall(DMGetCoordinateSection(subdm, &subCoordSection));
3111     PetscCall(PetscSectionSetNumFields(subCoordSection, 1));
3112     PetscCall(PetscSectionGetFieldComponents(coordSection, 0, &numComp));
3113     PetscCall(PetscSectionSetFieldComponents(subCoordSection, 0, numComp));
3114     PetscCall(PetscSectionSetChart(subCoordSection, firstSubVertex, firstSubVertex+numSubVertices));
3115     for (v = 0; v < numSubVertices; ++v) {
3116       const PetscInt vertex    = subVertices[v];
3117       const PetscInt subvertex = firstSubVertex+v;
3118       PetscInt       dof;
3119 
3120       PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3121       PetscCall(PetscSectionSetDof(subCoordSection, subvertex, dof));
3122       PetscCall(PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof));
3123     }
3124     PetscCall(PetscSectionSetUp(subCoordSection));
3125     PetscCall(PetscSectionGetStorageSize(subCoordSection, &coordSize));
3126     PetscCall(VecCreate(PETSC_COMM_SELF, &subCoordinates));
3127     PetscCall(PetscObjectGetName((PetscObject)coordinates,&name));
3128     PetscCall(PetscObjectSetName((PetscObject)subCoordinates,name));
3129     PetscCall(VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE));
3130     PetscCall(VecSetType(subCoordinates,VECSTANDARD));
3131     if (coordSize) {
3132       PetscCall(VecGetArray(coordinates,    &coords));
3133       PetscCall(VecGetArray(subCoordinates, &subCoords));
3134       for (v = 0; v < numSubVertices; ++v) {
3135         const PetscInt vertex    = subVertices[v];
3136         const PetscInt subvertex = firstSubVertex+v;
3137         PetscInt       dof, off, sdof, soff, d;
3138 
3139         PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3140         PetscCall(PetscSectionGetOffset(coordSection, vertex, &off));
3141         PetscCall(PetscSectionGetDof(subCoordSection, subvertex, &sdof));
3142         PetscCall(PetscSectionGetOffset(subCoordSection, subvertex, &soff));
3143         PetscCheck(dof == sdof,comm, PETSC_ERR_PLIB, "Coordinate dimension %" PetscInt_FMT " on subvertex %" PetscInt_FMT ", vertex %" PetscInt_FMT " should be %" PetscInt_FMT, sdof, subvertex, vertex, dof);
3144         for (d = 0; d < dof; ++d) subCoords[soff+d] = coords[off+d];
3145       }
3146       PetscCall(VecRestoreArray(coordinates,    &coords));
3147       PetscCall(VecRestoreArray(subCoordinates, &subCoords));
3148     }
3149     PetscCall(DMSetCoordinatesLocal(subdm, subCoordinates));
3150     PetscCall(VecDestroy(&subCoordinates));
3151   }
3152   /* Cleanup */
3153   if (subvertexIS) PetscCall(ISRestoreIndices(subvertexIS, &subVertices));
3154   PetscCall(ISDestroy(&subvertexIS));
3155   if (subcellIS) PetscCall(ISRestoreIndices(subcellIS, &subCells));
3156   PetscCall(ISDestroy(&subcellIS));
3157   PetscFunctionReturn(0);
3158 }
3159 
3160 /* TODO: Fix this to properly propogate up error conditions it may find */
3161 static inline PetscInt DMPlexFilterPoint_Internal(PetscInt point, PetscInt firstSubPoint, PetscInt numSubPoints, const PetscInt subPoints[])
3162 {
3163   PetscInt       subPoint;
3164   PetscErrorCode ierr;
3165 
3166   ierr = PetscFindInt(point, numSubPoints, subPoints, &subPoint); if (ierr) return -1;
3167   return subPoint < 0 ? subPoint : firstSubPoint+subPoint;
3168 }
3169 
3170 static PetscErrorCode DMPlexFilterLabels_Internal(DM dm, const PetscInt numSubPoints[], const PetscInt *subpoints[], const PetscInt firstSubPoint[], DM subdm)
3171 {
3172   PetscInt       Nl, l, d;
3173 
3174   PetscFunctionBegin;
3175   PetscCall(DMGetNumLabels(dm, &Nl));
3176   for (l = 0; l < Nl; ++l) {
3177     DMLabel         label, newlabel;
3178     const char     *lname;
3179     PetscBool       isDepth, isDim, isCelltype, isVTK;
3180     IS              valueIS;
3181     const PetscInt *values;
3182     PetscInt        Nv, v;
3183 
3184     PetscCall(DMGetLabelName(dm, l, &lname));
3185     PetscCall(PetscStrcmp(lname, "depth", &isDepth));
3186     PetscCall(PetscStrcmp(lname, "dim", &isDim));
3187     PetscCall(PetscStrcmp(lname, "celltype", &isCelltype));
3188     PetscCall(PetscStrcmp(lname, "vtk", &isVTK));
3189     if (isDepth || isDim || isCelltype || isVTK) continue;
3190     PetscCall(DMCreateLabel(subdm, lname));
3191     PetscCall(DMGetLabel(dm, lname, &label));
3192     PetscCall(DMGetLabel(subdm, lname, &newlabel));
3193     PetscCall(DMLabelGetDefaultValue(label, &v));
3194     PetscCall(DMLabelSetDefaultValue(newlabel, v));
3195     PetscCall(DMLabelGetValueIS(label, &valueIS));
3196     PetscCall(ISGetLocalSize(valueIS, &Nv));
3197     PetscCall(ISGetIndices(valueIS, &values));
3198     for (v = 0; v < Nv; ++v) {
3199       IS              pointIS;
3200       const PetscInt *points;
3201       PetscInt        Np, p;
3202 
3203       PetscCall(DMLabelGetStratumIS(label, values[v], &pointIS));
3204       PetscCall(ISGetLocalSize(pointIS, &Np));
3205       PetscCall(ISGetIndices(pointIS, &points));
3206       for (p = 0; p < Np; ++p) {
3207         const PetscInt point = points[p];
3208         PetscInt       subp;
3209 
3210         PetscCall(DMPlexGetPointDepth(dm, point, &d));
3211         subp = DMPlexFilterPoint_Internal(point, firstSubPoint[d], numSubPoints[d], subpoints[d]);
3212         if (subp >= 0) PetscCall(DMLabelSetValue(newlabel, subp, values[v]));
3213       }
3214       PetscCall(ISRestoreIndices(pointIS, &points));
3215       PetscCall(ISDestroy(&pointIS));
3216     }
3217     PetscCall(ISRestoreIndices(valueIS, &values));
3218     PetscCall(ISDestroy(&valueIS));
3219   }
3220   PetscFunctionReturn(0);
3221 }
3222 
3223 static PetscErrorCode DMPlexCreateSubmeshGeneric_Interpolated(DM dm, DMLabel label, PetscInt value, PetscBool markedFaces, PetscBool isCohesive, PetscInt cellHeight, DM subdm)
3224 {
3225   MPI_Comm         comm;
3226   DMLabel          subpointMap;
3227   IS              *subpointIS;
3228   const PetscInt **subpoints;
3229   PetscInt        *numSubPoints, *firstSubPoint, *coneNew, *orntNew;
3230   PetscInt         totSubPoints = 0, maxConeSize, dim, p, d, v;
3231   PetscMPIInt      rank;
3232 
3233   PetscFunctionBegin;
3234   PetscCall(PetscObjectGetComm((PetscObject)dm,&comm));
3235   PetscCallMPI(MPI_Comm_rank(comm, &rank));
3236   /* Create subpointMap which marks the submesh */
3237   PetscCall(DMLabelCreate(PETSC_COMM_SELF, "subpoint_map", &subpointMap));
3238   PetscCall(DMPlexSetSubpointMap(subdm, subpointMap));
3239   if (cellHeight) {
3240     if (isCohesive) PetscCall(DMPlexMarkCohesiveSubmesh_Interpolated(dm, label, value, subpointMap, subdm));
3241     else            PetscCall(DMPlexMarkSubmesh_Interpolated(dm, label, value, markedFaces, subpointMap, subdm));
3242   } else {
3243     DMLabel         depth;
3244     IS              pointIS;
3245     const PetscInt *points;
3246     PetscInt        numPoints=0;
3247 
3248     PetscCall(DMPlexGetDepthLabel(dm, &depth));
3249     PetscCall(DMLabelGetStratumIS(label, value, &pointIS));
3250     if (pointIS) {
3251       PetscCall(ISGetIndices(pointIS, &points));
3252       PetscCall(ISGetLocalSize(pointIS, &numPoints));
3253     }
3254     for (p = 0; p < numPoints; ++p) {
3255       PetscInt *closure = NULL;
3256       PetscInt  closureSize, c, pdim;
3257 
3258       PetscCall(DMPlexGetTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closure));
3259       for (c = 0; c < closureSize*2; c += 2) {
3260         PetscCall(DMLabelGetValue(depth, closure[c], &pdim));
3261         PetscCall(DMLabelSetValue(subpointMap, closure[c], pdim));
3262       }
3263       PetscCall(DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closure));
3264     }
3265     if (pointIS) PetscCall(ISRestoreIndices(pointIS, &points));
3266     PetscCall(ISDestroy(&pointIS));
3267   }
3268   /* Setup chart */
3269   PetscCall(DMGetDimension(dm, &dim));
3270   PetscCall(PetscMalloc4(dim+1,&numSubPoints,dim+1,&firstSubPoint,dim+1,&subpointIS,dim+1,&subpoints));
3271   for (d = 0; d <= dim; ++d) {
3272     PetscCall(DMLabelGetStratumSize(subpointMap, d, &numSubPoints[d]));
3273     totSubPoints += numSubPoints[d];
3274   }
3275   PetscCall(DMPlexSetChart(subdm, 0, totSubPoints));
3276   PetscCall(DMPlexSetVTKCellHeight(subdm, cellHeight));
3277   /* Set cone sizes */
3278   firstSubPoint[dim] = 0;
3279   firstSubPoint[0]   = firstSubPoint[dim] + numSubPoints[dim];
3280   if (dim > 1) {firstSubPoint[dim-1] = firstSubPoint[0]     + numSubPoints[0];}
3281   if (dim > 2) {firstSubPoint[dim-2] = firstSubPoint[dim-1] + numSubPoints[dim-1];}
3282   for (d = 0; d <= dim; ++d) {
3283     PetscCall(DMLabelGetStratumIS(subpointMap, d, &subpointIS[d]));
3284     if (subpointIS[d]) PetscCall(ISGetIndices(subpointIS[d], &subpoints[d]));
3285   }
3286   /* We do not want this label automatically computed, instead we compute it here */
3287   PetscCall(DMCreateLabel(subdm, "celltype"));
3288   for (d = 0; d <= dim; ++d) {
3289     for (p = 0; p < numSubPoints[d]; ++p) {
3290       const PetscInt  point    = subpoints[d][p];
3291       const PetscInt  subpoint = firstSubPoint[d] + p;
3292       const PetscInt *cone;
3293       PetscInt        coneSize;
3294 
3295       PetscCall(DMPlexGetConeSize(dm, point, &coneSize));
3296       if (cellHeight && (d == dim)) {
3297         PetscInt coneSizeNew, c, val;
3298 
3299         PetscCall(DMPlexGetCone(dm, point, &cone));
3300         for (c = 0, coneSizeNew = 0; c < coneSize; ++c) {
3301           PetscCall(DMLabelGetValue(subpointMap, cone[c], &val));
3302           if (val >= 0) coneSizeNew++;
3303         }
3304         PetscCall(DMPlexSetConeSize(subdm, subpoint, coneSizeNew));
3305         PetscCall(DMPlexSetCellType(subdm, subpoint, DM_POLYTOPE_FV_GHOST));
3306       } else {
3307         DMPolytopeType  ct;
3308 
3309         PetscCall(DMPlexSetConeSize(subdm, subpoint, coneSize));
3310         PetscCall(DMPlexGetCellType(dm, point, &ct));
3311         PetscCall(DMPlexSetCellType(subdm, subpoint, ct));
3312       }
3313     }
3314   }
3315   PetscCall(DMLabelDestroy(&subpointMap));
3316   PetscCall(DMSetUp(subdm));
3317   /* Set cones */
3318   PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, NULL));
3319   PetscCall(PetscMalloc2(maxConeSize,&coneNew,maxConeSize,&orntNew));
3320   for (d = 0; d <= dim; ++d) {
3321     for (p = 0; p < numSubPoints[d]; ++p) {
3322       const PetscInt  point    = subpoints[d][p];
3323       const PetscInt  subpoint = firstSubPoint[d] + p;
3324       const PetscInt *cone, *ornt;
3325       PetscInt        coneSize, subconeSize, coneSizeNew, c, subc, fornt = 0;
3326 
3327       if (d == dim-1) {
3328         const PetscInt *support, *cone, *ornt;
3329         PetscInt        supportSize, coneSize, s, subc;
3330 
3331         PetscCall(DMPlexGetSupport(dm, point, &support));
3332         PetscCall(DMPlexGetSupportSize(dm, point, &supportSize));
3333         for (s = 0; s < supportSize; ++s) {
3334           PetscBool isHybrid;
3335 
3336           PetscCall(DMPlexCellIsHybrid_Internal(dm, support[s], &isHybrid));
3337           if (!isHybrid) continue;
3338           PetscCall(PetscFindInt(support[s], numSubPoints[d+1], subpoints[d+1], &subc));
3339           if (subc >= 0) {
3340             const PetscInt ccell = subpoints[d+1][subc];
3341 
3342             PetscCall(DMPlexGetCone(dm, ccell, &cone));
3343             PetscCall(DMPlexGetConeSize(dm, ccell, &coneSize));
3344             PetscCall(DMPlexGetConeOrientation(dm, ccell, &ornt));
3345             for (c = 0; c < coneSize; ++c) {
3346               if (cone[c] == point) {
3347                 fornt = ornt[c];
3348                 break;
3349               }
3350             }
3351             break;
3352           }
3353         }
3354       }
3355       PetscCall(DMPlexGetConeSize(dm, point, &coneSize));
3356       PetscCall(DMPlexGetConeSize(subdm, subpoint, &subconeSize));
3357       PetscCall(DMPlexGetCone(dm, point, &cone));
3358       PetscCall(DMPlexGetConeOrientation(dm, point, &ornt));
3359       for (c = 0, coneSizeNew = 0; c < coneSize; ++c) {
3360         PetscCall(PetscFindInt(cone[c], numSubPoints[d-1], subpoints[d-1], &subc));
3361         if (subc >= 0) {
3362           coneNew[coneSizeNew] = firstSubPoint[d-1] + subc;
3363           orntNew[coneSizeNew] = ornt[c];
3364           ++coneSizeNew;
3365         }
3366       }
3367       PetscCheck(coneSizeNew == subconeSize,comm, PETSC_ERR_PLIB, "Number of cone points located %" PetscInt_FMT " does not match subcone size %" PetscInt_FMT, coneSizeNew, subconeSize);
3368       PetscCall(DMPlexSetCone(subdm, subpoint, coneNew));
3369       PetscCall(DMPlexSetConeOrientation(subdm, subpoint, orntNew));
3370       if (fornt < 0) PetscCall(DMPlexOrientPoint(subdm, subpoint, fornt));
3371     }
3372   }
3373   PetscCall(PetscFree2(coneNew,orntNew));
3374   PetscCall(DMPlexSymmetrize(subdm));
3375   PetscCall(DMPlexStratify(subdm));
3376   /* Build coordinates */
3377   {
3378     PetscSection coordSection, subCoordSection;
3379     Vec          coordinates, subCoordinates;
3380     PetscScalar *coords, *subCoords;
3381     PetscInt     cdim, numComp, coordSize;
3382     const char  *name;
3383 
3384     PetscCall(DMGetCoordinateDim(dm, &cdim));
3385     PetscCall(DMGetCoordinateSection(dm, &coordSection));
3386     PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
3387     PetscCall(DMGetCoordinateSection(subdm, &subCoordSection));
3388     PetscCall(PetscSectionSetNumFields(subCoordSection, 1));
3389     PetscCall(PetscSectionGetFieldComponents(coordSection, 0, &numComp));
3390     PetscCall(PetscSectionSetFieldComponents(subCoordSection, 0, numComp));
3391     PetscCall(PetscSectionSetChart(subCoordSection, firstSubPoint[0], firstSubPoint[0]+numSubPoints[0]));
3392     for (v = 0; v < numSubPoints[0]; ++v) {
3393       const PetscInt vertex    = subpoints[0][v];
3394       const PetscInt subvertex = firstSubPoint[0]+v;
3395       PetscInt       dof;
3396 
3397       PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3398       PetscCall(PetscSectionSetDof(subCoordSection, subvertex, dof));
3399       PetscCall(PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof));
3400     }
3401     PetscCall(PetscSectionSetUp(subCoordSection));
3402     PetscCall(PetscSectionGetStorageSize(subCoordSection, &coordSize));
3403     PetscCall(VecCreate(PETSC_COMM_SELF, &subCoordinates));
3404     PetscCall(PetscObjectGetName((PetscObject)coordinates,&name));
3405     PetscCall(PetscObjectSetName((PetscObject)subCoordinates,name));
3406     PetscCall(VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE));
3407     PetscCall(VecSetBlockSize(subCoordinates, cdim));
3408     PetscCall(VecSetType(subCoordinates,VECSTANDARD));
3409     PetscCall(VecGetArray(coordinates,    &coords));
3410     PetscCall(VecGetArray(subCoordinates, &subCoords));
3411     for (v = 0; v < numSubPoints[0]; ++v) {
3412       const PetscInt vertex    = subpoints[0][v];
3413       const PetscInt subvertex = firstSubPoint[0]+v;
3414       PetscInt dof, off, sdof, soff, d;
3415 
3416       PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3417       PetscCall(PetscSectionGetOffset(coordSection, vertex, &off));
3418       PetscCall(PetscSectionGetDof(subCoordSection, subvertex, &sdof));
3419       PetscCall(PetscSectionGetOffset(subCoordSection, subvertex, &soff));
3420       PetscCheck(dof == sdof,comm, PETSC_ERR_PLIB, "Coordinate dimension %" PetscInt_FMT " on subvertex %" PetscInt_FMT ", vertex %" PetscInt_FMT " should be %" PetscInt_FMT, sdof, subvertex, vertex, dof);
3421       for (d = 0; d < dof; ++d) subCoords[soff+d] = coords[off+d];
3422     }
3423     PetscCall(VecRestoreArray(coordinates,    &coords));
3424     PetscCall(VecRestoreArray(subCoordinates, &subCoords));
3425     PetscCall(DMSetCoordinatesLocal(subdm, subCoordinates));
3426     PetscCall(VecDestroy(&subCoordinates));
3427   }
3428   /* Build SF: We need this complexity because subpoints might not be selected on the owning process */
3429   {
3430     PetscSF            sfPoint, sfPointSub;
3431     IS                 subpIS;
3432     const PetscSFNode *remotePoints;
3433     PetscSFNode       *sremotePoints, *newLocalPoints, *newOwners;
3434     const PetscInt    *localPoints, *subpoints;
3435     PetscInt          *slocalPoints;
3436     PetscInt           numRoots, numLeaves, numSubpoints = 0, numSubroots, numSubleaves = 0, l, sl, ll, pStart, pEnd, p;
3437     PetscMPIInt        rank;
3438 
3439     PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank));
3440     PetscCall(DMGetPointSF(dm, &sfPoint));
3441     PetscCall(DMGetPointSF(subdm, &sfPointSub));
3442     PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
3443     PetscCall(DMPlexGetChart(subdm, NULL, &numSubroots));
3444     PetscCall(DMPlexGetSubpointIS(subdm, &subpIS));
3445     if (subpIS) {
3446       PetscCall(ISGetIndices(subpIS, &subpoints));
3447       PetscCall(ISGetLocalSize(subpIS, &numSubpoints));
3448     }
3449     PetscCall(PetscSFGetGraph(sfPoint, &numRoots, &numLeaves, &localPoints, &remotePoints));
3450     if (numRoots >= 0) {
3451       PetscCall(PetscMalloc2(pEnd-pStart,&newLocalPoints,numRoots,&newOwners));
3452       for (p = 0; p < pEnd-pStart; ++p) {
3453         newLocalPoints[p].rank  = -2;
3454         newLocalPoints[p].index = -2;
3455       }
3456       /* Set subleaves */
3457       for (l = 0; l < numLeaves; ++l) {
3458         const PetscInt point    = localPoints[l];
3459         const PetscInt subpoint = DMPlexFilterPoint_Internal(point, 0, numSubpoints, subpoints);
3460 
3461         if (subpoint < 0) continue;
3462         newLocalPoints[point-pStart].rank  = rank;
3463         newLocalPoints[point-pStart].index = subpoint;
3464         ++numSubleaves;
3465       }
3466       /* Must put in owned subpoints */
3467       for (p = pStart; p < pEnd; ++p) {
3468         const PetscInt subpoint = DMPlexFilterPoint_Internal(p, 0, numSubpoints, subpoints);
3469 
3470         if (subpoint < 0) {
3471           newOwners[p-pStart].rank  = -3;
3472           newOwners[p-pStart].index = -3;
3473         } else {
3474           newOwners[p-pStart].rank  = rank;
3475           newOwners[p-pStart].index = subpoint;
3476         }
3477       }
3478       PetscCall(PetscSFReduceBegin(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC));
3479       PetscCall(PetscSFReduceEnd(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC));
3480       PetscCall(PetscSFBcastBegin(sfPoint, MPIU_2INT, newOwners, newLocalPoints,MPI_REPLACE));
3481       PetscCall(PetscSFBcastEnd(sfPoint, MPIU_2INT, newOwners, newLocalPoints,MPI_REPLACE));
3482       PetscCall(PetscMalloc1(numSubleaves, &slocalPoints));
3483       PetscCall(PetscMalloc1(numSubleaves, &sremotePoints));
3484       for (l = 0, sl = 0, ll = 0; l < numLeaves; ++l) {
3485         const PetscInt point    = localPoints[l];
3486         const PetscInt subpoint = DMPlexFilterPoint_Internal(point, 0, numSubpoints, subpoints);
3487 
3488         if (subpoint < 0) continue;
3489         if (newLocalPoints[point].rank == rank) {++ll; continue;}
3490         slocalPoints[sl]        = subpoint;
3491         sremotePoints[sl].rank  = newLocalPoints[point].rank;
3492         sremotePoints[sl].index = newLocalPoints[point].index;
3493         PetscCheck(sremotePoints[sl].rank  >= 0,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote rank for local point %" PetscInt_FMT, point);
3494         PetscCheck(sremotePoints[sl].index >= 0,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote subpoint for local point %" PetscInt_FMT, point);
3495         ++sl;
3496       }
3497       PetscCheck(sl + ll == numSubleaves,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatch in number of subleaves %" PetscInt_FMT " + %" PetscInt_FMT " != %" PetscInt_FMT, sl, ll, numSubleaves);
3498       PetscCall(PetscFree2(newLocalPoints,newOwners));
3499       PetscCall(PetscSFSetGraph(sfPointSub, numSubroots, sl, slocalPoints, PETSC_OWN_POINTER, sremotePoints, PETSC_OWN_POINTER));
3500     }
3501     if (subpIS) {
3502       PetscCall(ISRestoreIndices(subpIS, &subpoints));
3503     }
3504   }
3505   /* Filter labels */
3506   PetscCall(DMPlexFilterLabels_Internal(dm, numSubPoints, subpoints, firstSubPoint, subdm));
3507   /* Cleanup */
3508   for (d = 0; d <= dim; ++d) {
3509     if (subpointIS[d]) PetscCall(ISRestoreIndices(subpointIS[d], &subpoints[d]));
3510     PetscCall(ISDestroy(&subpointIS[d]));
3511   }
3512   PetscCall(PetscFree4(numSubPoints,firstSubPoint,subpointIS,subpoints));
3513   PetscFunctionReturn(0);
3514 }
3515 
3516 static PetscErrorCode DMPlexCreateSubmesh_Interpolated(DM dm, DMLabel vertexLabel, PetscInt value, PetscBool markedFaces, DM subdm)
3517 {
3518   PetscFunctionBegin;
3519   PetscCall(DMPlexCreateSubmeshGeneric_Interpolated(dm, vertexLabel, value, markedFaces, PETSC_FALSE, 1, subdm));
3520   PetscFunctionReturn(0);
3521 }
3522 
3523 /*@
3524   DMPlexCreateSubmesh - Extract a hypersurface from the mesh using vertices defined by a label
3525 
3526   Input Parameters:
3527 + dm           - The original mesh
3528 . vertexLabel  - The DMLabel marking points contained in the surface
3529 . value        - The label value to use
3530 - markedFaces  - PETSC_TRUE if surface faces are marked in addition to vertices, PETSC_FALSE if only vertices are marked
3531 
3532   Output Parameter:
3533 . subdm - The surface mesh
3534 
3535   Note: This function produces a DMLabel mapping original points in the submesh to their depth. This can be obtained using DMPlexGetSubpointMap().
3536 
3537   Level: developer
3538 
3539 .seealso: `DMPlexGetSubpointMap()`, `DMGetLabel()`, `DMLabelSetValue()`
3540 @*/
3541 PetscErrorCode DMPlexCreateSubmesh(DM dm, DMLabel vertexLabel, PetscInt value, PetscBool markedFaces, DM *subdm)
3542 {
3543   DMPlexInterpolatedFlag interpolated;
3544   PetscInt       dim, cdim;
3545 
3546   PetscFunctionBegin;
3547   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3548   PetscValidPointer(subdm, 5);
3549   PetscCall(DMGetDimension(dm, &dim));
3550   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), subdm));
3551   PetscCall(DMSetType(*subdm, DMPLEX));
3552   PetscCall(DMSetDimension(*subdm, dim-1));
3553   PetscCall(DMGetCoordinateDim(dm, &cdim));
3554   PetscCall(DMSetCoordinateDim(*subdm, cdim));
3555   PetscCall(DMPlexIsInterpolated(dm, &interpolated));
3556   PetscCheck(interpolated != DMPLEX_INTERPOLATED_PARTIAL,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Not for partially interpolated meshes");
3557   if (interpolated) {
3558     PetscCall(DMPlexCreateSubmesh_Interpolated(dm, vertexLabel, value, markedFaces, *subdm));
3559   } else {
3560     PetscCall(DMPlexCreateSubmesh_Uninterpolated(dm, vertexLabel, value, *subdm));
3561   }
3562   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, *subdm));
3563   PetscFunctionReturn(0);
3564 }
3565 
3566 static PetscErrorCode DMPlexCreateCohesiveSubmesh_Uninterpolated(DM dm, PetscBool hasLagrange, const char label[], PetscInt value, DM subdm)
3567 {
3568   MPI_Comm        comm;
3569   DMLabel         subpointMap;
3570   IS              subvertexIS;
3571   const PetscInt *subVertices;
3572   PetscInt        numSubVertices, firstSubVertex, numSubCells, *subCells = NULL;
3573   PetscInt       *subface, maxConeSize, numSubFaces, firstSubFace, newFacePoint, nFV;
3574   PetscInt        c, f;
3575 
3576   PetscFunctionBegin;
3577   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
3578   /* Create subpointMap which marks the submesh */
3579   PetscCall(DMLabelCreate(PETSC_COMM_SELF, "subpoint_map", &subpointMap));
3580   PetscCall(DMPlexSetSubpointMap(subdm, subpointMap));
3581   PetscCall(DMLabelDestroy(&subpointMap));
3582   PetscCall(DMPlexMarkCohesiveSubmesh_Uninterpolated(dm, hasLagrange, label, value, subpointMap, &numSubFaces, &nFV, &subCells, subdm));
3583   /* Setup chart */
3584   PetscCall(DMLabelGetStratumSize(subpointMap, 0, &numSubVertices));
3585   PetscCall(DMLabelGetStratumSize(subpointMap, 2, &numSubCells));
3586   PetscCall(DMPlexSetChart(subdm, 0, numSubCells+numSubFaces+numSubVertices));
3587   PetscCall(DMPlexSetVTKCellHeight(subdm, 1));
3588   /* Set cone sizes */
3589   firstSubVertex = numSubCells;
3590   firstSubFace   = numSubCells+numSubVertices;
3591   newFacePoint   = firstSubFace;
3592   PetscCall(DMLabelGetStratumIS(subpointMap, 0, &subvertexIS));
3593   if (subvertexIS) PetscCall(ISGetIndices(subvertexIS, &subVertices));
3594   for (c = 0; c < numSubCells; ++c) {
3595     PetscCall(DMPlexSetConeSize(subdm, c, 1));
3596   }
3597   for (f = firstSubFace; f < firstSubFace+numSubFaces; ++f) {
3598     PetscCall(DMPlexSetConeSize(subdm, f, nFV));
3599   }
3600   PetscCall(DMSetUp(subdm));
3601   /* Create face cones */
3602   PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, NULL));
3603   PetscCall(DMGetWorkArray(subdm, maxConeSize, MPIU_INT, (void**) &subface));
3604   for (c = 0; c < numSubCells; ++c) {
3605     const PetscInt  cell    = subCells[c];
3606     const PetscInt  subcell = c;
3607     const PetscInt *cone, *cells;
3608     PetscBool       isHybrid;
3609     PetscInt        numCells, subVertex, p, v;
3610 
3611     PetscCall(DMPlexCellIsHybrid_Internal(dm, cell, &isHybrid));
3612     if (!isHybrid) continue;
3613     PetscCall(DMPlexGetCone(dm, cell, &cone));
3614     for (v = 0; v < nFV; ++v) {
3615       PetscCall(PetscFindInt(cone[v], numSubVertices, subVertices, &subVertex));
3616       subface[v] = firstSubVertex+subVertex;
3617     }
3618     PetscCall(DMPlexSetCone(subdm, newFacePoint, subface));
3619     PetscCall(DMPlexSetCone(subdm, subcell, &newFacePoint));
3620     PetscCall(DMPlexGetJoin(dm, nFV, cone, &numCells, &cells));
3621     /* Not true in parallel
3622     PetscCheck(numCells == 2,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive cells should separate two cells"); */
3623     for (p = 0; p < numCells; ++p) {
3624       PetscInt  negsubcell;
3625       PetscBool isHybrid;
3626 
3627       PetscCall(DMPlexCellIsHybrid_Internal(dm, cells[p], &isHybrid));
3628       if (isHybrid) continue;
3629       /* I know this is a crap search */
3630       for (negsubcell = 0; negsubcell < numSubCells; ++negsubcell) {
3631         if (subCells[negsubcell] == cells[p]) break;
3632       }
3633       PetscCheck(negsubcell != numSubCells,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not find negative face neighbor for cohesive cell %" PetscInt_FMT, cell);
3634       PetscCall(DMPlexSetCone(subdm, negsubcell, &newFacePoint));
3635     }
3636     PetscCall(DMPlexRestoreJoin(dm, nFV, cone, &numCells, &cells));
3637     ++newFacePoint;
3638   }
3639   PetscCall(DMRestoreWorkArray(subdm, maxConeSize, MPIU_INT, (void**) &subface));
3640   PetscCall(DMPlexSymmetrize(subdm));
3641   PetscCall(DMPlexStratify(subdm));
3642   /* Build coordinates */
3643   {
3644     PetscSection coordSection, subCoordSection;
3645     Vec          coordinates, subCoordinates;
3646     PetscScalar *coords, *subCoords;
3647     PetscInt     cdim, numComp, coordSize, v;
3648     const char  *name;
3649 
3650     PetscCall(DMGetCoordinateDim(dm, &cdim));
3651     PetscCall(DMGetCoordinateSection(dm, &coordSection));
3652     PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
3653     PetscCall(DMGetCoordinateSection(subdm, &subCoordSection));
3654     PetscCall(PetscSectionSetNumFields(subCoordSection, 1));
3655     PetscCall(PetscSectionGetFieldComponents(coordSection, 0, &numComp));
3656     PetscCall(PetscSectionSetFieldComponents(subCoordSection, 0, numComp));
3657     PetscCall(PetscSectionSetChart(subCoordSection, firstSubVertex, firstSubVertex+numSubVertices));
3658     for (v = 0; v < numSubVertices; ++v) {
3659       const PetscInt vertex    = subVertices[v];
3660       const PetscInt subvertex = firstSubVertex+v;
3661       PetscInt       dof;
3662 
3663       PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3664       PetscCall(PetscSectionSetDof(subCoordSection, subvertex, dof));
3665       PetscCall(PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof));
3666     }
3667     PetscCall(PetscSectionSetUp(subCoordSection));
3668     PetscCall(PetscSectionGetStorageSize(subCoordSection, &coordSize));
3669     PetscCall(VecCreate(PETSC_COMM_SELF, &subCoordinates));
3670     PetscCall(PetscObjectGetName((PetscObject)coordinates,&name));
3671     PetscCall(PetscObjectSetName((PetscObject)subCoordinates,name));
3672     PetscCall(VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE));
3673     PetscCall(VecSetBlockSize(subCoordinates, cdim));
3674     PetscCall(VecSetType(subCoordinates,VECSTANDARD));
3675     PetscCall(VecGetArray(coordinates,    &coords));
3676     PetscCall(VecGetArray(subCoordinates, &subCoords));
3677     for (v = 0; v < numSubVertices; ++v) {
3678       const PetscInt vertex    = subVertices[v];
3679       const PetscInt subvertex = firstSubVertex+v;
3680       PetscInt       dof, off, sdof, soff, d;
3681 
3682       PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3683       PetscCall(PetscSectionGetOffset(coordSection, vertex, &off));
3684       PetscCall(PetscSectionGetDof(subCoordSection, subvertex, &sdof));
3685       PetscCall(PetscSectionGetOffset(subCoordSection, subvertex, &soff));
3686       PetscCheck(dof == sdof,comm, PETSC_ERR_PLIB, "Coordinate dimension %" PetscInt_FMT " on subvertex %" PetscInt_FMT ", vertex %" PetscInt_FMT " should be %" PetscInt_FMT, sdof, subvertex, vertex, dof);
3687       for (d = 0; d < dof; ++d) subCoords[soff+d] = coords[off+d];
3688     }
3689     PetscCall(VecRestoreArray(coordinates,    &coords));
3690     PetscCall(VecRestoreArray(subCoordinates, &subCoords));
3691     PetscCall(DMSetCoordinatesLocal(subdm, subCoordinates));
3692     PetscCall(VecDestroy(&subCoordinates));
3693   }
3694   /* Build SF */
3695   CHKMEMQ;
3696   {
3697     PetscSF            sfPoint, sfPointSub;
3698     const PetscSFNode *remotePoints;
3699     PetscSFNode       *sremotePoints, *newLocalPoints, *newOwners;
3700     const PetscInt    *localPoints;
3701     PetscInt          *slocalPoints;
3702     PetscInt           numRoots, numLeaves, numSubRoots = numSubCells+numSubFaces+numSubVertices, numSubLeaves = 0, l, sl, ll, pStart, pEnd, p, vStart, vEnd;
3703     PetscMPIInt        rank;
3704 
3705     PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank));
3706     PetscCall(DMGetPointSF(dm, &sfPoint));
3707     PetscCall(DMGetPointSF(subdm, &sfPointSub));
3708     PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
3709     PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
3710     PetscCall(PetscSFGetGraph(sfPoint, &numRoots, &numLeaves, &localPoints, &remotePoints));
3711     if (numRoots >= 0) {
3712       /* Only vertices should be shared */
3713       PetscCall(PetscMalloc2(pEnd-pStart,&newLocalPoints,numRoots,&newOwners));
3714       for (p = 0; p < pEnd-pStart; ++p) {
3715         newLocalPoints[p].rank  = -2;
3716         newLocalPoints[p].index = -2;
3717       }
3718       /* Set subleaves */
3719       for (l = 0; l < numLeaves; ++l) {
3720         const PetscInt point    = localPoints[l];
3721         const PetscInt subPoint = DMPlexFilterPoint_Internal(point, firstSubVertex, numSubVertices, subVertices);
3722 
3723         PetscCheck(!(point < vStart) || !(point >= vEnd),PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Should not be mapping anything but vertices, %" PetscInt_FMT, point);
3724         if (subPoint < 0) continue;
3725         newLocalPoints[point-pStart].rank  = rank;
3726         newLocalPoints[point-pStart].index = subPoint;
3727         ++numSubLeaves;
3728       }
3729       /* Must put in owned subpoints */
3730       for (p = pStart; p < pEnd; ++p) {
3731         const PetscInt subPoint = DMPlexFilterPoint_Internal(p, firstSubVertex, numSubVertices, subVertices);
3732 
3733         if (subPoint < 0) {
3734           newOwners[p-pStart].rank  = -3;
3735           newOwners[p-pStart].index = -3;
3736         } else {
3737           newOwners[p-pStart].rank  = rank;
3738           newOwners[p-pStart].index = subPoint;
3739         }
3740       }
3741       PetscCall(PetscSFReduceBegin(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC));
3742       PetscCall(PetscSFReduceEnd(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC));
3743       PetscCall(PetscSFBcastBegin(sfPoint, MPIU_2INT, newOwners, newLocalPoints,MPI_REPLACE));
3744       PetscCall(PetscSFBcastEnd(sfPoint, MPIU_2INT, newOwners, newLocalPoints,MPI_REPLACE));
3745       PetscCall(PetscMalloc1(numSubLeaves,    &slocalPoints));
3746       PetscCall(PetscMalloc1(numSubLeaves, &sremotePoints));
3747       for (l = 0, sl = 0, ll = 0; l < numLeaves; ++l) {
3748         const PetscInt point    = localPoints[l];
3749         const PetscInt subPoint = DMPlexFilterPoint_Internal(point, firstSubVertex, numSubVertices, subVertices);
3750 
3751         if (subPoint < 0) continue;
3752         if (newLocalPoints[point].rank == rank) {++ll; continue;}
3753         slocalPoints[sl]        = subPoint;
3754         sremotePoints[sl].rank  = newLocalPoints[point].rank;
3755         sremotePoints[sl].index = newLocalPoints[point].index;
3756         PetscCheck(sremotePoints[sl].rank  >= 0,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote rank for local point %" PetscInt_FMT, point);
3757         PetscCheck(sremotePoints[sl].index >= 0,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote subpoint for local point %" PetscInt_FMT, point);
3758         ++sl;
3759       }
3760       PetscCall(PetscFree2(newLocalPoints,newOwners));
3761       PetscCheck(sl + ll == numSubLeaves,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatch in number of subleaves %" PetscInt_FMT " + %" PetscInt_FMT " != %" PetscInt_FMT, sl, ll, numSubLeaves);
3762       PetscCall(PetscSFSetGraph(sfPointSub, numSubRoots, sl, slocalPoints, PETSC_OWN_POINTER, sremotePoints, PETSC_OWN_POINTER));
3763     }
3764   }
3765   CHKMEMQ;
3766   /* Cleanup */
3767   if (subvertexIS) PetscCall(ISRestoreIndices(subvertexIS, &subVertices));
3768   PetscCall(ISDestroy(&subvertexIS));
3769   PetscCall(PetscFree(subCells));
3770   PetscFunctionReturn(0);
3771 }
3772 
3773 static PetscErrorCode DMPlexCreateCohesiveSubmesh_Interpolated(DM dm, const char labelname[], PetscInt value, DM subdm)
3774 {
3775   DMLabel        label = NULL;
3776 
3777   PetscFunctionBegin;
3778   if (labelname) PetscCall(DMGetLabel(dm, labelname, &label));
3779   PetscCall(DMPlexCreateSubmeshGeneric_Interpolated(dm, label, value, PETSC_FALSE, PETSC_TRUE, 1, subdm));
3780   PetscFunctionReturn(0);
3781 }
3782 
3783 /*@C
3784   DMPlexCreateCohesiveSubmesh - Extract from a mesh with cohesive cells the hypersurface defined by one face of the cells. Optionally, a Label can be given to restrict the cells.
3785 
3786   Input Parameters:
3787 + dm          - The original mesh
3788 . hasLagrange - The mesh has Lagrange unknowns in the cohesive cells
3789 . label       - A label name, or NULL
3790 - value  - A label value
3791 
3792   Output Parameter:
3793 . subdm - The surface mesh
3794 
3795   Note: This function produces a DMLabel mapping original points in the submesh to their depth. This can be obtained using DMPlexGetSubpointMap().
3796 
3797   Level: developer
3798 
3799 .seealso: `DMPlexGetSubpointMap()`, `DMPlexCreateSubmesh()`
3800 @*/
3801 PetscErrorCode DMPlexCreateCohesiveSubmesh(DM dm, PetscBool hasLagrange, const char label[], PetscInt value, DM *subdm)
3802 {
3803   PetscInt       dim, cdim, depth;
3804 
3805   PetscFunctionBegin;
3806   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3807   PetscValidPointer(subdm, 5);
3808   PetscCall(DMGetDimension(dm, &dim));
3809   PetscCall(DMPlexGetDepth(dm, &depth));
3810   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), subdm));
3811   PetscCall(DMSetType(*subdm, DMPLEX));
3812   PetscCall(DMSetDimension(*subdm, dim-1));
3813   PetscCall(DMGetCoordinateDim(dm, &cdim));
3814   PetscCall(DMSetCoordinateDim(*subdm, cdim));
3815   if (depth == dim) {
3816     PetscCall(DMPlexCreateCohesiveSubmesh_Interpolated(dm, label, value, *subdm));
3817   } else {
3818     PetscCall(DMPlexCreateCohesiveSubmesh_Uninterpolated(dm, hasLagrange, label, value, *subdm));
3819   }
3820   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, *subdm));
3821   PetscFunctionReturn(0);
3822 }
3823 
3824 /*@
3825   DMPlexFilter - Extract a subset of mesh cells defined by a label as a separate mesh
3826 
3827   Input Parameters:
3828 + dm        - The original mesh
3829 . cellLabel - The DMLabel marking cells contained in the new mesh
3830 - value     - The label value to use
3831 
3832   Output Parameter:
3833 . subdm - The new mesh
3834 
3835   Note: This function produces a DMLabel mapping original points in the submesh to their depth. This can be obtained using DMPlexGetSubpointMap().
3836 
3837   Level: developer
3838 
3839 .seealso: `DMPlexGetSubpointMap()`, `DMGetLabel()`, `DMLabelSetValue()`
3840 @*/
3841 PetscErrorCode DMPlexFilter(DM dm, DMLabel cellLabel, PetscInt value, DM *subdm)
3842 {
3843   PetscInt       dim;
3844 
3845   PetscFunctionBegin;
3846   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3847   PetscValidPointer(subdm, 4);
3848   PetscCall(DMGetDimension(dm, &dim));
3849   PetscCall(DMCreate(PetscObjectComm((PetscObject) dm), subdm));
3850   PetscCall(DMSetType(*subdm, DMPLEX));
3851   PetscCall(DMSetDimension(*subdm, dim));
3852   /* Extract submesh in place, could be empty on some procs, could have inconsistency if procs do not both extract a shared cell */
3853   PetscCall(DMPlexCreateSubmeshGeneric_Interpolated(dm, cellLabel, value, PETSC_FALSE, PETSC_FALSE, 0, *subdm));
3854   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, *subdm));
3855   PetscFunctionReturn(0);
3856 }
3857 
3858 /*@
3859   DMPlexGetSubpointMap - Returns a DMLabel with point dimension as values
3860 
3861   Input Parameter:
3862 . dm - The submesh DM
3863 
3864   Output Parameter:
3865 . subpointMap - The DMLabel of all the points from the original mesh in this submesh, or NULL if this is not a submesh
3866 
3867   Level: developer
3868 
3869 .seealso: `DMPlexCreateSubmesh()`, `DMPlexGetSubpointIS()`
3870 @*/
3871 PetscErrorCode DMPlexGetSubpointMap(DM dm, DMLabel *subpointMap)
3872 {
3873   PetscFunctionBegin;
3874   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3875   PetscValidPointer(subpointMap, 2);
3876   *subpointMap = ((DM_Plex*) dm->data)->subpointMap;
3877   PetscFunctionReturn(0);
3878 }
3879 
3880 /*@
3881   DMPlexSetSubpointMap - Sets the DMLabel with point dimension as values
3882 
3883   Input Parameters:
3884 + dm - The submesh DM
3885 - subpointMap - The DMLabel of all the points from the original mesh in this submesh
3886 
3887   Note: Should normally not be called by the user, since it is set in DMPlexCreateSubmesh()
3888 
3889   Level: developer
3890 
3891 .seealso: `DMPlexCreateSubmesh()`, `DMPlexGetSubpointIS()`
3892 @*/
3893 PetscErrorCode DMPlexSetSubpointMap(DM dm, DMLabel subpointMap)
3894 {
3895   DM_Plex       *mesh = (DM_Plex *) dm->data;
3896   DMLabel        tmp;
3897 
3898   PetscFunctionBegin;
3899   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3900   tmp  = mesh->subpointMap;
3901   mesh->subpointMap = subpointMap;
3902   PetscCall(PetscObjectReference((PetscObject) mesh->subpointMap));
3903   PetscCall(DMLabelDestroy(&tmp));
3904   PetscFunctionReturn(0);
3905 }
3906 
3907 static PetscErrorCode DMPlexCreateSubpointIS_Internal(DM dm, IS *subpointIS)
3908 {
3909   DMLabel        spmap;
3910   PetscInt       depth, d;
3911 
3912   PetscFunctionBegin;
3913   PetscCall(DMPlexGetSubpointMap(dm, &spmap));
3914   PetscCall(DMPlexGetDepth(dm, &depth));
3915   if (spmap && depth >= 0) {
3916     DM_Plex  *mesh = (DM_Plex *) dm->data;
3917     PetscInt *points, *depths;
3918     PetscInt  pStart, pEnd, p, off;
3919 
3920     PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
3921     PetscCheck(!pStart,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Submeshes must start the point numbering at 0, not %" PetscInt_FMT, pStart);
3922     PetscCall(PetscMalloc1(pEnd, &points));
3923     PetscCall(DMGetWorkArray(dm, depth+1, MPIU_INT, &depths));
3924     depths[0] = depth;
3925     depths[1] = 0;
3926     for (d = 2; d <= depth; ++d) {depths[d] = depth+1 - d;}
3927     for (d = 0, off = 0; d <= depth; ++d) {
3928       const PetscInt dep = depths[d];
3929       PetscInt       depStart, depEnd, n;
3930 
3931       PetscCall(DMPlexGetDepthStratum(dm, dep, &depStart, &depEnd));
3932       PetscCall(DMLabelGetStratumSize(spmap, dep, &n));
3933       if (((d < 2) && (depth > 1)) || (d == 1)) { /* Only check vertices and cells for now since the map is broken for others */
3934         PetscCheck(n == depEnd-depStart,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "The number of mapped submesh points %" PetscInt_FMT " at depth %" PetscInt_FMT " should be %" PetscInt_FMT, n, dep, depEnd-depStart);
3935       } else {
3936         if (!n) {
3937           if (d == 0) {
3938             /* Missing cells */
3939             for (p = 0; p < depEnd-depStart; ++p, ++off) points[off] = -1;
3940           } else {
3941             /* Missing faces */
3942             for (p = 0; p < depEnd-depStart; ++p, ++off) points[off] = PETSC_MAX_INT;
3943           }
3944         }
3945       }
3946       if (n) {
3947         IS              is;
3948         const PetscInt *opoints;
3949 
3950         PetscCall(DMLabelGetStratumIS(spmap, dep, &is));
3951         PetscCall(ISGetIndices(is, &opoints));
3952         for (p = 0; p < n; ++p, ++off) points[off] = opoints[p];
3953         PetscCall(ISRestoreIndices(is, &opoints));
3954         PetscCall(ISDestroy(&is));
3955       }
3956     }
3957     PetscCall(DMRestoreWorkArray(dm, depth+1, MPIU_INT, &depths));
3958     PetscCheck(off == pEnd,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "The number of mapped submesh points %" PetscInt_FMT " should be %" PetscInt_FMT, off, pEnd);
3959     PetscCall(ISCreateGeneral(PETSC_COMM_SELF, pEnd, points, PETSC_OWN_POINTER, subpointIS));
3960     PetscCall(PetscObjectStateGet((PetscObject) spmap, &mesh->subpointState));
3961   }
3962   PetscFunctionReturn(0);
3963 }
3964 
3965 /*@
3966   DMPlexGetSubpointIS - Returns an IS covering the entire subdm chart with the original points as data
3967 
3968   Input Parameter:
3969 . dm - The submesh DM
3970 
3971   Output Parameter:
3972 . subpointIS - The IS of all the points from the original mesh in this submesh, or NULL if this is not a submesh
3973 
3974   Note: This IS is guaranteed to be sorted by the construction of the submesh
3975 
3976   Level: developer
3977 
3978 .seealso: `DMPlexCreateSubmesh()`, `DMPlexGetSubpointMap()`
3979 @*/
3980 PetscErrorCode DMPlexGetSubpointIS(DM dm, IS *subpointIS)
3981 {
3982   DM_Plex         *mesh = (DM_Plex *) dm->data;
3983   DMLabel          spmap;
3984   PetscObjectState state;
3985 
3986   PetscFunctionBegin;
3987   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3988   PetscValidPointer(subpointIS, 2);
3989   PetscCall(DMPlexGetSubpointMap(dm, &spmap));
3990   PetscCall(PetscObjectStateGet((PetscObject) spmap, &state));
3991   if (state != mesh->subpointState || !mesh->subpointIS) PetscCall(DMPlexCreateSubpointIS_Internal(dm, &mesh->subpointIS));
3992   *subpointIS = mesh->subpointIS;
3993   PetscFunctionReturn(0);
3994 }
3995 
3996 /*@
3997   DMGetEnclosureRelation - Get the relationship between dmA and dmB
3998 
3999   Input Parameters:
4000 + dmA - The first DM
4001 - dmB - The second DM
4002 
4003   Output Parameter:
4004 . rel - The relation of dmA to dmB
4005 
4006   Level: intermediate
4007 
4008 .seealso: `DMGetEnclosurePoint()`
4009 @*/
4010 PetscErrorCode DMGetEnclosureRelation(DM dmA, DM dmB, DMEnclosureType *rel)
4011 {
4012   DM             plexA, plexB, sdm;
4013   DMLabel        spmap;
4014   PetscInt       pStartA, pEndA, pStartB, pEndB, NpA, NpB;
4015 
4016   PetscFunctionBegin;
4017   PetscValidPointer(rel, 3);
4018   *rel = DM_ENC_NONE;
4019   if (!dmA || !dmB) PetscFunctionReturn(0);
4020   PetscValidHeaderSpecific(dmA, DM_CLASSID, 1);
4021   PetscValidHeaderSpecific(dmB, DM_CLASSID, 2);
4022   if (dmA == dmB) {*rel = DM_ENC_EQUALITY; PetscFunctionReturn(0);}
4023   PetscCall(DMConvert(dmA, DMPLEX, &plexA));
4024   PetscCall(DMConvert(dmB, DMPLEX, &plexB));
4025   PetscCall(DMPlexGetChart(plexA, &pStartA, &pEndA));
4026   PetscCall(DMPlexGetChart(plexB, &pStartB, &pEndB));
4027   /* Assumption 1: subDMs have smaller charts than the DMs that they originate from
4028     - The degenerate case of a subdomain which includes all of the domain on some process can be treated as equality */
4029   if ((pStartA == pStartB) && (pEndA == pEndB)) {
4030     *rel = DM_ENC_EQUALITY;
4031     goto end;
4032   }
4033   NpA = pEndA - pStartA;
4034   NpB = pEndB - pStartB;
4035   if (NpA == NpB) goto end;
4036   sdm = NpA > NpB ? plexB : plexA; /* The other is the original, enclosing dm */
4037   PetscCall(DMPlexGetSubpointMap(sdm, &spmap));
4038   if (!spmap) goto end;
4039   /* TODO Check the space mapped to by subpointMap is same size as dm */
4040   if (NpA > NpB) {
4041     *rel = DM_ENC_SUPERMESH;
4042   } else {
4043     *rel = DM_ENC_SUBMESH;
4044   }
4045   end:
4046   PetscCall(DMDestroy(&plexA));
4047   PetscCall(DMDestroy(&plexB));
4048   PetscFunctionReturn(0);
4049 }
4050 
4051 /*@
4052   DMGetEnclosurePoint - Get the point pA in dmA which corresponds to the point pB in dmB
4053 
4054   Input Parameters:
4055 + dmA   - The first DM
4056 . dmB   - The second DM
4057 . etype - The type of enclosure relation that dmA has to dmB
4058 - pB    - A point of dmB
4059 
4060   Output Parameter:
4061 . pA    - The corresponding point of dmA
4062 
4063   Level: intermediate
4064 
4065 .seealso: `DMGetEnclosureRelation()`
4066 @*/
4067 PetscErrorCode DMGetEnclosurePoint(DM dmA, DM dmB, DMEnclosureType etype, PetscInt pB, PetscInt *pA)
4068 {
4069   DM              sdm;
4070   IS              subpointIS;
4071   const PetscInt *subpoints;
4072   PetscInt        numSubpoints;
4073 
4074   PetscFunctionBegin;
4075   /* TODO Cache the IS, making it look like an index */
4076   switch (etype) {
4077     case DM_ENC_SUPERMESH:
4078     sdm  = dmB;
4079     PetscCall(DMPlexGetSubpointIS(sdm, &subpointIS));
4080     PetscCall(ISGetIndices(subpointIS, &subpoints));
4081     *pA  = subpoints[pB];
4082     PetscCall(ISRestoreIndices(subpointIS, &subpoints));
4083     break;
4084     case DM_ENC_SUBMESH:
4085     sdm  = dmA;
4086     PetscCall(DMPlexGetSubpointIS(sdm, &subpointIS));
4087     PetscCall(ISGetLocalSize(subpointIS, &numSubpoints));
4088     PetscCall(ISGetIndices(subpointIS, &subpoints));
4089     PetscCall(PetscFindInt(pB, numSubpoints, subpoints, pA));
4090     if (*pA < 0) {
4091       PetscCall(DMViewFromOptions(dmA, NULL, "-dm_enc_A_view"));
4092       PetscCall(DMViewFromOptions(dmB, NULL, "-dm_enc_B_view"));
4093       SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Point %" PetscInt_FMT " not found in submesh", pB);
4094     }
4095     PetscCall(ISRestoreIndices(subpointIS, &subpoints));
4096     break;
4097     case DM_ENC_EQUALITY:
4098     case DM_ENC_NONE:
4099     *pA = pB;break;
4100     case DM_ENC_UNKNOWN:
4101     {
4102       DMEnclosureType enc;
4103 
4104       PetscCall(DMGetEnclosureRelation(dmA, dmB, &enc));
4105       PetscCall(DMGetEnclosurePoint(dmA, dmB, enc, pB, pA));
4106     }
4107     break;
4108     default: SETERRQ(PetscObjectComm((PetscObject) dmA), PETSC_ERR_ARG_OUTOFRANGE, "Invalid enclosure type %d", (int) etype);
4109   }
4110   PetscFunctionReturn(0);
4111 }
4112