xref: /petsc/src/dm/impls/plex/plexsubmesh.c (revision 3bdf08b3bec9e18857878e70df7ab0f813726179)
1 #include <petsc/private/dmpleximpl.h>    /*I      "petscdmplex.h"    I*/
2 #include <petsc/private/dmlabelimpl.h>   /*I      "petscdmlabel.h"   I*/
3 #include <petscsf.h>
4 
5 static PetscErrorCode DMPlexCellIsHybrid_Internal(DM dm, PetscInt p, PetscBool *isHybrid)
6 {
7   DMPolytopeType ct;
8 
9   PetscFunctionBegin;
10   PetscCall(DMPlexGetCellType(dm, p, &ct));
11   switch (ct) {
12     case DM_POLYTOPE_POINT_PRISM_TENSOR:
13     case DM_POLYTOPE_SEG_PRISM_TENSOR:
14     case DM_POLYTOPE_TRI_PRISM_TENSOR:
15     case DM_POLYTOPE_QUAD_PRISM_TENSOR:
16       *isHybrid = PETSC_TRUE;
17     default: *isHybrid = PETSC_FALSE;
18   }
19   PetscFunctionReturn(0);
20 }
21 
22 static PetscErrorCode DMPlexGetTensorPrismBounds_Internal(DM dm, PetscInt dim, PetscInt *cStart, PetscInt *cEnd)
23 {
24   DMLabel        ctLabel;
25 
26   PetscFunctionBegin;
27   if (cStart) *cStart = -1;
28   if (cEnd)   *cEnd   = -1;
29   PetscCall(DMPlexGetCellTypeLabel(dm, &ctLabel));
30   switch (dim) {
31     case 1: PetscCall(DMLabelGetStratumBounds(ctLabel, DM_POLYTOPE_POINT_PRISM_TENSOR, cStart, cEnd));break;
32     case 2: PetscCall(DMLabelGetStratumBounds(ctLabel, DM_POLYTOPE_SEG_PRISM_TENSOR, cStart, cEnd));break;
33     case 3:
34       PetscCall(DMLabelGetStratumBounds(ctLabel, DM_POLYTOPE_TRI_PRISM_TENSOR, cStart, cEnd));
35       if (*cStart < 0) PetscCall(DMLabelGetStratumBounds(ctLabel, DM_POLYTOPE_QUAD_PRISM_TENSOR, cStart, cEnd));
36       break;
37     default: PetscFunctionReturn(0);
38   }
39   PetscFunctionReturn(0);
40 }
41 
42 static PetscErrorCode DMPlexMarkBoundaryFaces_Internal(DM dm, PetscInt val, PetscInt cellHeight, DMLabel label)
43 {
44   PetscInt       fStart, fEnd, f;
45 
46   PetscFunctionBegin;
47   PetscCall(DMPlexGetHeightStratum(dm, cellHeight+1, &fStart, &fEnd));
48   for (f = fStart; f < fEnd; ++f) {
49     PetscInt supportSize;
50 
51     PetscCall(DMPlexGetSupportSize(dm, f, &supportSize));
52     if (supportSize == 1) {
53       if (val < 0) {
54         PetscInt *closure = NULL;
55         PetscInt  clSize, cl, cval;
56 
57         PetscCall(DMPlexGetTransitiveClosure(dm, f, PETSC_TRUE, &clSize, &closure));
58         for (cl = 0; cl < clSize*2; cl += 2) {
59           PetscCall(DMLabelGetValue(label, closure[cl], &cval));
60           if (cval < 0) continue;
61           PetscCall(DMLabelSetValue(label, f, cval));
62           break;
63         }
64         if (cl == clSize*2) PetscCall(DMLabelSetValue(label, f, 1));
65         PetscCall(DMPlexRestoreTransitiveClosure(dm, f, PETSC_TRUE, &clSize, &closure));
66       } else {
67         PetscCall(DMLabelSetValue(label, f, val));
68       }
69     }
70   }
71   PetscFunctionReturn(0);
72 }
73 
74 /*@
75   DMPlexMarkBoundaryFaces - Mark all faces on the boundary
76 
77   Not Collective
78 
79   Input Parameters:
80 + dm - The original DM
81 - val - The marker value, or PETSC_DETERMINE to use some value in the closure (or 1 if none are found)
82 
83   Output Parameter:
84 . label - The DMLabel marking boundary faces with the given value
85 
86   Level: developer
87 
88 .seealso: `DMLabelCreate()`, `DMCreateLabel()`
89 @*/
90 PetscErrorCode DMPlexMarkBoundaryFaces(DM dm, PetscInt val, DMLabel label)
91 {
92   DMPlexInterpolatedFlag  flg;
93 
94   PetscFunctionBegin;
95   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
96   PetscCall(DMPlexIsInterpolated(dm, &flg));
97   PetscCheck(flg == DMPLEX_INTERPOLATED_FULL,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM is not fully interpolated on this rank");
98   PetscCall(DMPlexMarkBoundaryFaces_Internal(dm, val, 0, label));
99   PetscFunctionReturn(0);
100 }
101 
102 static PetscErrorCode DMPlexLabelComplete_Internal(DM dm, DMLabel label, PetscBool completeCells)
103 {
104   IS              valueIS;
105   PetscSF         sfPoint;
106   const PetscInt *values;
107   PetscInt        numValues, v, cStart, cEnd, nroots;
108 
109   PetscFunctionBegin;
110   PetscCall(DMLabelGetNumValues(label, &numValues));
111   PetscCall(DMLabelGetValueIS(label, &valueIS));
112   PetscCall(DMPlexGetHeightStratum(dm,0,&cStart,&cEnd));
113   PetscCall(ISGetIndices(valueIS, &values));
114   for (v = 0; v < numValues; ++v) {
115     IS              pointIS;
116     const PetscInt *points;
117     PetscInt        numPoints, p;
118 
119     PetscCall(DMLabelGetStratumSize(label, values[v], &numPoints));
120     PetscCall(DMLabelGetStratumIS(label, values[v], &pointIS));
121     PetscCall(ISGetIndices(pointIS, &points));
122     for (p = 0; p < numPoints; ++p) {
123       PetscInt  q = points[p];
124       PetscInt *closure = NULL;
125       PetscInt  closureSize, c;
126 
127       if (cStart <= q && q < cEnd && !completeCells) { /* skip cells */
128         continue;
129       }
130       PetscCall(DMPlexGetTransitiveClosure(dm, q, PETSC_TRUE, &closureSize, &closure));
131       for (c = 0; c < closureSize*2; c += 2) {
132         PetscCall(DMLabelSetValue(label, closure[c], values[v]));
133       }
134       PetscCall(DMPlexRestoreTransitiveClosure(dm, q, PETSC_TRUE, &closureSize, &closure));
135     }
136     PetscCall(ISRestoreIndices(pointIS, &points));
137     PetscCall(ISDestroy(&pointIS));
138   }
139   PetscCall(ISRestoreIndices(valueIS, &values));
140   PetscCall(ISDestroy(&valueIS));
141   PetscCall(DMGetPointSF(dm, &sfPoint));
142   PetscCall(PetscSFGetGraph(sfPoint, &nroots, NULL, NULL, NULL));
143   if (nroots >= 0) {
144     DMLabel         lblRoots, lblLeaves;
145     IS              valueIS, pointIS;
146     const PetscInt *values;
147     PetscInt        numValues, v;
148 
149     /* Pull point contributions from remote leaves into local roots */
150     PetscCall(DMLabelGather(label, sfPoint, &lblLeaves));
151     PetscCall(DMLabelGetValueIS(lblLeaves, &valueIS));
152     PetscCall(ISGetLocalSize(valueIS, &numValues));
153     PetscCall(ISGetIndices(valueIS, &values));
154     for (v = 0; v < numValues; ++v) {
155       const PetscInt value = values[v];
156 
157       PetscCall(DMLabelGetStratumIS(lblLeaves, value, &pointIS));
158       PetscCall(DMLabelInsertIS(label, pointIS, value));
159       PetscCall(ISDestroy(&pointIS));
160     }
161     PetscCall(ISRestoreIndices(valueIS, &values));
162     PetscCall(ISDestroy(&valueIS));
163     PetscCall(DMLabelDestroy(&lblLeaves));
164     /* Push point contributions from roots into remote leaves */
165     PetscCall(DMLabelDistribute(label, sfPoint, &lblRoots));
166     PetscCall(DMLabelGetValueIS(lblRoots, &valueIS));
167     PetscCall(ISGetLocalSize(valueIS, &numValues));
168     PetscCall(ISGetIndices(valueIS, &values));
169     for (v = 0; v < numValues; ++v) {
170       const PetscInt value = values[v];
171 
172       PetscCall(DMLabelGetStratumIS(lblRoots, value, &pointIS));
173       PetscCall(DMLabelInsertIS(label, pointIS, value));
174       PetscCall(ISDestroy(&pointIS));
175     }
176     PetscCall(ISRestoreIndices(valueIS, &values));
177     PetscCall(ISDestroy(&valueIS));
178     PetscCall(DMLabelDestroy(&lblRoots));
179   }
180   PetscFunctionReturn(0);
181 }
182 
183 /*@
184   DMPlexLabelComplete - Starting with a label marking points on a surface, we add the transitive closure to the surface
185 
186   Input Parameters:
187 + dm - The DM
188 - label - A DMLabel marking the surface points
189 
190   Output Parameter:
191 . label - A DMLabel marking all surface points in the transitive closure
192 
193   Level: developer
194 
195 .seealso: `DMPlexLabelCohesiveComplete()`
196 @*/
197 PetscErrorCode DMPlexLabelComplete(DM dm, DMLabel label)
198 {
199   PetscFunctionBegin;
200   PetscCall(DMPlexLabelComplete_Internal(dm, label, PETSC_TRUE));
201   PetscFunctionReturn(0);
202 }
203 
204 /*@
205   DMPlexLabelAddCells - Starting with a label marking points on a surface, we add a cell for each point
206 
207   Input Parameters:
208 + dm - The DM
209 - label - A DMLabel marking the surface points
210 
211   Output Parameter:
212 . label - A DMLabel incorporating cells
213 
214   Level: developer
215 
216   Note: The cells allow FEM boundary conditions to be applied using the cell geometry
217 
218 .seealso: `DMPlexLabelAddFaceCells()`, `DMPlexLabelComplete()`, `DMPlexLabelCohesiveComplete()`
219 @*/
220 PetscErrorCode DMPlexLabelAddCells(DM dm, DMLabel label)
221 {
222   IS              valueIS;
223   const PetscInt *values;
224   PetscInt        numValues, v, cStart, cEnd;
225 
226   PetscFunctionBegin;
227   PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd));
228   PetscCall(DMLabelGetNumValues(label, &numValues));
229   PetscCall(DMLabelGetValueIS(label, &valueIS));
230   PetscCall(ISGetIndices(valueIS, &values));
231   for (v = 0; v < numValues; ++v) {
232     IS              pointIS;
233     const PetscInt *points;
234     PetscInt        numPoints, p;
235 
236     PetscCall(DMLabelGetStratumSize(label, values[v], &numPoints));
237     PetscCall(DMLabelGetStratumIS(label, values[v], &pointIS));
238     PetscCall(ISGetIndices(pointIS, &points));
239     for (p = 0; p < numPoints; ++p) {
240       PetscInt *closure = NULL;
241       PetscInt  closureSize, cl;
242 
243       PetscCall(DMPlexGetTransitiveClosure(dm, points[p], PETSC_FALSE, &closureSize, &closure));
244       for (cl = closureSize-1; cl > 0; --cl) {
245         const PetscInt cell = closure[cl*2];
246         if ((cell >= cStart) && (cell < cEnd)) {PetscCall(DMLabelSetValue(label, cell, values[v])); break;}
247       }
248       PetscCall(DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_FALSE, &closureSize, &closure));
249     }
250     PetscCall(ISRestoreIndices(pointIS, &points));
251     PetscCall(ISDestroy(&pointIS));
252   }
253   PetscCall(ISRestoreIndices(valueIS, &values));
254   PetscCall(ISDestroy(&valueIS));
255   PetscFunctionReturn(0);
256 }
257 
258 /*@
259   DMPlexLabelAddFaceCells - Starting with a label marking faces on a surface, we add a cell for each face
260 
261   Input Parameters:
262 + dm - The DM
263 - label - A DMLabel marking the surface points
264 
265   Output Parameter:
266 . label - A DMLabel incorporating cells
267 
268   Level: developer
269 
270   Note: The cells allow FEM boundary conditions to be applied using the cell geometry
271 
272 .seealso: `DMPlexLabelAddCells()`, `DMPlexLabelComplete()`, `DMPlexLabelCohesiveComplete()`
273 @*/
274 PetscErrorCode DMPlexLabelAddFaceCells(DM dm, DMLabel label)
275 {
276   IS              valueIS;
277   const PetscInt *values;
278   PetscInt        numValues, v, cStart, cEnd, fStart, fEnd;
279 
280   PetscFunctionBegin;
281   PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd));
282   PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
283   PetscCall(DMLabelGetNumValues(label, &numValues));
284   PetscCall(DMLabelGetValueIS(label, &valueIS));
285   PetscCall(ISGetIndices(valueIS, &values));
286   for (v = 0; v < numValues; ++v) {
287     IS              pointIS;
288     const PetscInt *points;
289     PetscInt        numPoints, p;
290 
291     PetscCall(DMLabelGetStratumSize(label, values[v], &numPoints));
292     PetscCall(DMLabelGetStratumIS(label, values[v], &pointIS));
293     PetscCall(ISGetIndices(pointIS, &points));
294     for (p = 0; p < numPoints; ++p) {
295       const PetscInt face = points[p];
296       PetscInt      *closure = NULL;
297       PetscInt       closureSize, cl;
298 
299       if ((face < fStart) || (face >= fEnd)) continue;
300       PetscCall(DMPlexGetTransitiveClosure(dm, face, PETSC_FALSE, &closureSize, &closure));
301       for (cl = closureSize-1; cl > 0; --cl) {
302         const PetscInt cell = closure[cl*2];
303         if ((cell >= cStart) && (cell < cEnd)) {PetscCall(DMLabelSetValue(label, cell, values[v])); break;}
304       }
305       PetscCall(DMPlexRestoreTransitiveClosure(dm, face, PETSC_FALSE, &closureSize, &closure));
306     }
307     PetscCall(ISRestoreIndices(pointIS, &points));
308     PetscCall(ISDestroy(&pointIS));
309   }
310   PetscCall(ISRestoreIndices(valueIS, &values));
311   PetscCall(ISDestroy(&valueIS));
312   PetscFunctionReturn(0);
313 }
314 
315 /*@
316   DMPlexLabelClearCells - Remove cells from a label
317 
318   Input Parameters:
319 + dm - The DM
320 - label - A DMLabel marking surface points and their adjacent cells
321 
322   Output Parameter:
323 . label - A DMLabel without cells
324 
325   Level: developer
326 
327   Note: This undoes DMPlexLabelAddCells() or DMPlexLabelAddFaceCells()
328 
329 .seealso: `DMPlexLabelComplete()`, `DMPlexLabelCohesiveComplete()`, `DMPlexLabelAddCells()`
330 @*/
331 PetscErrorCode DMPlexLabelClearCells(DM dm, DMLabel label)
332 {
333   IS              valueIS;
334   const PetscInt *values;
335   PetscInt        numValues, v, cStart, cEnd;
336 
337   PetscFunctionBegin;
338   PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd));
339   PetscCall(DMLabelGetNumValues(label, &numValues));
340   PetscCall(DMLabelGetValueIS(label, &valueIS));
341   PetscCall(ISGetIndices(valueIS, &values));
342   for (v = 0; v < numValues; ++v) {
343     IS              pointIS;
344     const PetscInt *points;
345     PetscInt        numPoints, p;
346 
347     PetscCall(DMLabelGetStratumSize(label, values[v], &numPoints));
348     PetscCall(DMLabelGetStratumIS(label, values[v], &pointIS));
349     PetscCall(ISGetIndices(pointIS, &points));
350     for (p = 0; p < numPoints; ++p) {
351       PetscInt point = points[p];
352 
353       if (point >= cStart && point < cEnd) {
354         PetscCall(DMLabelClearValue(label,point,values[v]));
355       }
356     }
357     PetscCall(ISRestoreIndices(pointIS, &points));
358     PetscCall(ISDestroy(&pointIS));
359   }
360   PetscCall(ISRestoreIndices(valueIS, &values));
361   PetscCall(ISDestroy(&valueIS));
362   PetscFunctionReturn(0);
363 }
364 
365 /* take (oldEnd, added) pairs, ordered by height and convert them to (oldstart, newstart) pairs, ordered by ascending
366  * index (skipping first, which is (0,0)) */
367 static inline PetscErrorCode DMPlexShiftPointSetUp_Internal(PetscInt depth, PetscInt depthShift[])
368 {
369   PetscInt d, off = 0;
370 
371   PetscFunctionBegin;
372   /* sort by (oldend): yes this is an O(n^2) sort, we expect depth <= 3 */
373   for (d = 0; d < depth; d++) {
374     PetscInt firstd = d;
375     PetscInt firstStart = depthShift[2*d];
376     PetscInt e;
377 
378     for (e = d+1; e <= depth; e++) {
379       if (depthShift[2*e] < firstStart) {
380         firstd = e;
381         firstStart = depthShift[2*d];
382       }
383     }
384     if (firstd != d) {
385       PetscInt swap[2];
386 
387       e = firstd;
388       swap[0] = depthShift[2*d];
389       swap[1] = depthShift[2*d+1];
390       depthShift[2*d]   = depthShift[2*e];
391       depthShift[2*d+1] = depthShift[2*e+1];
392       depthShift[2*e]   = swap[0];
393       depthShift[2*e+1] = swap[1];
394     }
395   }
396   /* convert (oldstart, added) to (oldstart, newstart) */
397   for (d = 0; d <= depth; d++) {
398     off += depthShift[2*d+1];
399     depthShift[2*d+1] = depthShift[2*d] + off;
400   }
401   PetscFunctionReturn(0);
402 }
403 
404 /* depthShift is a list of (old, new) pairs */
405 static inline PetscInt DMPlexShiftPoint_Internal(PetscInt p, PetscInt depth, PetscInt depthShift[])
406 {
407   PetscInt d;
408   PetscInt newOff = 0;
409 
410   for (d = 0; d <= depth; d++) {
411     if (p < depthShift[2*d]) return p + newOff;
412     else newOff = depthShift[2*d+1] - depthShift[2*d];
413   }
414   return p + newOff;
415 }
416 
417 /* depthShift is a list of (old, new) pairs */
418 static inline PetscInt DMPlexShiftPointInverse_Internal(PetscInt p, PetscInt depth, PetscInt depthShift[])
419 {
420   PetscInt d;
421   PetscInt newOff = 0;
422 
423   for (d = 0; d <= depth; d++) {
424     if (p < depthShift[2*d+1]) return p + newOff;
425     else newOff = depthShift[2*d] - depthShift[2*d+1];
426   }
427   return p + newOff;
428 }
429 
430 static PetscErrorCode DMPlexShiftSizes_Internal(DM dm, PetscInt depthShift[], DM dmNew)
431 {
432   PetscInt       depth = 0, d, pStart, pEnd, p;
433   DMLabel        depthLabel;
434 
435   PetscFunctionBegin;
436   PetscCall(DMPlexGetDepth(dm, &depth));
437   if (depth < 0) PetscFunctionReturn(0);
438   /* Step 1: Expand chart */
439   PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
440   pEnd = DMPlexShiftPoint_Internal(pEnd,depth,depthShift);
441   PetscCall(DMPlexSetChart(dmNew, pStart, pEnd));
442   PetscCall(DMCreateLabel(dmNew,"depth"));
443   PetscCall(DMPlexGetDepthLabel(dmNew,&depthLabel));
444   PetscCall(DMCreateLabel(dmNew, "celltype"));
445   /* Step 2: Set cone and support sizes */
446   for (d = 0; d <= depth; ++d) {
447     PetscInt pStartNew, pEndNew;
448     IS pIS;
449 
450     PetscCall(DMPlexGetDepthStratum(dm, d, &pStart, &pEnd));
451     pStartNew = DMPlexShiftPoint_Internal(pStart, depth, depthShift);
452     pEndNew = DMPlexShiftPoint_Internal(pEnd, depth, depthShift);
453     PetscCall(ISCreateStride(PETSC_COMM_SELF, pEndNew - pStartNew, pStartNew, 1, &pIS));
454     PetscCall(DMLabelSetStratumIS(depthLabel, d, pIS));
455     PetscCall(ISDestroy(&pIS));
456     for (p = pStart; p < pEnd; ++p) {
457       PetscInt       newp = DMPlexShiftPoint_Internal(p, depth, depthShift);
458       PetscInt       size;
459       DMPolytopeType ct;
460 
461       PetscCall(DMPlexGetConeSize(dm, p, &size));
462       PetscCall(DMPlexSetConeSize(dmNew, newp, size));
463       PetscCall(DMPlexGetSupportSize(dm, p, &size));
464       PetscCall(DMPlexSetSupportSize(dmNew, newp, size));
465       PetscCall(DMPlexGetCellType(dm, p, &ct));
466       PetscCall(DMPlexSetCellType(dmNew, newp, ct));
467     }
468   }
469   PetscFunctionReturn(0);
470 }
471 
472 static PetscErrorCode DMPlexShiftPoints_Internal(DM dm, PetscInt depthShift[], DM dmNew)
473 {
474   PetscInt      *newpoints;
475   PetscInt       depth = 0, maxConeSize, maxSupportSize, maxConeSizeNew, maxSupportSizeNew, pStart, pEnd, p;
476 
477   PetscFunctionBegin;
478   PetscCall(DMPlexGetDepth(dm, &depth));
479   if (depth < 0) PetscFunctionReturn(0);
480   PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize));
481   PetscCall(DMPlexGetMaxSizes(dmNew, &maxConeSizeNew, &maxSupportSizeNew));
482   PetscCall(PetscMalloc1(PetscMax(PetscMax(maxConeSize, maxSupportSize), PetscMax(maxConeSizeNew, maxSupportSizeNew)),&newpoints));
483   /* Step 5: Set cones and supports */
484   PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
485   for (p = pStart; p < pEnd; ++p) {
486     const PetscInt *points = NULL, *orientations = NULL;
487     PetscInt        size,sizeNew, i, newp = DMPlexShiftPoint_Internal(p, depth, depthShift);
488 
489     PetscCall(DMPlexGetConeSize(dm, p, &size));
490     PetscCall(DMPlexGetCone(dm, p, &points));
491     PetscCall(DMPlexGetConeOrientation(dm, p, &orientations));
492     for (i = 0; i < size; ++i) {
493       newpoints[i] = DMPlexShiftPoint_Internal(points[i], depth, depthShift);
494     }
495     PetscCall(DMPlexSetCone(dmNew, newp, newpoints));
496     PetscCall(DMPlexSetConeOrientation(dmNew, newp, orientations));
497     PetscCall(DMPlexGetSupportSize(dm, p, &size));
498     PetscCall(DMPlexGetSupportSize(dmNew, newp, &sizeNew));
499     PetscCall(DMPlexGetSupport(dm, p, &points));
500     for (i = 0; i < size; ++i) {
501       newpoints[i] = DMPlexShiftPoint_Internal(points[i], depth, depthShift);
502     }
503     for (i = size; i < sizeNew; ++i) newpoints[i] = 0;
504     PetscCall(DMPlexSetSupport(dmNew, newp, newpoints));
505   }
506   PetscCall(PetscFree(newpoints));
507   PetscFunctionReturn(0);
508 }
509 
510 static PetscErrorCode DMPlexShiftCoordinates_Internal(DM dm, PetscInt depthShift[], DM dmNew)
511 {
512   PetscSection   coordSection, newCoordSection;
513   Vec            coordinates, newCoordinates;
514   PetscScalar   *coords, *newCoords;
515   PetscInt       coordSize, sStart, sEnd;
516   PetscInt       dim, depth = 0, cStart, cEnd, cStartNew, cEndNew, c, vStart, vEnd, vStartNew, vEndNew, v;
517   PetscBool      hasCells;
518 
519   PetscFunctionBegin;
520   PetscCall(DMGetCoordinateDim(dm, &dim));
521   PetscCall(DMSetCoordinateDim(dmNew, dim));
522   PetscCall(DMPlexGetDepth(dm, &depth));
523   /* Step 8: Convert coordinates */
524   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
525   PetscCall(DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd));
526   PetscCall(DMPlexGetDepthStratum(dmNew, 0, &vStartNew, &vEndNew));
527   PetscCall(DMPlexGetHeightStratum(dmNew, 0, &cStartNew, &cEndNew));
528   PetscCall(DMGetCoordinateSection(dm, &coordSection));
529   PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dm), &newCoordSection));
530   PetscCall(PetscSectionSetNumFields(newCoordSection, 1));
531   PetscCall(PetscSectionSetFieldComponents(newCoordSection, 0, dim));
532   PetscCall(PetscSectionGetChart(coordSection, &sStart, &sEnd));
533   hasCells = sStart == cStart ? PETSC_TRUE : PETSC_FALSE;
534   PetscCall(PetscSectionSetChart(newCoordSection, hasCells ? cStartNew : vStartNew, vEndNew));
535   if (hasCells) {
536     for (c = cStart; c < cEnd; ++c) {
537       PetscInt cNew = DMPlexShiftPoint_Internal(c, depth, depthShift), dof;
538 
539       PetscCall(PetscSectionGetDof(coordSection, c, &dof));
540       PetscCall(PetscSectionSetDof(newCoordSection, cNew, dof));
541       PetscCall(PetscSectionSetFieldDof(newCoordSection, cNew, 0, dof));
542     }
543   }
544   for (v = vStartNew; v < vEndNew; ++v) {
545     PetscCall(PetscSectionSetDof(newCoordSection, v, dim));
546     PetscCall(PetscSectionSetFieldDof(newCoordSection, v, 0, dim));
547   }
548   PetscCall(PetscSectionSetUp(newCoordSection));
549   PetscCall(DMSetCoordinateSection(dmNew, PETSC_DETERMINE, newCoordSection));
550   PetscCall(PetscSectionGetStorageSize(newCoordSection, &coordSize));
551   PetscCall(VecCreate(PETSC_COMM_SELF, &newCoordinates));
552   PetscCall(PetscObjectSetName((PetscObject) newCoordinates, "coordinates"));
553   PetscCall(VecSetSizes(newCoordinates, coordSize, PETSC_DETERMINE));
554   PetscCall(VecSetBlockSize(newCoordinates, dim));
555   PetscCall(VecSetType(newCoordinates,VECSTANDARD));
556   PetscCall(DMSetCoordinatesLocal(dmNew, newCoordinates));
557   PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
558   PetscCall(VecGetArray(coordinates, &coords));
559   PetscCall(VecGetArray(newCoordinates, &newCoords));
560   if (hasCells) {
561     for (c = cStart; c < cEnd; ++c) {
562       PetscInt cNew = DMPlexShiftPoint_Internal(c, depth, depthShift), dof, off, noff, d;
563 
564       PetscCall(PetscSectionGetDof(coordSection, c, &dof));
565       PetscCall(PetscSectionGetOffset(coordSection, c, &off));
566       PetscCall(PetscSectionGetOffset(newCoordSection, cNew, &noff));
567       for (d = 0; d < dof; ++d) newCoords[noff+d] = coords[off+d];
568     }
569   }
570   for (v = vStart; v < vEnd; ++v) {
571     PetscInt dof, off, noff, d;
572 
573     PetscCall(PetscSectionGetDof(coordSection, v, &dof));
574     PetscCall(PetscSectionGetOffset(coordSection, v, &off));
575     PetscCall(PetscSectionGetOffset(newCoordSection, DMPlexShiftPoint_Internal(v, depth, depthShift), &noff));
576     for (d = 0; d < dof; ++d) newCoords[noff+d] = coords[off+d];
577   }
578   PetscCall(VecRestoreArray(coordinates, &coords));
579   PetscCall(VecRestoreArray(newCoordinates, &newCoords));
580   PetscCall(VecDestroy(&newCoordinates));
581   PetscCall(PetscSectionDestroy(&newCoordSection));
582   PetscFunctionReturn(0);
583 }
584 
585 static PetscErrorCode DMPlexShiftSF_Single(DM dm, PetscInt depthShift[], PetscSF sf, PetscSF sfNew)
586 {
587   const PetscSFNode *remotePoints;
588   PetscSFNode       *gremotePoints;
589   const PetscInt    *localPoints;
590   PetscInt          *glocalPoints, *newLocation, *newRemoteLocation;
591   PetscInt           numRoots, numLeaves, l, pStart, pEnd, depth = 0, totShift = 0;
592 
593   PetscFunctionBegin;
594   PetscCall(DMPlexGetDepth(dm, &depth));
595   PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
596   PetscCall(PetscSFGetGraph(sf, &numRoots, &numLeaves, &localPoints, &remotePoints));
597   totShift = DMPlexShiftPoint_Internal(pEnd, depth, depthShift) - pEnd;
598   if (numRoots >= 0) {
599     PetscCall(PetscMalloc2(numRoots, &newLocation, pEnd-pStart, &newRemoteLocation));
600     for (l = 0; l < numRoots; ++l) newLocation[l] = DMPlexShiftPoint_Internal(l, depth, depthShift);
601     PetscCall(PetscSFBcastBegin(sf, MPIU_INT, newLocation, newRemoteLocation, MPI_REPLACE));
602     PetscCall(PetscSFBcastEnd(sf, MPIU_INT, newLocation, newRemoteLocation, MPI_REPLACE));
603     PetscCall(PetscMalloc1(numLeaves, &glocalPoints));
604     PetscCall(PetscMalloc1(numLeaves, &gremotePoints));
605     for (l = 0; l < numLeaves; ++l) {
606       glocalPoints[l]        = DMPlexShiftPoint_Internal(localPoints[l], depth, depthShift);
607       gremotePoints[l].rank  = remotePoints[l].rank;
608       gremotePoints[l].index = newRemoteLocation[localPoints[l]];
609     }
610     PetscCall(PetscFree2(newLocation, newRemoteLocation));
611     PetscCall(PetscSFSetGraph(sfNew, numRoots + totShift, numLeaves, glocalPoints, PETSC_OWN_POINTER, gremotePoints, PETSC_OWN_POINTER));
612   }
613   PetscFunctionReturn(0);
614 }
615 
616 static PetscErrorCode DMPlexShiftSF_Internal(DM dm, PetscInt depthShift[], DM dmNew)
617 {
618   PetscSF        sfPoint, sfPointNew;
619   PetscBool      useNatural;
620 
621   PetscFunctionBegin;
622   /* Step 9: Convert pointSF */
623   PetscCall(DMGetPointSF(dm, &sfPoint));
624   PetscCall(DMGetPointSF(dmNew, &sfPointNew));
625   PetscCall(DMPlexShiftSF_Single(dm, depthShift, sfPoint, sfPointNew));
626   /* Step 9b: Convert naturalSF */
627   PetscCall(DMGetUseNatural(dm, &useNatural));
628   if (useNatural) {
629     PetscSF sfNat, sfNatNew;
630 
631     PetscCall(DMSetUseNatural(dmNew, useNatural));
632     PetscCall(DMGetNaturalSF(dm, &sfNat));
633     PetscCall(DMGetNaturalSF(dmNew, &sfNatNew));
634     PetscCall(DMPlexShiftSF_Single(dm, depthShift, sfNat, sfNatNew));
635   }
636   PetscFunctionReturn(0);
637 }
638 
639 static PetscErrorCode DMPlexShiftLabels_Internal(DM dm, PetscInt depthShift[], DM dmNew)
640 {
641   PetscInt           depth = 0, numLabels, l;
642 
643   PetscFunctionBegin;
644   PetscCall(DMPlexGetDepth(dm, &depth));
645   /* Step 10: Convert labels */
646   PetscCall(DMGetNumLabels(dm, &numLabels));
647   for (l = 0; l < numLabels; ++l) {
648     DMLabel         label, newlabel;
649     const char     *lname;
650     PetscBool       isDepth, isDim;
651     IS              valueIS;
652     const PetscInt *values;
653     PetscInt        numValues, val;
654 
655     PetscCall(DMGetLabelName(dm, l, &lname));
656     PetscCall(PetscStrcmp(lname, "depth", &isDepth));
657     if (isDepth) continue;
658     PetscCall(PetscStrcmp(lname, "dim", &isDim));
659     if (isDim) continue;
660     PetscCall(DMCreateLabel(dmNew, lname));
661     PetscCall(DMGetLabel(dm, lname, &label));
662     PetscCall(DMGetLabel(dmNew, lname, &newlabel));
663     PetscCall(DMLabelGetDefaultValue(label,&val));
664     PetscCall(DMLabelSetDefaultValue(newlabel,val));
665     PetscCall(DMLabelGetValueIS(label, &valueIS));
666     PetscCall(ISGetLocalSize(valueIS, &numValues));
667     PetscCall(ISGetIndices(valueIS, &values));
668     for (val = 0; val < numValues; ++val) {
669       IS              pointIS;
670       const PetscInt *points;
671       PetscInt        numPoints, p;
672 
673       PetscCall(DMLabelGetStratumIS(label, values[val], &pointIS));
674       PetscCall(ISGetLocalSize(pointIS, &numPoints));
675       PetscCall(ISGetIndices(pointIS, &points));
676       for (p = 0; p < numPoints; ++p) {
677         const PetscInt newpoint = DMPlexShiftPoint_Internal(points[p], depth, depthShift);
678 
679         PetscCall(DMLabelSetValue(newlabel, newpoint, values[val]));
680       }
681       PetscCall(ISRestoreIndices(pointIS, &points));
682       PetscCall(ISDestroy(&pointIS));
683     }
684     PetscCall(ISRestoreIndices(valueIS, &values));
685     PetscCall(ISDestroy(&valueIS));
686   }
687   PetscFunctionReturn(0);
688 }
689 
690 static PetscErrorCode DMPlexCreateVTKLabel_Internal(DM dm, PetscBool createGhostLabel, DM dmNew)
691 {
692   PetscSF            sfPoint;
693   DMLabel            vtkLabel, ghostLabel = NULL;
694   const PetscSFNode *leafRemote;
695   const PetscInt    *leafLocal;
696   PetscInt           cellHeight, cStart, cEnd, c, fStart, fEnd, f, numLeaves, l;
697   PetscMPIInt        rank;
698 
699   PetscFunctionBegin;
700    /* Step 11: Make label for output (vtk) and to mark ghost points (ghost) */
701   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
702   PetscCall(DMGetPointSF(dm, &sfPoint));
703   PetscCall(DMPlexGetVTKCellHeight(dmNew, &cellHeight));
704   PetscCall(DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd));
705   PetscCall(PetscSFGetGraph(sfPoint, NULL, &numLeaves, &leafLocal, &leafRemote));
706   PetscCall(DMCreateLabel(dmNew, "vtk"));
707   PetscCall(DMGetLabel(dmNew, "vtk", &vtkLabel));
708   if (createGhostLabel) {
709     PetscCall(DMCreateLabel(dmNew, "ghost"));
710     PetscCall(DMGetLabel(dmNew, "ghost", &ghostLabel));
711   }
712   for (l = 0, c = cStart; l < numLeaves && c < cEnd; ++l, ++c) {
713     for (; c < leafLocal[l] && c < cEnd; ++c) {
714       PetscCall(DMLabelSetValue(vtkLabel, c, 1));
715     }
716     if (leafLocal[l] >= cEnd) break;
717     if (leafRemote[l].rank == rank) {
718       PetscCall(DMLabelSetValue(vtkLabel, c, 1));
719     } else if (ghostLabel) {
720       PetscCall(DMLabelSetValue(ghostLabel, c, 2));
721     }
722   }
723   for (; c < cEnd; ++c) {
724     PetscCall(DMLabelSetValue(vtkLabel, c, 1));
725   }
726   if (ghostLabel) {
727     PetscCall(DMPlexGetHeightStratum(dmNew, 1, &fStart, &fEnd));
728     for (f = fStart; f < fEnd; ++f) {
729       PetscInt numCells;
730 
731       PetscCall(DMPlexGetSupportSize(dmNew, f, &numCells));
732       if (numCells < 2) {
733         PetscCall(DMLabelSetValue(ghostLabel, f, 1));
734       } else {
735         const PetscInt *cells = NULL;
736         PetscInt        vA, vB;
737 
738         PetscCall(DMPlexGetSupport(dmNew, f, &cells));
739         PetscCall(DMLabelGetValue(vtkLabel, cells[0], &vA));
740         PetscCall(DMLabelGetValue(vtkLabel, cells[1], &vB));
741         if (vA != 1 && vB != 1) PetscCall(DMLabelSetValue(ghostLabel, f, 1));
742       }
743     }
744   }
745   PetscFunctionReturn(0);
746 }
747 
748 static PetscErrorCode DMPlexShiftTree_Internal(DM dm, PetscInt depthShift[], DM dmNew)
749 {
750   DM             refTree;
751   PetscSection   pSec;
752   PetscInt       *parents, *childIDs;
753 
754   PetscFunctionBegin;
755   PetscCall(DMPlexGetReferenceTree(dm,&refTree));
756   PetscCall(DMPlexSetReferenceTree(dmNew,refTree));
757   PetscCall(DMPlexGetTree(dm,&pSec,&parents,&childIDs,NULL,NULL));
758   if (pSec) {
759     PetscInt p, pStart, pEnd, *parentsShifted, pStartShifted, pEndShifted, depth;
760     PetscInt *childIDsShifted;
761     PetscSection pSecShifted;
762 
763     PetscCall(PetscSectionGetChart(pSec,&pStart,&pEnd));
764     PetscCall(DMPlexGetDepth(dm,&depth));
765     pStartShifted = DMPlexShiftPoint_Internal(pStart,depth,depthShift);
766     pEndShifted   = DMPlexShiftPoint_Internal(pEnd,depth,depthShift);
767     PetscCall(PetscMalloc2(pEndShifted - pStartShifted,&parentsShifted,pEndShifted-pStartShifted,&childIDsShifted));
768     PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dmNew),&pSecShifted));
769     PetscCall(PetscSectionSetChart(pSecShifted,pStartShifted,pEndShifted));
770     for (p = pStartShifted; p < pEndShifted; p++) {
771       /* start off assuming no children */
772       PetscCall(PetscSectionSetDof(pSecShifted,p,0));
773     }
774     for (p = pStart; p < pEnd; p++) {
775       PetscInt dof;
776       PetscInt pNew = DMPlexShiftPoint_Internal(p,depth,depthShift);
777 
778       PetscCall(PetscSectionGetDof(pSec,p,&dof));
779       PetscCall(PetscSectionSetDof(pSecShifted,pNew,dof));
780     }
781     PetscCall(PetscSectionSetUp(pSecShifted));
782     for (p = pStart; p < pEnd; p++) {
783       PetscInt dof;
784       PetscInt pNew = DMPlexShiftPoint_Internal(p,depth,depthShift);
785 
786       PetscCall(PetscSectionGetDof(pSec,p,&dof));
787       if (dof) {
788         PetscInt off, offNew;
789 
790         PetscCall(PetscSectionGetOffset(pSec,p,&off));
791         PetscCall(PetscSectionGetOffset(pSecShifted,pNew,&offNew));
792         parentsShifted[offNew] = DMPlexShiftPoint_Internal(parents[off],depth,depthShift);
793         childIDsShifted[offNew] = childIDs[off];
794       }
795     }
796     PetscCall(DMPlexSetTree(dmNew,pSecShifted,parentsShifted,childIDsShifted));
797     PetscCall(PetscFree2(parentsShifted,childIDsShifted));
798     PetscCall(PetscSectionDestroy(&pSecShifted));
799   }
800   PetscFunctionReturn(0);
801 }
802 
803 static PetscErrorCode DMPlexConstructGhostCells_Internal(DM dm, DMLabel label, PetscInt *numGhostCells, DM gdm)
804 {
805   PetscSF               sf;
806   IS                    valueIS;
807   const PetscInt       *values, *leaves;
808   PetscInt             *depthShift;
809   PetscInt              d, depth = 0, nleaves, loc, Ng, numFS, fs, fStart, fEnd, ghostCell, cEnd, c;
810   PetscBool             isper;
811   const PetscReal      *maxCell, *L;
812   const DMBoundaryType *bd;
813 
814   PetscFunctionBegin;
815   PetscCall(DMGetPointSF(dm, &sf));
816   PetscCall(PetscSFGetGraph(sf, NULL, &nleaves, &leaves, NULL));
817   nleaves = PetscMax(0, nleaves);
818   PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
819   /* Count ghost cells */
820   PetscCall(DMLabelGetValueIS(label, &valueIS));
821   PetscCall(ISGetLocalSize(valueIS, &numFS));
822   PetscCall(ISGetIndices(valueIS, &values));
823   Ng   = 0;
824   for (fs = 0; fs < numFS; ++fs) {
825     IS              faceIS;
826     const PetscInt *faces;
827     PetscInt        numFaces, f, numBdFaces = 0;
828 
829     PetscCall(DMLabelGetStratumIS(label, values[fs], &faceIS));
830     PetscCall(ISGetLocalSize(faceIS, &numFaces));
831     PetscCall(ISGetIndices(faceIS, &faces));
832     for (f = 0; f < numFaces; ++f) {
833       PetscInt numChildren;
834 
835       PetscCall(PetscFindInt(faces[f], nleaves, leaves, &loc));
836       PetscCall(DMPlexGetTreeChildren(dm,faces[f],&numChildren,NULL));
837       /* non-local and ancestors points don't get to register ghosts */
838       if (loc >= 0 || numChildren) continue;
839       if ((faces[f] >= fStart) && (faces[f] < fEnd)) ++numBdFaces;
840     }
841     Ng += numBdFaces;
842     PetscCall(ISRestoreIndices(faceIS, &faces));
843     PetscCall(ISDestroy(&faceIS));
844   }
845   PetscCall(DMPlexGetDepth(dm, &depth));
846   PetscCall(PetscMalloc1(2*(depth+1), &depthShift));
847   for (d = 0; d <= depth; d++) {
848     PetscInt dEnd;
849 
850     PetscCall(DMPlexGetDepthStratum(dm,d,NULL,&dEnd));
851     depthShift[2*d]   = dEnd;
852     depthShift[2*d+1] = 0;
853   }
854   if (depth >= 0) depthShift[2*depth+1] = Ng;
855   PetscCall(DMPlexShiftPointSetUp_Internal(depth,depthShift));
856   PetscCall(DMPlexShiftSizes_Internal(dm, depthShift, gdm));
857   /* Step 3: Set cone/support sizes for new points */
858   PetscCall(DMPlexGetHeightStratum(dm, 0, NULL, &cEnd));
859   for (c = cEnd; c < cEnd + Ng; ++c) {
860     PetscCall(DMPlexSetConeSize(gdm, c, 1));
861   }
862   for (fs = 0; fs < numFS; ++fs) {
863     IS              faceIS;
864     const PetscInt *faces;
865     PetscInt        numFaces, f;
866 
867     PetscCall(DMLabelGetStratumIS(label, values[fs], &faceIS));
868     PetscCall(ISGetLocalSize(faceIS, &numFaces));
869     PetscCall(ISGetIndices(faceIS, &faces));
870     for (f = 0; f < numFaces; ++f) {
871       PetscInt size, numChildren;
872 
873       PetscCall(PetscFindInt(faces[f], nleaves, leaves, &loc));
874       PetscCall(DMPlexGetTreeChildren(dm,faces[f],&numChildren,NULL));
875       if (loc >= 0 || numChildren) continue;
876       if ((faces[f] < fStart) || (faces[f] >= fEnd)) continue;
877       PetscCall(DMPlexGetSupportSize(dm, faces[f], &size));
878       PetscCheck(size == 1,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM has boundary face %" PetscInt_FMT " with %" PetscInt_FMT " support cells", faces[f], size);
879       PetscCall(DMPlexSetSupportSize(gdm, faces[f] + Ng, 2));
880     }
881     PetscCall(ISRestoreIndices(faceIS, &faces));
882     PetscCall(ISDestroy(&faceIS));
883   }
884   /* Step 4: Setup ghosted DM */
885   PetscCall(DMSetUp(gdm));
886   PetscCall(DMPlexShiftPoints_Internal(dm, depthShift, gdm));
887   /* Step 6: Set cones and supports for new points */
888   ghostCell = cEnd;
889   for (fs = 0; fs < numFS; ++fs) {
890     IS              faceIS;
891     const PetscInt *faces;
892     PetscInt        numFaces, f;
893 
894     PetscCall(DMLabelGetStratumIS(label, values[fs], &faceIS));
895     PetscCall(ISGetLocalSize(faceIS, &numFaces));
896     PetscCall(ISGetIndices(faceIS, &faces));
897     for (f = 0; f < numFaces; ++f) {
898       PetscInt newFace = faces[f] + Ng, numChildren;
899 
900       PetscCall(PetscFindInt(faces[f], nleaves, leaves, &loc));
901       PetscCall(DMPlexGetTreeChildren(dm,faces[f],&numChildren,NULL));
902       if (loc >= 0 || numChildren) continue;
903       if ((faces[f] < fStart) || (faces[f] >= fEnd)) continue;
904       PetscCall(DMPlexSetCone(gdm, ghostCell, &newFace));
905       PetscCall(DMPlexInsertSupport(gdm, newFace, 1, ghostCell));
906       ++ghostCell;
907     }
908     PetscCall(ISRestoreIndices(faceIS, &faces));
909     PetscCall(ISDestroy(&faceIS));
910   }
911   PetscCall(ISRestoreIndices(valueIS, &values));
912   PetscCall(ISDestroy(&valueIS));
913   PetscCall(DMPlexShiftCoordinates_Internal(dm, depthShift, gdm));
914   PetscCall(DMPlexShiftSF_Internal(dm, depthShift, gdm));
915   PetscCall(DMPlexShiftLabels_Internal(dm, depthShift, gdm));
916   PetscCall(DMPlexCreateVTKLabel_Internal(dm, PETSC_TRUE, gdm));
917   PetscCall(DMPlexShiftTree_Internal(dm, depthShift, gdm));
918   PetscCall(PetscFree(depthShift));
919   for (c = cEnd; c < cEnd + Ng; ++c) {
920     PetscCall(DMPlexSetCellType(gdm, c, DM_POLYTOPE_FV_GHOST));
921   }
922   /* Step 7: Periodicity */
923   PetscCall(DMGetPeriodicity(dm, &isper, &maxCell, &L, &bd));
924   PetscCall(DMSetPeriodicity(gdm, isper, maxCell,  L,  bd));
925   if (numGhostCells) *numGhostCells = Ng;
926   PetscFunctionReturn(0);
927 }
928 
929 /*@C
930   DMPlexConstructGhostCells - Construct ghost cells which connect to every boundary face
931 
932   Collective on dm
933 
934   Input Parameters:
935 + dm - The original DM
936 - labelName - The label specifying the boundary faces, or "Face Sets" if this is NULL
937 
938   Output Parameters:
939 + numGhostCells - The number of ghost cells added to the DM
940 - dmGhosted - The new DM
941 
942   Note: If no label exists of that name, one will be created marking all boundary faces
943 
944   Level: developer
945 
946 .seealso: `DMCreate()`
947 @*/
948 PetscErrorCode DMPlexConstructGhostCells(DM dm, const char labelName[], PetscInt *numGhostCells, DM *dmGhosted)
949 {
950   DM             gdm;
951   DMLabel        label;
952   const char    *name = labelName ? labelName : "Face Sets";
953   PetscInt       dim, Ng = 0;
954   PetscBool      useCone, useClosure;
955 
956   PetscFunctionBegin;
957   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
958   if (numGhostCells) PetscValidIntPointer(numGhostCells, 3);
959   PetscValidPointer(dmGhosted, 4);
960   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &gdm));
961   PetscCall(DMSetType(gdm, DMPLEX));
962   PetscCall(DMGetDimension(dm, &dim));
963   PetscCall(DMSetDimension(gdm, dim));
964   PetscCall(DMGetBasicAdjacency(dm, &useCone, &useClosure));
965   PetscCall(DMSetBasicAdjacency(gdm, useCone,  useClosure));
966   PetscCall(DMGetLabel(dm, name, &label));
967   if (!label) {
968     /* Get label for boundary faces */
969     PetscCall(DMCreateLabel(dm, name));
970     PetscCall(DMGetLabel(dm, name, &label));
971     PetscCall(DMPlexMarkBoundaryFaces(dm, 1, label));
972   }
973   PetscCall(DMPlexConstructGhostCells_Internal(dm, label, &Ng, gdm));
974   PetscCall(DMCopyDisc(dm, gdm));
975   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, gdm));
976   gdm->setfromoptionscalled = dm->setfromoptionscalled;
977   if (numGhostCells) *numGhostCells = Ng;
978   *dmGhosted = gdm;
979   PetscFunctionReturn(0);
980 }
981 
982 static PetscErrorCode DivideCells_Private(DM dm, DMLabel label, DMPlexPointQueue queue)
983 {
984   PetscInt dim, d, shift = 100, *pStart, *pEnd;
985 
986   PetscFunctionBegin;
987   PetscCall(DMGetDimension(dm, &dim));
988   PetscCall(PetscMalloc2(dim, &pStart, dim, &pEnd));
989   for (d = 0; d < dim; ++d) {
990     PetscCall(DMPlexGetDepthStratum(dm, d, &pStart[d], &pEnd[d]));
991   }
992   while (!DMPlexPointQueueEmpty(queue)) {
993     PetscInt  cell    = -1;
994     PetscInt *closure = NULL;
995     PetscInt  closureSize, cl, cval;
996 
997     PetscCall(DMPlexPointQueueDequeue(queue, &cell));
998     PetscCall(DMLabelGetValue(label, cell, &cval));
999     PetscCall(DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
1000     /* Mark points in the cell closure that touch the fault */
1001     for (d = 0; d < dim; ++d) {
1002       for (cl = 0; cl < closureSize*2; cl += 2) {
1003         const PetscInt clp = closure[cl];
1004         PetscInt       clval;
1005 
1006         if ((clp < pStart[d]) || (clp >= pEnd[d])) continue;
1007         PetscCall(DMLabelGetValue(label, clp, &clval));
1008         if (clval == -1) {
1009           const PetscInt *cone;
1010           PetscInt        coneSize, c;
1011 
1012           /* If a cone point touches the fault, then this point touches the fault */
1013           PetscCall(DMPlexGetCone(dm, clp, &cone));
1014           PetscCall(DMPlexGetConeSize(dm, clp, &coneSize));
1015           for (c = 0; c < coneSize; ++c) {
1016             PetscInt cpval;
1017 
1018             PetscCall(DMLabelGetValue(label, cone[c], &cpval));
1019             if (cpval != -1) {
1020               PetscInt dep;
1021 
1022               PetscCall(DMPlexGetPointDepth(dm, clp, &dep));
1023               clval = cval < 0 ? -(shift+dep) : shift+dep;
1024               PetscCall(DMLabelSetValue(label, clp, clval));
1025               break;
1026             }
1027           }
1028         }
1029         /* Mark neighbor cells through marked faces (these cells must also touch the fault) */
1030         if (d == dim-1 && clval != -1) {
1031           const PetscInt *support;
1032           PetscInt        supportSize, s, nval;
1033 
1034           PetscCall(DMPlexGetSupport(dm, clp, &support));
1035           PetscCall(DMPlexGetSupportSize(dm, clp, &supportSize));
1036           for (s = 0; s < supportSize; ++s) {
1037             PetscCall(DMLabelGetValue(label, support[s], &nval));
1038             if (nval == -1) {
1039               PetscCall(DMLabelSetValue(label, support[s], clval < 0 ? clval-1 : clval+1));
1040               PetscCall(DMPlexPointQueueEnqueue(queue, support[s]));
1041             }
1042           }
1043         }
1044       }
1045     }
1046     PetscCall(DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
1047   }
1048   PetscCall(PetscFree2(pStart, pEnd));
1049   PetscFunctionReturn(0);
1050 }
1051 
1052 typedef struct {
1053   DM               dm;
1054   DMPlexPointQueue queue;
1055 } PointDivision;
1056 
1057 static PetscErrorCode divideCell(DMLabel label, PetscInt p, PetscInt val, void *ctx)
1058 {
1059   PointDivision  *div  = (PointDivision *) ctx;
1060   PetscInt        cval = val < 0 ? val-1 : val+1;
1061   const PetscInt *support;
1062   PetscInt        supportSize, s;
1063 
1064   PetscFunctionBegin;
1065   PetscCall(DMPlexGetSupport(div->dm, p, &support));
1066   PetscCall(DMPlexGetSupportSize(div->dm, p, &supportSize));
1067   for (s = 0; s < supportSize; ++s) {
1068     PetscCall(DMLabelSetValue(label, support[s], cval));
1069     PetscCall(DMPlexPointQueueEnqueue(div->queue, support[s]));
1070   }
1071   PetscFunctionReturn(0);
1072 }
1073 
1074 /* Mark cells by label propagation */
1075 static PetscErrorCode DMPlexLabelFaultHalo(DM dm, DMLabel faultLabel)
1076 {
1077   DMPlexPointQueue queue = NULL;
1078   PointDivision    div;
1079   PetscSF          pointSF;
1080   IS               pointIS;
1081   const PetscInt  *points;
1082   PetscBool        empty;
1083   PetscInt         dim, shift = 100, n, i;
1084 
1085   PetscFunctionBegin;
1086   PetscCall(DMGetDimension(dm, &dim));
1087   PetscCall(DMPlexPointQueueCreate(1024, &queue));
1088   div.dm    = dm;
1089   div.queue = queue;
1090   /* Enqueue cells on fault */
1091   PetscCall(DMLabelGetStratumIS(faultLabel, shift+dim, &pointIS));
1092   if (pointIS) {
1093     PetscCall(ISGetLocalSize(pointIS, &n));
1094     PetscCall(ISGetIndices(pointIS, &points));
1095     for (i = 0; i < n; ++i) {PetscCall(DMPlexPointQueueEnqueue(queue, points[i]));}
1096     PetscCall(ISRestoreIndices(pointIS, &points));
1097     PetscCall(ISDestroy(&pointIS));
1098   }
1099   PetscCall(DMLabelGetStratumIS(faultLabel, -(shift+dim), &pointIS));
1100   if (pointIS) {
1101     PetscCall(ISGetLocalSize(pointIS, &n));
1102     PetscCall(ISGetIndices(pointIS, &points));
1103     for (i = 0; i < n; ++i) {PetscCall(DMPlexPointQueueEnqueue(queue, points[i]));}
1104     PetscCall(ISRestoreIndices(pointIS, &points));
1105     PetscCall(ISDestroy(&pointIS));
1106   }
1107 
1108   PetscCall(DMGetPointSF(dm, &pointSF));
1109   PetscCall(DMLabelPropagateBegin(faultLabel, pointSF));
1110   /* While edge queue is not empty: */
1111   PetscCall(DMPlexPointQueueEmptyCollective((PetscObject) dm, queue, &empty));
1112   while (!empty) {
1113     PetscCall(DivideCells_Private(dm, faultLabel, queue));
1114     PetscCall(DMLabelPropagatePush(faultLabel, pointSF, divideCell, &div));
1115     PetscCall(DMPlexPointQueueEmptyCollective((PetscObject) dm, queue, &empty));
1116   }
1117   PetscCall(DMLabelPropagateEnd(faultLabel, pointSF));
1118   PetscCall(DMPlexPointQueueDestroy(&queue));
1119   PetscFunctionReturn(0);
1120 }
1121 
1122 /*
1123   We are adding three kinds of points here:
1124     Replicated:     Copies of points which exist in the mesh, such as vertices identified across a fault
1125     Non-replicated: Points which exist on the fault, but are not replicated
1126     Ghost:          These are shared fault faces which are not owned by this process. These do not produce hybrid cells and do not replicate
1127     Hybrid:         Entirely new points, such as cohesive cells
1128 
1129   When creating subsequent cohesive cells, we shift the old hybrid cells to the end of the numbering at
1130   each depth so that the new split/hybrid points can be inserted as a block.
1131 */
1132 static PetscErrorCode DMPlexConstructCohesiveCells_Internal(DM dm, DMLabel label, DMLabel splitLabel, DM sdm)
1133 {
1134   MPI_Comm         comm;
1135   IS               valueIS;
1136   PetscInt         numSP = 0;          /* The number of depths for which we have replicated points */
1137   const PetscInt  *values;             /* List of depths for which we have replicated points */
1138   IS              *splitIS;
1139   IS              *unsplitIS;
1140   IS               ghostIS;
1141   PetscInt        *numSplitPoints;     /* The number of replicated points at each depth */
1142   PetscInt        *numUnsplitPoints;   /* The number of non-replicated points at each depth which still give rise to hybrid points */
1143   PetscInt        *numHybridPoints;    /* The number of new hybrid points at each depth */
1144   PetscInt        *numHybridPointsOld; /* The number of existing hybrid points at each depth */
1145   PetscInt         numGhostPoints;     /* The number of unowned, shared fault faces */
1146   const PetscInt **splitPoints;        /* Replicated points for each depth */
1147   const PetscInt **unsplitPoints;      /* Non-replicated points for each depth */
1148   const PetscInt  *ghostPoints;        /* Ghost fault faces */
1149   PetscSection     coordSection;
1150   Vec              coordinates;
1151   PetscScalar     *coords;
1152   PetscInt        *depthMax;           /* The first hybrid point at each depth in the original mesh */
1153   PetscInt        *depthEnd;           /* The point limit at each depth in the original mesh */
1154   PetscInt        *depthShift;         /* Number of replicated+hybrid points at each depth */
1155   PetscInt        *pMaxNew;            /* The first replicated point at each depth in the new mesh, hybrids come after this */
1156   PetscInt        *coneNew, *coneONew, *supportNew;
1157   PetscInt         shift = 100, shift2 = 200, depth = 0, dep, dim, d, sp, maxConeSize, maxSupportSize, maxConeSizeNew, maxSupportSizeNew, numLabels, vStart, vEnd, pEnd, p, v;
1158 
1159   PetscFunctionBegin;
1160   PetscCall(PetscObjectGetComm((PetscObject)dm,&comm));
1161   PetscCall(DMGetDimension(dm, &dim));
1162   PetscCall(DMPlexGetDepth(dm, &depth));
1163   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
1164   /* We do not want this label automatically computed, instead we compute it here */
1165   PetscCall(DMCreateLabel(sdm, "celltype"));
1166   /* Count split points and add cohesive cells */
1167   PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize));
1168   PetscCall(PetscMalloc5(depth+1,&depthMax,depth+1,&depthEnd,2*(depth+1),&depthShift,depth+1,&pMaxNew,depth+1,&numHybridPointsOld));
1169   PetscCall(PetscMalloc7(depth+1,&splitIS,depth+1,&unsplitIS,depth+1,&numSplitPoints,depth+1,&numUnsplitPoints,depth+1,&numHybridPoints,depth+1,&splitPoints,depth+1,&unsplitPoints));
1170   for (d = 0; d <= depth; ++d) {
1171     PetscCall(DMPlexGetDepthStratum(dm, d, NULL, &pMaxNew[d]));
1172     PetscCall(DMPlexGetTensorPrismBounds_Internal(dm, d, &depthMax[d], NULL));
1173     depthEnd[d]           = pMaxNew[d];
1174     depthMax[d]           = depthMax[d] < 0 ? depthEnd[d] : depthMax[d];
1175     numSplitPoints[d]     = 0;
1176     numUnsplitPoints[d]   = 0;
1177     numHybridPoints[d]    = 0;
1178     numHybridPointsOld[d] = depthMax[d] < 0 ? 0 : depthEnd[d] - depthMax[d];
1179     splitPoints[d]        = NULL;
1180     unsplitPoints[d]      = NULL;
1181     splitIS[d]            = NULL;
1182     unsplitIS[d]          = NULL;
1183     /* we are shifting the existing hybrid points with the stratum behind them, so
1184      * the split comes at the end of the normal points, i.e., at depthMax[d] */
1185     depthShift[2*d]       = depthMax[d];
1186     depthShift[2*d+1]     = 0;
1187   }
1188   numGhostPoints = 0;
1189   ghostPoints    = NULL;
1190   if (label) {
1191     PetscCall(DMLabelGetValueIS(label, &valueIS));
1192     PetscCall(ISGetLocalSize(valueIS, &numSP));
1193     PetscCall(ISGetIndices(valueIS, &values));
1194   }
1195   for (sp = 0; sp < numSP; ++sp) {
1196     const PetscInt dep = values[sp];
1197 
1198     if ((dep < 0) || (dep > depth)) continue;
1199     PetscCall(DMLabelGetStratumIS(label, dep, &splitIS[dep]));
1200     if (splitIS[dep]) {
1201       PetscCall(ISGetLocalSize(splitIS[dep], &numSplitPoints[dep]));
1202       PetscCall(ISGetIndices(splitIS[dep], &splitPoints[dep]));
1203     }
1204     PetscCall(DMLabelGetStratumIS(label, shift2+dep, &unsplitIS[dep]));
1205     if (unsplitIS[dep]) {
1206       PetscCall(ISGetLocalSize(unsplitIS[dep], &numUnsplitPoints[dep]));
1207       PetscCall(ISGetIndices(unsplitIS[dep], &unsplitPoints[dep]));
1208     }
1209   }
1210   PetscCall(DMLabelGetStratumIS(label, shift2+dim-1, &ghostIS));
1211   if (ghostIS) {
1212     PetscCall(ISGetLocalSize(ghostIS, &numGhostPoints));
1213     PetscCall(ISGetIndices(ghostIS, &ghostPoints));
1214   }
1215   /* Calculate number of hybrid points */
1216   for (d = 1; d <= depth; ++d) numHybridPoints[d]     = numSplitPoints[d-1] + numUnsplitPoints[d-1]; /* There is a hybrid cell/face/edge for every split face/edge/vertex   */
1217   for (d = 0; d <= depth; ++d) depthShift[2*d+1]      = numSplitPoints[d] + numHybridPoints[d];
1218   PetscCall(DMPlexShiftPointSetUp_Internal(depth,depthShift));
1219   /* the end of the points in this stratum that come before the new points:
1220    * shifting pMaxNew[d] gets the new start of the next stratum, then count back the old hybrid points and the newly
1221    * added points */
1222   for (d = 0; d <= depth; ++d) pMaxNew[d]             = DMPlexShiftPoint_Internal(pMaxNew[d],depth,depthShift) - (numHybridPointsOld[d] + numSplitPoints[d] + numHybridPoints[d]);
1223   PetscCall(DMPlexShiftSizes_Internal(dm, depthShift, sdm));
1224   /* Step 3: Set cone/support sizes for new points */
1225   for (dep = 0; dep <= depth; ++dep) {
1226     for (p = 0; p < numSplitPoints[dep]; ++p) {
1227       const PetscInt  oldp   = splitPoints[dep][p];
1228       const PetscInt  newp   = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*oldp + depthOffset[dep]*/;
1229       const PetscInt  splitp = p    + pMaxNew[dep];
1230       const PetscInt *support;
1231       DMPolytopeType  ct;
1232       PetscInt        coneSize, supportSize, qf, qn, qp, e;
1233 
1234       PetscCall(DMPlexGetConeSize(dm, oldp, &coneSize));
1235       PetscCall(DMPlexSetConeSize(sdm, splitp, coneSize));
1236       PetscCall(DMPlexGetSupportSize(dm, oldp, &supportSize));
1237       PetscCall(DMPlexSetSupportSize(sdm, splitp, supportSize));
1238       PetscCall(DMPlexGetCellType(dm, oldp, &ct));
1239       PetscCall(DMPlexSetCellType(sdm, splitp, ct));
1240       if (dep == depth-1) {
1241         const PetscInt hybcell = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1242 
1243         /* Add cohesive cells, they are prisms */
1244         PetscCall(DMPlexSetConeSize(sdm, hybcell, 2 + coneSize));
1245         switch (coneSize) {
1246           case 2: PetscCall(DMPlexSetCellType(sdm, hybcell, DM_POLYTOPE_SEG_PRISM_TENSOR));break;
1247           case 3: PetscCall(DMPlexSetCellType(sdm, hybcell, DM_POLYTOPE_TRI_PRISM_TENSOR));break;
1248           case 4: PetscCall(DMPlexSetCellType(sdm, hybcell, DM_POLYTOPE_QUAD_PRISM_TENSOR));break;
1249         }
1250         /* Shared fault faces with only one support cell now have two with the cohesive cell */
1251         /*   TODO Check thaat oldp has rootdegree == 1 */
1252         if (supportSize == 1) {
1253           const PetscInt *support;
1254           PetscInt        val;
1255 
1256           PetscCall(DMPlexGetSupport(dm, oldp, &support));
1257           PetscCall(DMLabelGetValue(label, support[0], &val));
1258           if (val < 0) PetscCall(DMPlexSetSupportSize(sdm, splitp, 2));
1259           else         PetscCall(DMPlexSetSupportSize(sdm, newp,   2));
1260         }
1261       } else if (dep == 0) {
1262         const PetscInt hybedge = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1263 
1264         PetscCall(DMPlexGetSupport(dm, oldp, &support));
1265         for (e = 0, qn = 0, qp = 0, qf = 0; e < supportSize; ++e) {
1266           PetscInt val;
1267 
1268           PetscCall(DMLabelGetValue(label, support[e], &val));
1269           if (val == 1) ++qf;
1270           if ((val == 1) || (val ==  (shift + 1))) ++qn;
1271           if ((val == 1) || (val == -(shift + 1))) ++qp;
1272         }
1273         /* Split old vertex: Edges into original vertex and new cohesive edge */
1274         PetscCall(DMPlexSetSupportSize(sdm, newp, qn+1));
1275         /* Split new vertex: Edges into split vertex and new cohesive edge */
1276         PetscCall(DMPlexSetSupportSize(sdm, splitp, qp+1));
1277         /* Add hybrid edge */
1278         PetscCall(DMPlexSetConeSize(sdm, hybedge, 2));
1279         PetscCall(DMPlexSetSupportSize(sdm, hybedge, qf));
1280         PetscCall(DMPlexSetCellType(sdm, hybedge, DM_POLYTOPE_POINT_PRISM_TENSOR));
1281       } else if (dep == dim-2) {
1282         const PetscInt hybface = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1283 
1284         PetscCall(DMPlexGetSupport(dm, oldp, &support));
1285         for (e = 0, qn = 0, qp = 0, qf = 0; e < supportSize; ++e) {
1286           PetscInt val;
1287 
1288           PetscCall(DMLabelGetValue(label, support[e], &val));
1289           if (val == dim-1) ++qf;
1290           if ((val == dim-1) || (val ==  (shift + dim-1))) ++qn;
1291           if ((val == dim-1) || (val == -(shift + dim-1))) ++qp;
1292         }
1293         /* Split old edge: Faces into original edge and cohesive face (positive side?) */
1294         PetscCall(DMPlexSetSupportSize(sdm, newp, qn+1));
1295         /* Split new edge: Faces into split edge and cohesive face (negative side?) */
1296         PetscCall(DMPlexSetSupportSize(sdm, splitp, qp+1));
1297         /* Add hybrid face */
1298         PetscCall(DMPlexSetConeSize(sdm, hybface, 4));
1299         PetscCall(DMPlexSetSupportSize(sdm, hybface, qf));
1300         PetscCall(DMPlexSetCellType(sdm, hybface, DM_POLYTOPE_SEG_PRISM_TENSOR));
1301       }
1302     }
1303   }
1304   for (dep = 0; dep <= depth; ++dep) {
1305     for (p = 0; p < numUnsplitPoints[dep]; ++p) {
1306       const PetscInt  oldp   = unsplitPoints[dep][p];
1307       const PetscInt  newp   = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*oldp + depthOffset[dep]*/;
1308       const PetscInt *support;
1309       PetscInt        coneSize, supportSize, qf, e, s;
1310 
1311       PetscCall(DMPlexGetConeSize(dm, oldp, &coneSize));
1312       PetscCall(DMPlexGetSupportSize(dm, oldp, &supportSize));
1313       PetscCall(DMPlexGetSupport(dm, oldp, &support));
1314       if (dep == 0) {
1315         const PetscInt hybedge = p + pMaxNew[dep+1] + numSplitPoints[dep+1] + numSplitPoints[dep];
1316 
1317         /* Unsplit vertex: Edges into original vertex, split edges, and new cohesive edge twice */
1318         for (s = 0, qf = 0; s < supportSize; ++s, ++qf) {
1319           PetscCall(PetscFindInt(support[s], numSplitPoints[dep+1], splitPoints[dep+1], &e));
1320           if (e >= 0) ++qf;
1321         }
1322         PetscCall(DMPlexSetSupportSize(sdm, newp, qf+2));
1323         /* Add hybrid edge */
1324         PetscCall(DMPlexSetConeSize(sdm, hybedge, 2));
1325         for (e = 0, qf = 0; e < supportSize; ++e) {
1326           PetscInt val;
1327 
1328           PetscCall(DMLabelGetValue(label, support[e], &val));
1329           /* Split and unsplit edges produce hybrid faces */
1330           if (val == 1) ++qf;
1331           if (val == (shift2 + 1)) ++qf;
1332         }
1333         PetscCall(DMPlexSetSupportSize(sdm, hybedge, qf));
1334         PetscCall(DMPlexSetCellType(sdm, hybedge, DM_POLYTOPE_POINT_PRISM_TENSOR));
1335       } else if (dep == dim-2) {
1336         const PetscInt hybface = p + pMaxNew[dep+1] + numSplitPoints[dep+1] + numSplitPoints[dep];
1337         PetscInt       val;
1338 
1339         for (e = 0, qf = 0; e < supportSize; ++e) {
1340           PetscCall(DMLabelGetValue(label, support[e], &val));
1341           if (val == dim-1) qf += 2;
1342           else              ++qf;
1343         }
1344         /* Unsplit edge: Faces into original edge, split face, and cohesive face twice */
1345         PetscCall(DMPlexSetSupportSize(sdm, newp, qf+2));
1346         /* Add hybrid face */
1347         for (e = 0, qf = 0; e < supportSize; ++e) {
1348           PetscCall(DMLabelGetValue(label, support[e], &val));
1349           if (val == dim-1) ++qf;
1350         }
1351         PetscCall(DMPlexSetConeSize(sdm, hybface, 4));
1352         PetscCall(DMPlexSetSupportSize(sdm, hybface, qf));
1353         PetscCall(DMPlexSetCellType(sdm, hybface, DM_POLYTOPE_SEG_PRISM_TENSOR));
1354       }
1355     }
1356   }
1357   /* Step 4: Setup split DM */
1358   PetscCall(DMSetUp(sdm));
1359   PetscCall(DMPlexShiftPoints_Internal(dm, depthShift, sdm));
1360   PetscCall(DMPlexGetMaxSizes(sdm, &maxConeSizeNew, &maxSupportSizeNew));
1361   PetscCall(PetscMalloc3(PetscMax(maxConeSize, maxConeSizeNew)*3,&coneNew,PetscMax(maxConeSize, maxConeSizeNew)*3,&coneONew,PetscMax(maxSupportSize, maxSupportSizeNew),&supportNew));
1362   /* Step 6: Set cones and supports for new points */
1363   for (dep = 0; dep <= depth; ++dep) {
1364     for (p = 0; p < numSplitPoints[dep]; ++p) {
1365       const PetscInt  oldp   = splitPoints[dep][p];
1366       const PetscInt  newp   = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*oldp + depthOffset[dep]*/;
1367       const PetscInt  splitp = p    + pMaxNew[dep];
1368       const PetscInt *cone, *support, *ornt;
1369       DMPolytopeType  ct;
1370       PetscInt        coneSize, supportSize, q, qf, qn, qp, v, e, s;
1371 
1372       PetscCall(DMPlexGetCellType(dm, oldp, &ct));
1373       PetscCall(DMPlexGetConeSize(dm, oldp, &coneSize));
1374       PetscCall(DMPlexGetCone(dm, oldp, &cone));
1375       PetscCall(DMPlexGetConeOrientation(dm, oldp, &ornt));
1376       PetscCall(DMPlexGetSupportSize(dm, oldp, &supportSize));
1377       PetscCall(DMPlexGetSupport(dm, oldp, &support));
1378       if (dep == depth-1) {
1379         PetscBool       hasUnsplit = PETSC_FALSE;
1380         const PetscInt  hybcell    = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1381         const PetscInt *supportF;
1382 
1383         coneONew[0] = coneONew[1] = -1000;
1384         /* Split face:       copy in old face to new face to start */
1385         PetscCall(DMPlexGetSupport(sdm, newp,  &supportF));
1386         PetscCall(DMPlexSetSupport(sdm, splitp, supportF));
1387         /* Split old face:   old vertices/edges in cone so no change */
1388         /* Split new face:   new vertices/edges in cone */
1389         for (q = 0; q < coneSize; ++q) {
1390           PetscCall(PetscFindInt(cone[q], numSplitPoints[dep-1], splitPoints[dep-1], &v));
1391           if (v < 0) {
1392             PetscCall(PetscFindInt(cone[q], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &v));
1393             PetscCheck(v >= 0,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %" PetscInt_FMT " in split or unsplit points of depth %" PetscInt_FMT, cone[q], dep-1);
1394             coneNew[2+q] = DMPlexShiftPoint_Internal(cone[q], depth, depthShift) /*cone[q] + depthOffset[dep-1]*/;
1395             hasUnsplit   = PETSC_TRUE;
1396           } else {
1397             coneNew[2+q] = v + pMaxNew[dep-1];
1398             if (dep > 1) {
1399               const PetscInt *econe;
1400               PetscInt        econeSize, r, vs, vu;
1401 
1402               PetscCall(DMPlexGetConeSize(dm, cone[q], &econeSize));
1403               PetscCall(DMPlexGetCone(dm, cone[q], &econe));
1404               for (r = 0; r < econeSize; ++r) {
1405                 PetscCall(PetscFindInt(econe[r], numSplitPoints[dep-2],   splitPoints[dep-2],   &vs));
1406                 PetscCall(PetscFindInt(econe[r], numUnsplitPoints[dep-2], unsplitPoints[dep-2], &vu));
1407                 if (vs >= 0) continue;
1408                 PetscCheck(vu >= 0,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %" PetscInt_FMT " in split or unsplit points of depth %" PetscInt_FMT, econe[r], dep-2);
1409                 hasUnsplit   = PETSC_TRUE;
1410               }
1411             }
1412           }
1413         }
1414         PetscCall(DMPlexSetCone(sdm, splitp, &coneNew[2]));
1415         PetscCall(DMPlexSetConeOrientation(sdm, splitp, ornt));
1416         /* Face support */
1417         PetscInt vals[2];
1418 
1419         PetscCall(DMLabelGetValue(label, support[0], &vals[0]));
1420         if (supportSize > 1) PetscCall(DMLabelGetValue(label, support[1], &vals[1]));
1421         else                 vals[1] = -vals[0];
1422         PetscCheck(vals[0]*vals[1] < 0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid support labels %" PetscInt_FMT " %" PetscInt_FMT, vals[0], vals[1]);
1423 
1424         for (s = 0; s < 2; ++s) {
1425           if (s >= supportSize) {
1426             if (vals[s] < 0) {
1427               /* Ghost old face:   Replace negative side cell with cohesive cell */
1428               PetscCall(DMPlexInsertSupport(sdm, newp,   s, hybcell));
1429             } else {
1430               /* Ghost new face:   Replace positive side cell with cohesive cell */
1431               PetscCall(DMPlexInsertSupport(sdm, splitp, s, hybcell));
1432             }
1433           } else {
1434             if (vals[s] < 0) {
1435               /* Split old face:   Replace negative side cell with cohesive cell */
1436               PetscCall(DMPlexInsertSupport(sdm, newp,   s, hybcell));
1437             } else {
1438               /* Split new face:   Replace positive side cell with cohesive cell */
1439               PetscCall(DMPlexInsertSupport(sdm, splitp, s, hybcell));
1440             }
1441           }
1442         }
1443         /* Get orientation for cohesive face using the positive side cell */
1444         {
1445           const PetscInt *ncone, *nconeO;
1446           PetscInt        nconeSize, nc, ocell;
1447           PetscBool       flip = PETSC_FALSE;
1448 
1449           if (supportSize > 1) {ocell = vals[0] < 0 ? support[1] : support[0];}
1450           else                 {ocell = support[0]; flip = vals[0] < 0 ? PETSC_TRUE : PETSC_FALSE;}
1451           PetscCall(DMPlexGetConeSize(dm, ocell, &nconeSize));
1452           PetscCall(DMPlexGetCone(dm, ocell, &ncone));
1453           PetscCall(DMPlexGetConeOrientation(dm, ocell, &nconeO));
1454           for (nc = 0; nc < nconeSize; ++nc) {
1455             if (ncone[nc] == oldp) {
1456               coneONew[0] = flip ? -(nconeO[nc]+1) : nconeO[nc];
1457               break;
1458             }
1459           }
1460           PetscCheck(nc < nconeSize, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate face %" PetscInt_FMT " in neighboring cell %" PetscInt_FMT, oldp, ocell);
1461         }
1462         /* Cohesive cell:    Old and new split face, then new cohesive faces */
1463         {
1464           const PetscInt No = DMPolytopeTypeGetNumArrangments(ct)/2;
1465           PetscCheck((coneONew[0] >= -No) && (coneONew[0] < No), PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid %s orientation %" PetscInt_FMT, DMPolytopeTypes[ct], coneONew[0]);
1466         }
1467         const PetscInt *arr = DMPolytopeTypeGetArrangment(ct, coneONew[0]);
1468 
1469         coneNew[0]  = newp;   /* Extracted negative side orientation above */
1470         coneNew[1]  = splitp;
1471         coneONew[1] = coneONew[0];
1472         for (q = 0; q < coneSize; ++q) {
1473           /* Hybrid faces must follow order from oriented end face */
1474           const PetscInt qa = arr[q*2+0];
1475           const PetscInt qo = arr[q*2+1];
1476           DMPolytopeType ft = dep == 2 ? DM_POLYTOPE_SEGMENT : DM_POLYTOPE_POINT;
1477 
1478           PetscCall(PetscFindInt(cone[qa], numSplitPoints[dep-1], splitPoints[dep-1], &v));
1479           if (v < 0) {
1480             PetscCall(PetscFindInt(cone[qa], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &v));
1481             coneNew[2+q]  = v + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep-1];
1482           } else {
1483             coneNew[2+q]  = v + pMaxNew[dep] + numSplitPoints[dep];
1484           }
1485           coneONew[2+q] = DMPolytopeTypeComposeOrientation(ft, qo, ornt[qa]);
1486         }
1487         PetscCall(DMPlexSetCone(sdm, hybcell, coneNew));
1488         PetscCall(DMPlexSetConeOrientation(sdm, hybcell, coneONew));
1489         /* Label the hybrid cells on the boundary of the split */
1490         if (hasUnsplit) PetscCall(DMLabelSetValue(label, -hybcell, dim));
1491       } else if (dep == 0) {
1492         const PetscInt hybedge = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1493 
1494         /* Split old vertex: Edges in old split faces and new cohesive edge */
1495         for (e = 0, qn = 0; e < supportSize; ++e) {
1496           PetscInt val;
1497 
1498           PetscCall(DMLabelGetValue(label, support[e], &val));
1499           if ((val == 1) || (val == (shift + 1))) {
1500             supportNew[qn++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1501           }
1502         }
1503         supportNew[qn] = hybedge;
1504         PetscCall(DMPlexSetSupport(sdm, newp, supportNew));
1505         /* Split new vertex: Edges in new split faces and new cohesive edge */
1506         for (e = 0, qp = 0; e < supportSize; ++e) {
1507           PetscInt val, edge;
1508 
1509           PetscCall(DMLabelGetValue(label, support[e], &val));
1510           if (val == 1) {
1511             PetscCall(PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &edge));
1512             PetscCheck(edge >= 0,comm, PETSC_ERR_ARG_WRONG, "Edge %" PetscInt_FMT " is not a split edge", support[e]);
1513             supportNew[qp++] = edge + pMaxNew[dep+1];
1514           } else if (val == -(shift + 1)) {
1515             supportNew[qp++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1516           }
1517         }
1518         supportNew[qp] = hybedge;
1519         PetscCall(DMPlexSetSupport(sdm, splitp, supportNew));
1520         /* Hybrid edge:    Old and new split vertex */
1521         coneNew[0] = newp;
1522         coneNew[1] = splitp;
1523         PetscCall(DMPlexSetCone(sdm, hybedge, coneNew));
1524         for (e = 0, qf = 0; e < supportSize; ++e) {
1525           PetscInt val, edge;
1526 
1527           PetscCall(DMLabelGetValue(label, support[e], &val));
1528           if (val == 1) {
1529             PetscCall(PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &edge));
1530             PetscCheck(edge >= 0,comm, PETSC_ERR_ARG_WRONG, "Edge %" PetscInt_FMT " is not a split edge", support[e]);
1531             supportNew[qf++] = edge + pMaxNew[dep+2] + numSplitPoints[dep+2];
1532           }
1533         }
1534         PetscCall(DMPlexSetSupport(sdm, hybedge, supportNew));
1535       } else if (dep == dim-2) {
1536         const PetscInt hybface = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1537 
1538         /* Split old edge:   old vertices in cone so no change */
1539         /* Split new edge:   new vertices in cone */
1540         for (q = 0; q < coneSize; ++q) {
1541           PetscCall(PetscFindInt(cone[q], numSplitPoints[dep-1], splitPoints[dep-1], &v));
1542           if (v < 0) {
1543             PetscCall(PetscFindInt(cone[q], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &v));
1544             PetscCheck(v >= 0,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %" PetscInt_FMT " in split or unsplit points of depth %" PetscInt_FMT, cone[q], dep-1);
1545             coneNew[q] = DMPlexShiftPoint_Internal(cone[q], depth, depthShift) /*cone[q] + depthOffset[dep-1]*/;
1546           } else {
1547             coneNew[q] = v + pMaxNew[dep-1];
1548           }
1549         }
1550         PetscCall(DMPlexSetCone(sdm, splitp, coneNew));
1551         /* Split old edge: Faces in positive side cells and old split faces */
1552         for (e = 0, q = 0; e < supportSize; ++e) {
1553           PetscInt val;
1554 
1555           PetscCall(DMLabelGetValue(label, support[e], &val));
1556           if (val == dim-1) {
1557             supportNew[q++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1558           } else if (val == (shift + dim-1)) {
1559             supportNew[q++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1560           }
1561         }
1562         supportNew[q++] = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1563         PetscCall(DMPlexSetSupport(sdm, newp, supportNew));
1564         /* Split new edge: Faces in negative side cells and new split faces */
1565         for (e = 0, q = 0; e < supportSize; ++e) {
1566           PetscInt val, face;
1567 
1568           PetscCall(DMLabelGetValue(label, support[e], &val));
1569           if (val == dim-1) {
1570             PetscCall(PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &face));
1571             PetscCheck(face >= 0,comm, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " is not a split face", support[e]);
1572             supportNew[q++] = face + pMaxNew[dep+1];
1573           } else if (val == -(shift + dim-1)) {
1574             supportNew[q++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1575           }
1576         }
1577         supportNew[q++] = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1578         PetscCall(DMPlexSetSupport(sdm, splitp, supportNew));
1579         /* Hybrid face */
1580         coneNew[0] = newp;
1581         coneNew[1] = splitp;
1582         for (v = 0; v < coneSize; ++v) {
1583           PetscInt vertex;
1584           PetscCall(PetscFindInt(cone[v], numSplitPoints[dep-1], splitPoints[dep-1], &vertex));
1585           if (vertex < 0) {
1586             PetscCall(PetscFindInt(cone[v], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &vertex));
1587             PetscCheck(vertex >= 0,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %" PetscInt_FMT " in split or unsplit points of depth %" PetscInt_FMT, cone[v], dep-1);
1588             coneNew[2+v] = vertex + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep-1];
1589           } else {
1590             coneNew[2+v] = vertex + pMaxNew[dep] + numSplitPoints[dep];
1591           }
1592         }
1593         PetscCall(DMPlexSetCone(sdm, hybface, coneNew));
1594         for (e = 0, qf = 0; e < supportSize; ++e) {
1595           PetscInt val, face;
1596 
1597           PetscCall(DMLabelGetValue(label, support[e], &val));
1598           if (val == dim-1) {
1599             PetscCall(PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &face));
1600             PetscCheck(face >= 0,comm, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " is not a split face", support[e]);
1601             supportNew[qf++] = face + pMaxNew[dep+2] + numSplitPoints[dep+2];
1602           }
1603         }
1604         PetscCall(DMPlexSetSupport(sdm, hybface, supportNew));
1605       }
1606     }
1607   }
1608   for (dep = 0; dep <= depth; ++dep) {
1609     for (p = 0; p < numUnsplitPoints[dep]; ++p) {
1610       const PetscInt  oldp   = unsplitPoints[dep][p];
1611       const PetscInt  newp   = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*oldp + depthOffset[dep]*/;
1612       const PetscInt *cone, *support;
1613       PetscInt        coneSize, supportSize, supportSizeNew, q, qf, e, f, s;
1614 
1615       PetscCall(DMPlexGetConeSize(dm, oldp, &coneSize));
1616       PetscCall(DMPlexGetCone(dm, oldp, &cone));
1617       PetscCall(DMPlexGetSupportSize(dm, oldp, &supportSize));
1618       PetscCall(DMPlexGetSupport(dm, oldp, &support));
1619       if (dep == 0) {
1620         const PetscInt hybedge = p + pMaxNew[dep+1] + numSplitPoints[dep+1] + numSplitPoints[dep];
1621 
1622         /* Unsplit vertex */
1623         PetscCall(DMPlexGetSupportSize(sdm, newp, &supportSizeNew));
1624         for (s = 0, q = 0; s < supportSize; ++s) {
1625           supportNew[q++] = DMPlexShiftPoint_Internal(support[s], depth, depthShift) /*support[s] + depthOffset[dep+1]*/;
1626           PetscCall(PetscFindInt(support[s], numSplitPoints[dep+1], splitPoints[dep+1], &e));
1627           if (e >= 0) {
1628             supportNew[q++] = e + pMaxNew[dep+1];
1629           }
1630         }
1631         supportNew[q++] = hybedge;
1632         supportNew[q++] = hybedge;
1633         PetscCheck(q == supportSizeNew,comm, PETSC_ERR_ARG_WRONG, "Support size %" PetscInt_FMT " != %" PetscInt_FMT " for vertex %" PetscInt_FMT, q, supportSizeNew, newp);
1634         PetscCall(DMPlexSetSupport(sdm, newp, supportNew));
1635         /* Hybrid edge */
1636         coneNew[0] = newp;
1637         coneNew[1] = newp;
1638         PetscCall(DMPlexSetCone(sdm, hybedge, coneNew));
1639         for (e = 0, qf = 0; e < supportSize; ++e) {
1640           PetscInt val, edge;
1641 
1642           PetscCall(DMLabelGetValue(label, support[e], &val));
1643           if (val == 1) {
1644             PetscCall(PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &edge));
1645             PetscCheck(edge >= 0,comm, PETSC_ERR_ARG_WRONG, "Edge %" PetscInt_FMT " is not a split edge", support[e]);
1646             supportNew[qf++] = edge + pMaxNew[dep+2] + numSplitPoints[dep+2];
1647           } else if  (val ==  (shift2 + 1)) {
1648             PetscCall(PetscFindInt(support[e], numUnsplitPoints[dep+1], unsplitPoints[dep+1], &edge));
1649             PetscCheck(edge >= 0,comm, PETSC_ERR_ARG_WRONG, "Edge %" PetscInt_FMT " is not a unsplit edge", support[e]);
1650             supportNew[qf++] = edge + pMaxNew[dep+2] + numSplitPoints[dep+2] + numSplitPoints[dep+1];
1651           }
1652         }
1653         PetscCall(DMPlexSetSupport(sdm, hybedge, supportNew));
1654       } else if (dep == dim-2) {
1655         const PetscInt hybface = p + pMaxNew[dep+1] + numSplitPoints[dep+1] + numSplitPoints[dep];
1656 
1657         /* Unsplit edge: Faces into original edge, split face, and hybrid face twice */
1658         for (f = 0, qf = 0; f < supportSize; ++f) {
1659           PetscInt val, face;
1660 
1661           PetscCall(DMLabelGetValue(label, support[f], &val));
1662           if (val == dim-1) {
1663             PetscCall(PetscFindInt(support[f], numSplitPoints[dep+1], splitPoints[dep+1], &face));
1664             PetscCheck(face >= 0,comm, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " is not a split face", support[f]);
1665             supportNew[qf++] = DMPlexShiftPoint_Internal(support[f], depth, depthShift) /*support[f] + depthOffset[dep+1]*/;
1666             supportNew[qf++] = face + pMaxNew[dep+1];
1667           } else {
1668             supportNew[qf++] = DMPlexShiftPoint_Internal(support[f], depth, depthShift) /*support[f] + depthOffset[dep+1]*/;
1669           }
1670         }
1671         supportNew[qf++] = hybface;
1672         supportNew[qf++] = hybface;
1673         PetscCall(DMPlexGetSupportSize(sdm, newp, &supportSizeNew));
1674         PetscCheck(qf == supportSizeNew,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Support size for unsplit edge %" PetscInt_FMT " is %" PetscInt_FMT " != %" PetscInt_FMT, newp, qf, supportSizeNew);
1675         PetscCall(DMPlexSetSupport(sdm, newp, supportNew));
1676         /* Add hybrid face */
1677         coneNew[0] = newp;
1678         coneNew[1] = newp;
1679         PetscCall(PetscFindInt(cone[0], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &v));
1680         PetscCheck(v >= 0,comm, PETSC_ERR_ARG_WRONG, "Vertex %" PetscInt_FMT " is not an unsplit vertex", cone[0]);
1681         coneNew[2] = v + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep-1];
1682         PetscCall(PetscFindInt(cone[1], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &v));
1683         PetscCheck(v >= 0,comm, PETSC_ERR_ARG_WRONG, "Vertex %" PetscInt_FMT " is not an unsplit vertex", cone[1]);
1684         coneNew[3] = v + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep-1];
1685         PetscCall(DMPlexSetCone(sdm, hybface, coneNew));
1686         for (f = 0, qf = 0; f < supportSize; ++f) {
1687           PetscInt val, face;
1688 
1689           PetscCall(DMLabelGetValue(label, support[f], &val));
1690           if (val == dim-1) {
1691             PetscCall(PetscFindInt(support[f], numSplitPoints[dep+1], splitPoints[dep+1], &face));
1692             supportNew[qf++] = face + pMaxNew[dep+2] + numSplitPoints[dep+2];
1693           }
1694         }
1695         PetscCall(DMPlexGetSupportSize(sdm, hybface, &supportSizeNew));
1696         PetscCheck(qf == supportSizeNew,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Support size for hybrid face %" PetscInt_FMT " is %" PetscInt_FMT " != %" PetscInt_FMT, hybface, qf, supportSizeNew);
1697         PetscCall(DMPlexSetSupport(sdm, hybface, supportNew));
1698       }
1699     }
1700   }
1701   /* Step 6b: Replace split points in negative side cones */
1702   for (sp = 0; sp < numSP; ++sp) {
1703     PetscInt        dep = values[sp];
1704     IS              pIS;
1705     PetscInt        numPoints;
1706     const PetscInt *points;
1707 
1708     if (dep >= 0) continue;
1709     PetscCall(DMLabelGetStratumIS(label, dep, &pIS));
1710     if (!pIS) continue;
1711     dep  = -dep - shift;
1712     PetscCall(ISGetLocalSize(pIS, &numPoints));
1713     PetscCall(ISGetIndices(pIS, &points));
1714     for (p = 0; p < numPoints; ++p) {
1715       const PetscInt  oldp = points[p];
1716       const PetscInt  newp = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*depthOffset[dep] + oldp*/;
1717       const PetscInt *cone;
1718       PetscInt        coneSize, c;
1719       /* PetscBool       replaced = PETSC_FALSE; */
1720 
1721       /* Negative edge: replace split vertex */
1722       /* Negative cell: replace split face */
1723       PetscCall(DMPlexGetConeSize(sdm, newp, &coneSize));
1724       PetscCall(DMPlexGetCone(sdm, newp, &cone));
1725       for (c = 0; c < coneSize; ++c) {
1726         const PetscInt coldp = DMPlexShiftPointInverse_Internal(cone[c],depth,depthShift);
1727         PetscInt       csplitp, cp, val;
1728 
1729         PetscCall(DMLabelGetValue(label, coldp, &val));
1730         if (val == dep-1) {
1731           PetscCall(PetscFindInt(coldp, numSplitPoints[dep-1], splitPoints[dep-1], &cp));
1732           PetscCheck(cp >= 0,comm, PETSC_ERR_ARG_WRONG, "Point %" PetscInt_FMT " is not a split point of dimension %" PetscInt_FMT, oldp, dep-1);
1733           csplitp  = pMaxNew[dep-1] + cp;
1734           PetscCall(DMPlexInsertCone(sdm, newp, c, csplitp));
1735           /* replaced = PETSC_TRUE; */
1736         }
1737       }
1738       /* Cells with only a vertex or edge on the submesh have no replacement */
1739       /* PetscCheck(replaced,comm, PETSC_ERR_ARG_WRONG, "The cone of point %d does not contain split points", oldp); */
1740     }
1741     PetscCall(ISRestoreIndices(pIS, &points));
1742     PetscCall(ISDestroy(&pIS));
1743   }
1744   /* Step 7: Coordinates */
1745   PetscCall(DMPlexShiftCoordinates_Internal(dm, depthShift, sdm));
1746   PetscCall(DMGetCoordinateSection(sdm, &coordSection));
1747   PetscCall(DMGetCoordinatesLocal(sdm, &coordinates));
1748   PetscCall(VecGetArray(coordinates, &coords));
1749   for (v = 0; v < (numSplitPoints ? numSplitPoints[0] : 0); ++v) {
1750     const PetscInt newp   = DMPlexShiftPoint_Internal(splitPoints[0][v], depth, depthShift) /*depthOffset[0] + splitPoints[0][v]*/;
1751     const PetscInt splitp = pMaxNew[0] + v;
1752     PetscInt       dof, off, soff, d;
1753 
1754     PetscCall(PetscSectionGetDof(coordSection, newp, &dof));
1755     PetscCall(PetscSectionGetOffset(coordSection, newp, &off));
1756     PetscCall(PetscSectionGetOffset(coordSection, splitp, &soff));
1757     for (d = 0; d < dof; ++d) coords[soff+d] = coords[off+d];
1758   }
1759   PetscCall(VecRestoreArray(coordinates, &coords));
1760   /* Step 8: SF, if I can figure this out we can split the mesh in parallel */
1761   PetscCall(DMPlexShiftSF_Internal(dm, depthShift, sdm));
1762   /*   TODO We need to associate the ghost points with the correct replica */
1763   /* Step 9: Labels */
1764   PetscCall(DMPlexShiftLabels_Internal(dm, depthShift, sdm));
1765   PetscCall(DMPlexCreateVTKLabel_Internal(dm, PETSC_FALSE, sdm));
1766   PetscCall(DMGetNumLabels(sdm, &numLabels));
1767   for (dep = 0; dep <= depth; ++dep) {
1768     for (p = 0; p < numSplitPoints[dep]; ++p) {
1769       const PetscInt newp   = DMPlexShiftPoint_Internal(splitPoints[dep][p], depth, depthShift) /*depthOffset[dep] + splitPoints[dep][p]*/;
1770       const PetscInt splitp = pMaxNew[dep] + p;
1771       PetscInt       l;
1772 
1773       if (splitLabel) {
1774         const PetscInt val = 100 + dep;
1775 
1776         PetscCall(DMLabelSetValue(splitLabel, newp,    val));
1777         PetscCall(DMLabelSetValue(splitLabel, splitp, -val));
1778       }
1779       for (l = 0; l < numLabels; ++l) {
1780         DMLabel     mlabel;
1781         const char *lname;
1782         PetscInt    val;
1783         PetscBool   isDepth;
1784 
1785         PetscCall(DMGetLabelName(sdm, l, &lname));
1786         PetscCall(PetscStrcmp(lname, "depth", &isDepth));
1787         if (isDepth) continue;
1788         PetscCall(DMGetLabel(sdm, lname, &mlabel));
1789         PetscCall(DMLabelGetValue(mlabel, newp, &val));
1790         if (val >= 0) {
1791           PetscCall(DMLabelSetValue(mlabel, splitp, val));
1792         }
1793       }
1794     }
1795   }
1796   for (sp = 0; sp < numSP; ++sp) {
1797     const PetscInt dep = values[sp];
1798 
1799     if ((dep < 0) || (dep > depth)) continue;
1800     if (splitIS[dep]) PetscCall(ISRestoreIndices(splitIS[dep], &splitPoints[dep]));
1801     PetscCall(ISDestroy(&splitIS[dep]));
1802     if (unsplitIS[dep]) PetscCall(ISRestoreIndices(unsplitIS[dep], &unsplitPoints[dep]));
1803     PetscCall(ISDestroy(&unsplitIS[dep]));
1804   }
1805   if (ghostIS) PetscCall(ISRestoreIndices(ghostIS, &ghostPoints));
1806   PetscCall(ISDestroy(&ghostIS));
1807   if (label) {
1808     PetscCall(ISRestoreIndices(valueIS, &values));
1809     PetscCall(ISDestroy(&valueIS));
1810   }
1811   for (d = 0; d <= depth; ++d) {
1812     PetscCall(DMPlexGetDepthStratum(sdm, d, NULL, &pEnd));
1813     pMaxNew[d] = pEnd - numHybridPoints[d] - numHybridPointsOld[d];
1814   }
1815   PetscCall(PetscFree3(coneNew, coneONew, supportNew));
1816   PetscCall(PetscFree5(depthMax, depthEnd, depthShift, pMaxNew, numHybridPointsOld));
1817   PetscCall(PetscFree7(splitIS, unsplitIS, numSplitPoints, numUnsplitPoints, numHybridPoints, splitPoints, unsplitPoints));
1818   PetscFunctionReturn(0);
1819 }
1820 
1821 /*@C
1822   DMPlexConstructCohesiveCells - Construct cohesive cells which split the face along an internal interface
1823 
1824   Collective on dm
1825 
1826   Input Parameters:
1827 + dm - The original DM
1828 - label - The label specifying the boundary faces (this could be auto-generated)
1829 
1830   Output Parameters:
1831 + splitLabel - The label containing the split points, or NULL if no output is desired
1832 - dmSplit - The new DM
1833 
1834   Level: developer
1835 
1836 .seealso: `DMCreate()`, `DMPlexLabelCohesiveComplete()`
1837 @*/
1838 PetscErrorCode DMPlexConstructCohesiveCells(DM dm, DMLabel label, DMLabel splitLabel, DM *dmSplit)
1839 {
1840   DM             sdm;
1841   PetscInt       dim;
1842 
1843   PetscFunctionBegin;
1844   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1845   PetscValidPointer(dmSplit, 4);
1846   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &sdm));
1847   PetscCall(DMSetType(sdm, DMPLEX));
1848   PetscCall(DMGetDimension(dm, &dim));
1849   PetscCall(DMSetDimension(sdm, dim));
1850   switch (dim) {
1851   case 2:
1852   case 3:
1853     PetscCall(DMPlexConstructCohesiveCells_Internal(dm, label, splitLabel, sdm));
1854     break;
1855   default:
1856     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cannot construct cohesive cells for dimension %" PetscInt_FMT, dim);
1857   }
1858   *dmSplit = sdm;
1859   PetscFunctionReturn(0);
1860 }
1861 
1862 /* Returns the side of the surface for a given cell with a face on the surface */
1863 static PetscErrorCode GetSurfaceSide_Static(DM dm, DM subdm, PetscInt numSubpoints, const PetscInt *subpoints, PetscInt cell, PetscInt face, PetscBool *pos)
1864 {
1865   const PetscInt *cone, *ornt;
1866   PetscInt        dim, coneSize, c;
1867 
1868   PetscFunctionBegin;
1869   *pos = PETSC_TRUE;
1870   PetscCall(DMGetDimension(dm, &dim));
1871   PetscCall(DMPlexGetConeSize(dm, cell, &coneSize));
1872   PetscCall(DMPlexGetCone(dm, cell, &cone));
1873   PetscCall(DMPlexGetConeOrientation(dm, cell, &ornt));
1874   for (c = 0; c < coneSize; ++c) {
1875     if (cone[c] == face) {
1876       PetscInt o = ornt[c];
1877 
1878       if (subdm) {
1879         const PetscInt *subcone, *subornt;
1880         PetscInt        subpoint, subface, subconeSize, sc;
1881 
1882         PetscCall(PetscFindInt(cell, numSubpoints, subpoints, &subpoint));
1883         PetscCall(PetscFindInt(face, numSubpoints, subpoints, &subface));
1884         PetscCall(DMPlexGetConeSize(subdm, subpoint, &subconeSize));
1885         PetscCall(DMPlexGetCone(subdm, subpoint, &subcone));
1886         PetscCall(DMPlexGetConeOrientation(subdm, subpoint, &subornt));
1887         for (sc = 0; sc < subconeSize; ++sc) {
1888           if (subcone[sc] == subface) {
1889             o = subornt[0];
1890             break;
1891           }
1892         }
1893         PetscCheck(sc < subconeSize,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not find subpoint %" PetscInt_FMT " (%" PetscInt_FMT ") in cone for subpoint %" PetscInt_FMT " (%" PetscInt_FMT ")", subface, face, subpoint, cell);
1894       }
1895       if (o >= 0) *pos = PETSC_TRUE;
1896       else        *pos = PETSC_FALSE;
1897       break;
1898     }
1899   }
1900   PetscCheck(c != coneSize,PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Cell %" PetscInt_FMT " in split face %" PetscInt_FMT " support does not have it in the cone", cell, face);
1901   PetscFunctionReturn(0);
1902 }
1903 
1904 static PetscErrorCode CheckFaultEdge_Private(DM dm, DMLabel label)
1905 {
1906   IS              facePosIS, faceNegIS, dimIS;
1907   const PetscInt *points;
1908   PetscInt        dim, numPoints, p, shift = 100, shift2 = 200;
1909 
1910   PetscFunctionBegin;
1911   PetscCall(DMGetDimension(dm, &dim));
1912   /* If any faces touching the fault divide cells on either side, split them */
1913   PetscCall(DMLabelGetStratumIS(label,   shift+dim-1,  &facePosIS));
1914   PetscCall(DMLabelGetStratumIS(label, -(shift+dim-1), &faceNegIS));
1915   if (!facePosIS || !faceNegIS) {
1916     PetscCall(ISDestroy(&facePosIS));
1917     PetscCall(ISDestroy(&faceNegIS));
1918     PetscFunctionReturn(0);
1919   }
1920   PetscCall(ISExpand(facePosIS, faceNegIS, &dimIS));
1921   PetscCall(ISDestroy(&facePosIS));
1922   PetscCall(ISDestroy(&faceNegIS));
1923   PetscCall(ISGetLocalSize(dimIS, &numPoints));
1924   PetscCall(ISGetIndices(dimIS, &points));
1925   for (p = 0; p < numPoints; ++p) {
1926     const PetscInt  point = points[p];
1927     const PetscInt *support;
1928     PetscInt        supportSize, valA, valB;
1929 
1930     PetscCall(DMPlexGetSupportSize(dm, point, &supportSize));
1931     if (supportSize != 2) continue;
1932     PetscCall(DMPlexGetSupport(dm, point, &support));
1933     PetscCall(DMLabelGetValue(label, support[0], &valA));
1934     PetscCall(DMLabelGetValue(label, support[1], &valB));
1935     if ((valA == -1) || (valB == -1)) continue;
1936     if (valA*valB > 0) continue;
1937     /* Check that this face is not incident on only unsplit faces, meaning has at least one split face */
1938     {
1939       PetscInt *closure = NULL;
1940       PetscBool split   = PETSC_FALSE;
1941       PetscInt  closureSize, cl;
1942 
1943       PetscCall(DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
1944       for (cl = 0; cl < closureSize*2; cl += 2) {
1945         PetscCall(DMLabelGetValue(label, closure[cl], &valA));
1946         if ((valA >= 0) && (valA <= dim)) {split = PETSC_TRUE; break;}
1947       }
1948       PetscCall(DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
1949       if (!split) continue;
1950     }
1951     /* Split the face */
1952     PetscCall(DMLabelGetValue(label, point, &valA));
1953     PetscCall(DMLabelClearValue(label, point, valA));
1954     PetscCall(DMLabelSetValue(label, point, dim-1));
1955     /* Label its closure:
1956       unmarked: label as unsplit
1957       incident: relabel as split
1958       split:    do nothing
1959     */
1960     {
1961       PetscInt *closure = NULL;
1962       PetscInt  closureSize, cl, dep;
1963 
1964       PetscCall(DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
1965       for (cl = 0; cl < closureSize*2; cl += 2) {
1966         PetscCall(DMLabelGetValue(label, closure[cl], &valA));
1967         if (valA == -1) { /* Mark as unsplit */
1968           PetscCall(DMPlexGetPointDepth(dm, closure[cl], &dep));
1969           PetscCall(DMLabelSetValue(label, closure[cl], shift2+dep));
1970         } else if (((valA >= shift) && (valA < shift2)) || ((valA <= -shift) && (valA > -shift2))) {
1971           PetscCall(DMPlexGetPointDepth(dm, closure[cl], &dep));
1972           PetscCall(DMLabelClearValue(label, closure[cl], valA));
1973           PetscCall(DMLabelSetValue(label, closure[cl], dep));
1974         }
1975       }
1976       PetscCall(DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
1977     }
1978   }
1979   PetscCall(ISRestoreIndices(dimIS, &points));
1980   PetscCall(ISDestroy(&dimIS));
1981   PetscFunctionReturn(0);
1982 }
1983 
1984 /*@
1985   DMPlexLabelCohesiveComplete - Starting with a label marking points on an internal surface, we add all other mesh pieces
1986   to complete the surface
1987 
1988   Input Parameters:
1989 + dm     - The DM
1990 . label  - A DMLabel marking the surface
1991 . blabel - A DMLabel marking the vertices on the boundary which will not be duplicated, or NULL to find them automatically
1992 . flip   - Flag to flip the submesh normal and replace points on the other side
1993 - subdm  - The subDM associated with the label, or NULL
1994 
1995   Output Parameter:
1996 . label - A DMLabel marking all surface points
1997 
1998   Note: The vertices in blabel are called "unsplit" in the terminology from hybrid cell creation.
1999 
2000   Level: developer
2001 
2002 .seealso: `DMPlexConstructCohesiveCells()`, `DMPlexLabelComplete()`
2003 @*/
2004 PetscErrorCode DMPlexLabelCohesiveComplete(DM dm, DMLabel label, DMLabel blabel, PetscBool flip, DM subdm)
2005 {
2006   DMLabel         depthLabel;
2007   IS              dimIS, subpointIS = NULL;
2008   const PetscInt *points, *subpoints;
2009   const PetscInt  rev   = flip ? -1 : 1;
2010   PetscInt        shift = 100, shift2 = 200, shift3 = 300, dim, depth, numPoints, numSubpoints, p, val;
2011 
2012   PetscFunctionBegin;
2013   PetscCall(DMPlexGetDepth(dm, &depth));
2014   PetscCall(DMGetDimension(dm, &dim));
2015   PetscCall(DMPlexGetDepthLabel(dm, &depthLabel));
2016   if (subdm) {
2017     PetscCall(DMPlexGetSubpointIS(subdm, &subpointIS));
2018     if (subpointIS) {
2019       PetscCall(ISGetLocalSize(subpointIS, &numSubpoints));
2020       PetscCall(ISGetIndices(subpointIS, &subpoints));
2021     }
2022   }
2023   /* Mark cell on the fault, and its faces which touch the fault: cell orientation for face gives the side of the fault */
2024   PetscCall(DMLabelGetStratumIS(label, dim-1, &dimIS));
2025   if (!dimIS) goto divide;
2026   PetscCall(ISGetLocalSize(dimIS, &numPoints));
2027   PetscCall(ISGetIndices(dimIS, &points));
2028   for (p = 0; p < numPoints; ++p) { /* Loop over fault faces */
2029     const PetscInt *support;
2030     PetscInt        supportSize, s;
2031 
2032     PetscCall(DMPlexGetSupportSize(dm, points[p], &supportSize));
2033 #if 0
2034     if (supportSize != 2) {
2035       const PetscInt *lp;
2036       PetscInt        Nlp, pind;
2037 
2038       /* Check that for a cell with a single support face, that face is in the SF */
2039       /*   THis check only works for the remote side. We would need root side information */
2040       PetscCall(PetscSFGetGraph(dm->sf, NULL, &Nlp, &lp, NULL));
2041       PetscCall(PetscFindInt(points[p], Nlp, lp, &pind));
2042       PetscCheck(pind >= 0,PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Split face %" PetscInt_FMT " has %" PetscInt_FMT " != 2 supports, and the face is not shared with another process", points[p], supportSize);
2043     }
2044 #endif
2045     PetscCall(DMPlexGetSupport(dm, points[p], &support));
2046     for (s = 0; s < supportSize; ++s) {
2047       const PetscInt *cone;
2048       PetscInt        coneSize, c;
2049       PetscBool       pos;
2050 
2051       PetscCall(GetSurfaceSide_Static(dm, subdm, numSubpoints, subpoints, support[s], points[p], &pos));
2052       if (pos) PetscCall(DMLabelSetValue(label, support[s],  rev*(shift+dim)));
2053       else     PetscCall(DMLabelSetValue(label, support[s], -rev*(shift+dim)));
2054       if (rev < 0) pos = !pos ? PETSC_TRUE : PETSC_FALSE;
2055       /* Put faces touching the fault in the label */
2056       PetscCall(DMPlexGetConeSize(dm, support[s], &coneSize));
2057       PetscCall(DMPlexGetCone(dm, support[s], &cone));
2058       for (c = 0; c < coneSize; ++c) {
2059         const PetscInt point = cone[c];
2060 
2061         PetscCall(DMLabelGetValue(label, point, &val));
2062         if (val == -1) {
2063           PetscInt *closure = NULL;
2064           PetscInt  closureSize, cl;
2065 
2066           PetscCall(DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
2067           for (cl = 0; cl < closureSize*2; cl += 2) {
2068             const PetscInt clp  = closure[cl];
2069             PetscInt       bval = -1;
2070 
2071             PetscCall(DMLabelGetValue(label, clp, &val));
2072             if (blabel) PetscCall(DMLabelGetValue(blabel, clp, &bval));
2073             if ((val >= 0) && (val < dim-1) && (bval < 0)) {
2074               PetscCall(DMLabelSetValue(label, point, pos == PETSC_TRUE ? shift+dim-1 : -(shift+dim-1)));
2075               break;
2076             }
2077           }
2078           PetscCall(DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
2079         }
2080       }
2081     }
2082   }
2083   PetscCall(ISRestoreIndices(dimIS, &points));
2084   PetscCall(ISDestroy(&dimIS));
2085   /* Mark boundary points as unsplit */
2086   if (blabel) {
2087     IS bdIS;
2088 
2089     PetscCall(DMLabelGetStratumIS(blabel, 1, &bdIS));
2090     PetscCall(ISGetLocalSize(bdIS, &numPoints));
2091     PetscCall(ISGetIndices(bdIS, &points));
2092     for (p = 0; p < numPoints; ++p) {
2093       const PetscInt point = points[p];
2094       PetscInt       val, bval;
2095 
2096       PetscCall(DMLabelGetValue(blabel, point, &bval));
2097       if (bval >= 0) {
2098         PetscCall(DMLabelGetValue(label, point, &val));
2099         if ((val < 0) || (val > dim)) {
2100           /* This could be a point added from splitting a vertex on an adjacent fault, otherwise its just wrong */
2101           PetscCall(DMLabelClearValue(blabel, point, bval));
2102         }
2103       }
2104     }
2105     for (p = 0; p < numPoints; ++p) {
2106       const PetscInt point = points[p];
2107       PetscInt       val, bval;
2108 
2109       PetscCall(DMLabelGetValue(blabel, point, &bval));
2110       if (bval >= 0) {
2111         const PetscInt *cone,    *support;
2112         PetscInt        coneSize, supportSize, s, valA, valB, valE;
2113 
2114         /* Mark as unsplit */
2115         PetscCall(DMLabelGetValue(label, point, &val));
2116         PetscCheck(val >= 0 && val <= dim, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %" PetscInt_FMT " has label value %" PetscInt_FMT ", should be part of the fault", point, val);
2117         PetscCall(DMLabelClearValue(label, point, val));
2118         PetscCall(DMLabelSetValue(label, point, shift2+val));
2119         /* Check for cross-edge
2120              A cross-edge has endpoints which are both on the boundary of the surface, but the edge itself is not. */
2121         if (val != 0) continue;
2122         PetscCall(DMPlexGetSupport(dm, point, &support));
2123         PetscCall(DMPlexGetSupportSize(dm, point, &supportSize));
2124         for (s = 0; s < supportSize; ++s) {
2125           PetscCall(DMPlexGetCone(dm, support[s], &cone));
2126           PetscCall(DMPlexGetConeSize(dm, support[s], &coneSize));
2127           PetscCheck(coneSize == 2, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Edge %" PetscInt_FMT " has %" PetscInt_FMT " vertices != 2", support[s], coneSize);
2128           PetscCall(DMLabelGetValue(blabel, cone[0], &valA));
2129           PetscCall(DMLabelGetValue(blabel, cone[1], &valB));
2130           PetscCall(DMLabelGetValue(blabel, support[s], &valE));
2131           if ((valE < 0) && (valA >= 0) && (valB >= 0) && (cone[0] != cone[1])) PetscCall(DMLabelSetValue(blabel, support[s], 2));
2132         }
2133       }
2134     }
2135     PetscCall(ISRestoreIndices(bdIS, &points));
2136     PetscCall(ISDestroy(&bdIS));
2137   }
2138   /* Mark ghost fault cells */
2139   {
2140     PetscSF         sf;
2141     const PetscInt *leaves;
2142     PetscInt         Nl, l;
2143 
2144     PetscCall(DMGetPointSF(dm, &sf));
2145     PetscCall(PetscSFGetGraph(sf, NULL, &Nl, &leaves, NULL));
2146     PetscCall(DMLabelGetStratumIS(label, dim-1, &dimIS));
2147     if (!dimIS) goto divide;
2148     PetscCall(ISGetLocalSize(dimIS, &numPoints));
2149     PetscCall(ISGetIndices(dimIS, &points));
2150     if (Nl > 0) {
2151       for (p = 0; p < numPoints; ++p) {
2152         const PetscInt point = points[p];
2153         PetscInt       val;
2154 
2155         PetscCall(PetscFindInt(point, Nl, leaves, &l));
2156         if (l >= 0) {
2157           PetscInt *closure = NULL;
2158           PetscInt  closureSize, cl;
2159 
2160           PetscCall(DMLabelGetValue(label, point, &val));
2161           PetscCheck((val == dim-1) || (val == shift2+dim-1), PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %" PetscInt_FMT " has label value %" PetscInt_FMT ", should be a fault face", point, val);
2162           PetscCall(DMLabelClearValue(label, point, val));
2163           PetscCall(DMLabelSetValue(label, point, shift3+val));
2164           PetscCall(DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
2165           for (cl = 2; cl < closureSize*2; cl += 2) {
2166             const PetscInt clp  = closure[cl];
2167 
2168             PetscCall(DMLabelGetValue(label, clp, &val));
2169             PetscCheck(val != -1, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %" PetscInt_FMT " is missing from label, but is in the closure of a fault face", point);
2170             PetscCall(DMLabelClearValue(label, clp, val));
2171             PetscCall(DMLabelSetValue(label, clp, shift3+val));
2172           }
2173           PetscCall(DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
2174         }
2175       }
2176     }
2177     PetscCall(ISRestoreIndices(dimIS, &points));
2178     PetscCall(ISDestroy(&dimIS));
2179   }
2180   divide:
2181   if (subpointIS) PetscCall(ISRestoreIndices(subpointIS, &subpoints));
2182   PetscCall(DMPlexLabelFaultHalo(dm, label));
2183   PetscCall(CheckFaultEdge_Private(dm, label));
2184   PetscFunctionReturn(0);
2185 }
2186 
2187 /* Check that no cell have all vertices on the fault */
2188 PetscErrorCode DMPlexCheckValidSubmesh_Private(DM dm, DMLabel label, DM subdm)
2189 {
2190   IS              subpointIS;
2191   const PetscInt *dmpoints;
2192   PetscInt        defaultValue, cStart, cEnd, c, vStart, vEnd;
2193 
2194   PetscFunctionBegin;
2195   if (!label) PetscFunctionReturn(0);
2196   PetscCall(DMLabelGetDefaultValue(label, &defaultValue));
2197   PetscCall(DMPlexGetSubpointIS(subdm, &subpointIS));
2198   if (!subpointIS) PetscFunctionReturn(0);
2199   PetscCall(DMPlexGetHeightStratum(subdm, 0, &cStart, &cEnd));
2200   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
2201   PetscCall(ISGetIndices(subpointIS, &dmpoints));
2202   for (c = cStart; c < cEnd; ++c) {
2203     PetscBool invalidCell = PETSC_TRUE;
2204     PetscInt *closure     = NULL;
2205     PetscInt  closureSize, cl;
2206 
2207     PetscCall(DMPlexGetTransitiveClosure(dm, dmpoints[c], PETSC_TRUE, &closureSize, &closure));
2208     for (cl = 0; cl < closureSize*2; cl += 2) {
2209       PetscInt value = 0;
2210 
2211       if ((closure[cl] < vStart) || (closure[cl] >= vEnd)) continue;
2212       PetscCall(DMLabelGetValue(label, closure[cl], &value));
2213       if (value == defaultValue) {invalidCell = PETSC_FALSE; break;}
2214     }
2215     PetscCall(DMPlexRestoreTransitiveClosure(dm, dmpoints[c], PETSC_TRUE, &closureSize, &closure));
2216     if (invalidCell) {
2217       PetscCall(ISRestoreIndices(subpointIS, &dmpoints));
2218       PetscCall(ISDestroy(&subpointIS));
2219       PetscCall(DMDestroy(&subdm));
2220       SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Ambiguous submesh. Cell %" PetscInt_FMT " has all of its vertices on the submesh.", dmpoints[c]);
2221     }
2222   }
2223   PetscCall(ISRestoreIndices(subpointIS, &dmpoints));
2224   PetscFunctionReturn(0);
2225 }
2226 
2227 /*@
2228   DMPlexCreateHybridMesh - Create a mesh with hybrid cells along an internal interface
2229 
2230   Collective on dm
2231 
2232   Input Parameters:
2233 + dm - The original DM
2234 . label - The label specifying the interface vertices
2235 - bdlabel - The optional label specifying the interface boundary vertices
2236 
2237   Output Parameters:
2238 + hybridLabel - The label fully marking the interface, or NULL if no output is desired
2239 . splitLabel - The label containing the split points, or NULL if no output is desired
2240 . dmInterface - The new interface DM, or NULL
2241 - dmHybrid - The new DM with cohesive cells
2242 
2243   Note: The hybridLabel indicates what parts of the original mesh impinged on the on division surface. For points
2244   directly on the division surface, they are labeled with their dimension, so an edge 7 on the division surface would be
2245   7 (1) in hybridLabel. For points that impinge from the positive side, they are labeled with 100+dim, so an edge 6 with
2246   one vertex 3 on the surface would be 6 (101) and 3 (0) in hybridLabel. If an edge 9 from the negative side of the
2247   surface also hits vertex 3, it would be 9 (-101) in hybridLabel.
2248 
2249   The splitLabel indicates what points in the new hybrid mesh were the result of splitting points in the original
2250   mesh. The label value is +=100+dim for each point. For example, if two edges 10 and 14 in the hybrid resulting from
2251   splitting an edge in the original mesh, you would have 10 (101) and 14 (-101) in the splitLabel.
2252 
2253   The dmInterface is a DM built from the original division surface. It has a label which can be retrieved using
2254   DMPlexGetSubpointMap() which maps each point back to the point in the surface of the original mesh.
2255 
2256   Level: developer
2257 
2258 .seealso: `DMPlexConstructCohesiveCells()`, `DMPlexLabelCohesiveComplete()`, `DMPlexGetSubpointMap()`, `DMCreate()`
2259 @*/
2260 PetscErrorCode DMPlexCreateHybridMesh(DM dm, DMLabel label, DMLabel bdlabel, DMLabel *hybridLabel, DMLabel *splitLabel, DM *dmInterface, DM *dmHybrid)
2261 {
2262   DM             idm;
2263   DMLabel        subpointMap, hlabel, slabel = NULL;
2264   PetscInt       dim;
2265 
2266   PetscFunctionBegin;
2267   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2268   if (bdlabel) PetscValidPointer(bdlabel, 3);
2269   if (hybridLabel) PetscValidPointer(hybridLabel, 4);
2270   if (splitLabel)  PetscValidPointer(splitLabel, 5);
2271   if (dmInterface) PetscValidPointer(dmInterface, 6);
2272   PetscValidPointer(dmHybrid, 7);
2273   PetscCall(DMGetDimension(dm, &dim));
2274   PetscCall(DMPlexCreateSubmesh(dm, label, 1, PETSC_FALSE, &idm));
2275   PetscCall(DMPlexCheckValidSubmesh_Private(dm, label, idm));
2276   PetscCall(DMPlexOrient(idm));
2277   PetscCall(DMPlexGetSubpointMap(idm, &subpointMap));
2278   PetscCall(DMLabelDuplicate(subpointMap, &hlabel));
2279   PetscCall(DMLabelClearStratum(hlabel, dim));
2280   if (splitLabel) {
2281     const char *name;
2282     char        sname[PETSC_MAX_PATH_LEN];
2283 
2284     PetscCall(PetscObjectGetName((PetscObject) hlabel, &name));
2285     PetscCall(PetscStrncpy(sname, name, PETSC_MAX_PATH_LEN));
2286     PetscCall(PetscStrcat(sname, " split"));
2287     PetscCall(DMLabelCreate(PETSC_COMM_SELF, sname, &slabel));
2288   }
2289   PetscCall(DMPlexLabelCohesiveComplete(dm, hlabel, bdlabel, PETSC_FALSE, idm));
2290   if (dmInterface) {*dmInterface = idm;}
2291   else             PetscCall(DMDestroy(&idm));
2292   PetscCall(DMPlexConstructCohesiveCells(dm, hlabel, slabel, dmHybrid));
2293   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, *dmHybrid));
2294   if (hybridLabel) *hybridLabel = hlabel;
2295   else             PetscCall(DMLabelDestroy(&hlabel));
2296   if (splitLabel)  *splitLabel  = slabel;
2297   {
2298     DM      cdm;
2299     DMLabel ctLabel;
2300 
2301     /* We need to somehow share the celltype label with the coordinate dm */
2302     PetscCall(DMGetCoordinateDM(*dmHybrid, &cdm));
2303     PetscCall(DMPlexGetCellTypeLabel(*dmHybrid, &ctLabel));
2304     PetscCall(DMSetLabel(cdm, ctLabel));
2305   }
2306   PetscFunctionReturn(0);
2307 }
2308 
2309 /* Here we need the explicit assumption that:
2310 
2311      For any marked cell, the marked vertices constitute a single face
2312 */
2313 static PetscErrorCode DMPlexMarkSubmesh_Uninterpolated(DM dm, DMLabel vertexLabel, PetscInt value, DMLabel subpointMap, PetscInt *numFaces, PetscInt *nFV, DM subdm)
2314 {
2315   IS               subvertexIS = NULL;
2316   const PetscInt  *subvertices;
2317   PetscInt        *pStart, *pEnd, pSize;
2318   PetscInt         depth, dim, d, numSubVerticesInitial = 0, v;
2319 
2320   PetscFunctionBegin;
2321   *numFaces = 0;
2322   *nFV      = 0;
2323   PetscCall(DMPlexGetDepth(dm, &depth));
2324   PetscCall(DMGetDimension(dm, &dim));
2325   pSize = PetscMax(depth, dim) + 1;
2326   PetscCall(PetscMalloc2(pSize, &pStart, pSize, &pEnd));
2327   for (d = 0; d <= depth; ++d) {
2328     PetscCall(DMPlexGetSimplexOrBoxCells(dm, depth-d, &pStart[d], &pEnd[d]));
2329   }
2330   /* Loop over initial vertices and mark all faces in the collective star() */
2331   if (vertexLabel) PetscCall(DMLabelGetStratumIS(vertexLabel, value, &subvertexIS));
2332   if (subvertexIS) {
2333     PetscCall(ISGetSize(subvertexIS, &numSubVerticesInitial));
2334     PetscCall(ISGetIndices(subvertexIS, &subvertices));
2335   }
2336   for (v = 0; v < numSubVerticesInitial; ++v) {
2337     const PetscInt vertex = subvertices[v];
2338     PetscInt      *star   = NULL;
2339     PetscInt       starSize, s, numCells = 0, c;
2340 
2341     PetscCall(DMPlexGetTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star));
2342     for (s = 0; s < starSize*2; s += 2) {
2343       const PetscInt point = star[s];
2344       if ((point >= pStart[depth]) && (point < pEnd[depth])) star[numCells++] = point;
2345     }
2346     for (c = 0; c < numCells; ++c) {
2347       const PetscInt cell    = star[c];
2348       PetscInt      *closure = NULL;
2349       PetscInt       closureSize, cl;
2350       PetscInt       cellLoc, numCorners = 0, faceSize = 0;
2351 
2352       PetscCall(DMLabelGetValue(subpointMap, cell, &cellLoc));
2353       if (cellLoc == 2) continue;
2354       PetscCheck(cellLoc < 0,PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Cell %" PetscInt_FMT " has dimension %" PetscInt_FMT " in the surface label", cell, cellLoc);
2355       PetscCall(DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
2356       for (cl = 0; cl < closureSize*2; cl += 2) {
2357         const PetscInt point = closure[cl];
2358         PetscInt       vertexLoc;
2359 
2360         if ((point >= pStart[0]) && (point < pEnd[0])) {
2361           ++numCorners;
2362           PetscCall(DMLabelGetValue(vertexLabel, point, &vertexLoc));
2363           if (vertexLoc == value) closure[faceSize++] = point;
2364         }
2365       }
2366       if (!(*nFV)) PetscCall(DMPlexGetNumFaceVertices(dm, dim, numCorners, nFV));
2367       PetscCheck(faceSize <= *nFV,PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Invalid submesh: Too many vertices %" PetscInt_FMT " of an element on the surface", faceSize);
2368       if (faceSize == *nFV) {
2369         const PetscInt *cells = NULL;
2370         PetscInt        numCells, nc;
2371 
2372         ++(*numFaces);
2373         for (cl = 0; cl < faceSize; ++cl) {
2374           PetscCall(DMLabelSetValue(subpointMap, closure[cl], 0));
2375         }
2376         PetscCall(DMPlexGetJoin(dm, faceSize, closure, &numCells, &cells));
2377         for (nc = 0; nc < numCells; ++nc) {
2378           PetscCall(DMLabelSetValue(subpointMap, cells[nc], 2));
2379         }
2380         PetscCall(DMPlexRestoreJoin(dm, faceSize, closure, &numCells, &cells));
2381       }
2382       PetscCall(DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
2383     }
2384     PetscCall(DMPlexRestoreTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star));
2385   }
2386   if (subvertexIS) {
2387     PetscCall(ISRestoreIndices(subvertexIS, &subvertices));
2388   }
2389   PetscCall(ISDestroy(&subvertexIS));
2390   PetscCall(PetscFree2(pStart, pEnd));
2391   PetscFunctionReturn(0);
2392 }
2393 
2394 static PetscErrorCode DMPlexMarkSubmesh_Interpolated(DM dm, DMLabel vertexLabel, PetscInt value, PetscBool markedFaces, DMLabel subpointMap, DM subdm)
2395 {
2396   IS               subvertexIS = NULL;
2397   const PetscInt  *subvertices;
2398   PetscInt        *pStart, *pEnd;
2399   PetscInt         dim, d, numSubVerticesInitial = 0, v;
2400 
2401   PetscFunctionBegin;
2402   PetscCall(DMGetDimension(dm, &dim));
2403   PetscCall(PetscMalloc2(dim+1, &pStart, dim+1, &pEnd));
2404   for (d = 0; d <= dim; ++d) {
2405     PetscCall(DMPlexGetSimplexOrBoxCells(dm, dim-d, &pStart[d], &pEnd[d]));
2406   }
2407   /* Loop over initial vertices and mark all faces in the collective star() */
2408   if (vertexLabel) {
2409     PetscCall(DMLabelGetStratumIS(vertexLabel, value, &subvertexIS));
2410     if (subvertexIS) {
2411       PetscCall(ISGetSize(subvertexIS, &numSubVerticesInitial));
2412       PetscCall(ISGetIndices(subvertexIS, &subvertices));
2413     }
2414   }
2415   for (v = 0; v < numSubVerticesInitial; ++v) {
2416     const PetscInt vertex = subvertices[v];
2417     PetscInt      *star   = NULL;
2418     PetscInt       starSize, s, numFaces = 0, f;
2419 
2420     PetscCall(DMPlexGetTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star));
2421     for (s = 0; s < starSize*2; s += 2) {
2422       const PetscInt point = star[s];
2423       PetscInt       faceLoc;
2424 
2425       if ((point >= pStart[dim-1]) && (point < pEnd[dim-1])) {
2426         if (markedFaces) {
2427           PetscCall(DMLabelGetValue(vertexLabel, point, &faceLoc));
2428           if (faceLoc < 0) continue;
2429         }
2430         star[numFaces++] = point;
2431       }
2432     }
2433     for (f = 0; f < numFaces; ++f) {
2434       const PetscInt face    = star[f];
2435       PetscInt      *closure = NULL;
2436       PetscInt       closureSize, c;
2437       PetscInt       faceLoc;
2438 
2439       PetscCall(DMLabelGetValue(subpointMap, face, &faceLoc));
2440       if (faceLoc == dim-1) continue;
2441       PetscCheck(faceLoc < 0,PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Face %" PetscInt_FMT " has dimension %" PetscInt_FMT " in the surface label", face, faceLoc);
2442       PetscCall(DMPlexGetTransitiveClosure(dm, face, PETSC_TRUE, &closureSize, &closure));
2443       for (c = 0; c < closureSize*2; c += 2) {
2444         const PetscInt point = closure[c];
2445         PetscInt       vertexLoc;
2446 
2447         if ((point >= pStart[0]) && (point < pEnd[0])) {
2448           PetscCall(DMLabelGetValue(vertexLabel, point, &vertexLoc));
2449           if (vertexLoc != value) break;
2450         }
2451       }
2452       if (c == closureSize*2) {
2453         const PetscInt *support;
2454         PetscInt        supportSize, s;
2455 
2456         for (c = 0; c < closureSize*2; c += 2) {
2457           const PetscInt point = closure[c];
2458 
2459           for (d = 0; d < dim; ++d) {
2460             if ((point >= pStart[d]) && (point < pEnd[d])) {
2461               PetscCall(DMLabelSetValue(subpointMap, point, d));
2462               break;
2463             }
2464           }
2465         }
2466         PetscCall(DMPlexGetSupportSize(dm, face, &supportSize));
2467         PetscCall(DMPlexGetSupport(dm, face, &support));
2468         for (s = 0; s < supportSize; ++s) {
2469           PetscCall(DMLabelSetValue(subpointMap, support[s], dim));
2470         }
2471       }
2472       PetscCall(DMPlexRestoreTransitiveClosure(dm, face, PETSC_TRUE, &closureSize, &closure));
2473     }
2474     PetscCall(DMPlexRestoreTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star));
2475   }
2476   if (subvertexIS) PetscCall(ISRestoreIndices(subvertexIS, &subvertices));
2477   PetscCall(ISDestroy(&subvertexIS));
2478   PetscCall(PetscFree2(pStart, pEnd));
2479   PetscFunctionReturn(0);
2480 }
2481 
2482 static PetscErrorCode DMPlexMarkCohesiveSubmesh_Uninterpolated(DM dm, PetscBool hasLagrange, const char labelname[], PetscInt value, DMLabel subpointMap, PetscInt *numFaces, PetscInt *nFV, PetscInt *subCells[], DM subdm)
2483 {
2484   DMLabel         label = NULL;
2485   const PetscInt *cone;
2486   PetscInt        dim, cMax, cEnd, c, subc = 0, p, coneSize = -1;
2487 
2488   PetscFunctionBegin;
2489   *numFaces = 0;
2490   *nFV = 0;
2491   if (labelname) PetscCall(DMGetLabel(dm, labelname, &label));
2492   *subCells = NULL;
2493   PetscCall(DMGetDimension(dm, &dim));
2494   PetscCall(DMPlexGetTensorPrismBounds_Internal(dm, dim, &cMax, &cEnd));
2495   if (cMax < 0) PetscFunctionReturn(0);
2496   if (label) {
2497     for (c = cMax; c < cEnd; ++c) {
2498       PetscInt val;
2499 
2500       PetscCall(DMLabelGetValue(label, c, &val));
2501       if (val == value) {
2502         ++(*numFaces);
2503         PetscCall(DMPlexGetConeSize(dm, c, &coneSize));
2504       }
2505     }
2506   } else {
2507     *numFaces = cEnd - cMax;
2508     PetscCall(DMPlexGetConeSize(dm, cMax, &coneSize));
2509   }
2510   PetscCall(PetscMalloc1(*numFaces *2, subCells));
2511   if (!(*numFaces)) PetscFunctionReturn(0);
2512   *nFV = hasLagrange ? coneSize/3 : coneSize/2;
2513   for (c = cMax; c < cEnd; ++c) {
2514     const PetscInt *cells;
2515     PetscInt        numCells;
2516 
2517     if (label) {
2518       PetscInt val;
2519 
2520       PetscCall(DMLabelGetValue(label, c, &val));
2521       if (val != value) continue;
2522     }
2523     PetscCall(DMPlexGetCone(dm, c, &cone));
2524     for (p = 0; p < *nFV; ++p) {
2525       PetscCall(DMLabelSetValue(subpointMap, cone[p], 0));
2526     }
2527     /* Negative face */
2528     PetscCall(DMPlexGetJoin(dm, *nFV, cone, &numCells, &cells));
2529     /* Not true in parallel
2530     PetscCheck(numCells == 2,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive cells should separate two cells"); */
2531     for (p = 0; p < numCells; ++p) {
2532       PetscCall(DMLabelSetValue(subpointMap, cells[p], 2));
2533       (*subCells)[subc++] = cells[p];
2534     }
2535     PetscCall(DMPlexRestoreJoin(dm, *nFV, cone, &numCells, &cells));
2536     /* Positive face is not included */
2537   }
2538   PetscFunctionReturn(0);
2539 }
2540 
2541 static PetscErrorCode DMPlexMarkCohesiveSubmesh_Interpolated(DM dm, DMLabel label, PetscInt value, DMLabel subpointMap, DM subdm)
2542 {
2543   PetscInt      *pStart, *pEnd;
2544   PetscInt       dim, cMax, cEnd, c, d;
2545 
2546   PetscFunctionBegin;
2547   PetscCall(DMGetDimension(dm, &dim));
2548   PetscCall(DMPlexGetTensorPrismBounds_Internal(dm, dim, &cMax, &cEnd));
2549   if (cMax < 0) PetscFunctionReturn(0);
2550   PetscCall(PetscMalloc2(dim+1,&pStart,dim+1,&pEnd));
2551   for (d = 0; d <= dim; ++d) PetscCall(DMPlexGetDepthStratum(dm, d, &pStart[d], &pEnd[d]));
2552   for (c = cMax; c < cEnd; ++c) {
2553     const PetscInt *cone;
2554     PetscInt       *closure = NULL;
2555     PetscInt        fconeSize, coneSize, closureSize, cl, val;
2556 
2557     if (label) {
2558       PetscCall(DMLabelGetValue(label, c, &val));
2559       if (val != value) continue;
2560     }
2561     PetscCall(DMPlexGetConeSize(dm, c, &coneSize));
2562     PetscCall(DMPlexGetCone(dm, c, &cone));
2563     PetscCall(DMPlexGetConeSize(dm, cone[0], &fconeSize));
2564     PetscCheck(coneSize == (fconeSize ? fconeSize : 1) + 2,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive cells should separate two cells");
2565     /* Negative face */
2566     PetscCall(DMPlexGetTransitiveClosure(dm, cone[0], PETSC_TRUE, &closureSize, &closure));
2567     for (cl = 0; cl < closureSize*2; cl += 2) {
2568       const PetscInt point = closure[cl];
2569 
2570       for (d = 0; d <= dim; ++d) {
2571         if ((point >= pStart[d]) && (point < pEnd[d])) {
2572           PetscCall(DMLabelSetValue(subpointMap, point, d));
2573           break;
2574         }
2575       }
2576     }
2577     PetscCall(DMPlexRestoreTransitiveClosure(dm, cone[0], PETSC_TRUE, &closureSize, &closure));
2578     /* Cells -- positive face is not included */
2579     for (cl = 0; cl < 1; ++cl) {
2580       const PetscInt *support;
2581       PetscInt        supportSize, s;
2582 
2583       PetscCall(DMPlexGetSupportSize(dm, cone[cl], &supportSize));
2584       /* PetscCheck(supportSize == 2,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive faces should separate two cells"); */
2585       PetscCall(DMPlexGetSupport(dm, cone[cl], &support));
2586       for (s = 0; s < supportSize; ++s) {
2587         PetscCall(DMLabelSetValue(subpointMap, support[s], dim));
2588       }
2589     }
2590   }
2591   PetscCall(PetscFree2(pStart, pEnd));
2592   PetscFunctionReturn(0);
2593 }
2594 
2595 static PetscErrorCode DMPlexGetFaceOrientation(DM dm, PetscInt cell, PetscInt numCorners, PetscInt indices[], PetscInt oppositeVertex, PetscInt origVertices[], PetscInt faceVertices[], PetscBool *posOriented)
2596 {
2597   MPI_Comm       comm;
2598   PetscBool      posOrient = PETSC_FALSE;
2599   const PetscInt debug     = 0;
2600   PetscInt       cellDim, faceSize, f;
2601 
2602   PetscFunctionBegin;
2603   PetscCall(PetscObjectGetComm((PetscObject)dm,&comm));
2604   PetscCall(DMGetDimension(dm, &cellDim));
2605   if (debug) PetscCall(PetscPrintf(comm, "cellDim: %" PetscInt_FMT " numCorners: %" PetscInt_FMT "\n", cellDim, numCorners));
2606 
2607   if (cellDim == 1 && numCorners == 2) {
2608     /* Triangle */
2609     faceSize  = numCorners-1;
2610     posOrient = !(oppositeVertex%2) ? PETSC_TRUE : PETSC_FALSE;
2611   } else if (cellDim == 2 && numCorners == 3) {
2612     /* Triangle */
2613     faceSize  = numCorners-1;
2614     posOrient = !(oppositeVertex%2) ? PETSC_TRUE : PETSC_FALSE;
2615   } else if (cellDim == 3 && numCorners == 4) {
2616     /* Tetrahedron */
2617     faceSize  = numCorners-1;
2618     posOrient = (oppositeVertex%2) ? PETSC_TRUE : PETSC_FALSE;
2619   } else if (cellDim == 1 && numCorners == 3) {
2620     /* Quadratic line */
2621     faceSize  = 1;
2622     posOrient = PETSC_TRUE;
2623   } else if (cellDim == 2 && numCorners == 4) {
2624     /* Quads */
2625     faceSize = 2;
2626     if ((indices[1] > indices[0]) && (indices[1] - indices[0] == 1)) {
2627       posOrient = PETSC_TRUE;
2628     } else if ((indices[0] == 3) && (indices[1] == 0)) {
2629       posOrient = PETSC_TRUE;
2630     } else {
2631       if (((indices[0] > indices[1]) && (indices[0] - indices[1] == 1)) || ((indices[0] == 0) && (indices[1] == 3))) {
2632         posOrient = PETSC_FALSE;
2633       } else SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid quad crossedge");
2634     }
2635   } else if (cellDim == 2 && numCorners == 6) {
2636     /* Quadratic triangle (I hate this) */
2637     /* Edges are determined by the first 2 vertices (corners of edges) */
2638     const PetscInt faceSizeTri = 3;
2639     PetscInt       sortedIndices[3], i, iFace;
2640     PetscBool      found                    = PETSC_FALSE;
2641     PetscInt       faceVerticesTriSorted[9] = {
2642       0, 3,  4, /* bottom */
2643       1, 4,  5, /* right */
2644       2, 3,  5, /* left */
2645     };
2646     PetscInt       faceVerticesTri[9] = {
2647       0, 3,  4, /* bottom */
2648       1, 4,  5, /* right */
2649       2, 5,  3, /* left */
2650     };
2651 
2652     for (i = 0; i < faceSizeTri; ++i) sortedIndices[i] = indices[i];
2653     PetscCall(PetscSortInt(faceSizeTri, sortedIndices));
2654     for (iFace = 0; iFace < 3; ++iFace) {
2655       const PetscInt ii = iFace*faceSizeTri;
2656       PetscInt       fVertex, cVertex;
2657 
2658       if ((sortedIndices[0] == faceVerticesTriSorted[ii+0]) &&
2659           (sortedIndices[1] == faceVerticesTriSorted[ii+1])) {
2660         for (fVertex = 0; fVertex < faceSizeTri; ++fVertex) {
2661           for (cVertex = 0; cVertex < faceSizeTri; ++cVertex) {
2662             if (indices[cVertex] == faceVerticesTri[ii+fVertex]) {
2663               faceVertices[fVertex] = origVertices[cVertex];
2664               break;
2665             }
2666           }
2667         }
2668         found = PETSC_TRUE;
2669         break;
2670       }
2671     }
2672     PetscCheck(found,comm, PETSC_ERR_ARG_WRONG, "Invalid tri crossface");
2673     if (posOriented) *posOriented = PETSC_TRUE;
2674     PetscFunctionReturn(0);
2675   } else if (cellDim == 2 && numCorners == 9) {
2676     /* Quadratic quad (I hate this) */
2677     /* Edges are determined by the first 2 vertices (corners of edges) */
2678     const PetscInt faceSizeQuad = 3;
2679     PetscInt       sortedIndices[3], i, iFace;
2680     PetscBool      found                      = PETSC_FALSE;
2681     PetscInt       faceVerticesQuadSorted[12] = {
2682       0, 1,  4, /* bottom */
2683       1, 2,  5, /* right */
2684       2, 3,  6, /* top */
2685       0, 3,  7, /* left */
2686     };
2687     PetscInt       faceVerticesQuad[12] = {
2688       0, 1,  4, /* bottom */
2689       1, 2,  5, /* right */
2690       2, 3,  6, /* top */
2691       3, 0,  7, /* left */
2692     };
2693 
2694     for (i = 0; i < faceSizeQuad; ++i) sortedIndices[i] = indices[i];
2695     PetscCall(PetscSortInt(faceSizeQuad, sortedIndices));
2696     for (iFace = 0; iFace < 4; ++iFace) {
2697       const PetscInt ii = iFace*faceSizeQuad;
2698       PetscInt       fVertex, cVertex;
2699 
2700       if ((sortedIndices[0] == faceVerticesQuadSorted[ii+0]) &&
2701           (sortedIndices[1] == faceVerticesQuadSorted[ii+1])) {
2702         for (fVertex = 0; fVertex < faceSizeQuad; ++fVertex) {
2703           for (cVertex = 0; cVertex < faceSizeQuad; ++cVertex) {
2704             if (indices[cVertex] == faceVerticesQuad[ii+fVertex]) {
2705               faceVertices[fVertex] = origVertices[cVertex];
2706               break;
2707             }
2708           }
2709         }
2710         found = PETSC_TRUE;
2711         break;
2712       }
2713     }
2714     PetscCheck(found,comm, PETSC_ERR_ARG_WRONG, "Invalid quad crossface");
2715     if (posOriented) *posOriented = PETSC_TRUE;
2716     PetscFunctionReturn(0);
2717   } else if (cellDim == 3 && numCorners == 8) {
2718     /* Hexes
2719        A hex is two oriented quads with the normal of the first
2720        pointing up at the second.
2721 
2722           7---6
2723          /|  /|
2724         4---5 |
2725         | 1-|-2
2726         |/  |/
2727         0---3
2728 
2729         Faces are determined by the first 4 vertices (corners of faces) */
2730     const PetscInt faceSizeHex = 4;
2731     PetscInt       sortedIndices[4], i, iFace;
2732     PetscBool      found                     = PETSC_FALSE;
2733     PetscInt       faceVerticesHexSorted[24] = {
2734       0, 1, 2, 3,  /* bottom */
2735       4, 5, 6, 7,  /* top */
2736       0, 3, 4, 5,  /* front */
2737       2, 3, 5, 6,  /* right */
2738       1, 2, 6, 7,  /* back */
2739       0, 1, 4, 7,  /* left */
2740     };
2741     PetscInt       faceVerticesHex[24] = {
2742       1, 2, 3, 0,  /* bottom */
2743       4, 5, 6, 7,  /* top */
2744       0, 3, 5, 4,  /* front */
2745       3, 2, 6, 5,  /* right */
2746       2, 1, 7, 6,  /* back */
2747       1, 0, 4, 7,  /* left */
2748     };
2749 
2750     for (i = 0; i < faceSizeHex; ++i) sortedIndices[i] = indices[i];
2751     PetscCall(PetscSortInt(faceSizeHex, sortedIndices));
2752     for (iFace = 0; iFace < 6; ++iFace) {
2753       const PetscInt ii = iFace*faceSizeHex;
2754       PetscInt       fVertex, cVertex;
2755 
2756       if ((sortedIndices[0] == faceVerticesHexSorted[ii+0]) &&
2757           (sortedIndices[1] == faceVerticesHexSorted[ii+1]) &&
2758           (sortedIndices[2] == faceVerticesHexSorted[ii+2]) &&
2759           (sortedIndices[3] == faceVerticesHexSorted[ii+3])) {
2760         for (fVertex = 0; fVertex < faceSizeHex; ++fVertex) {
2761           for (cVertex = 0; cVertex < faceSizeHex; ++cVertex) {
2762             if (indices[cVertex] == faceVerticesHex[ii+fVertex]) {
2763               faceVertices[fVertex] = origVertices[cVertex];
2764               break;
2765             }
2766           }
2767         }
2768         found = PETSC_TRUE;
2769         break;
2770       }
2771     }
2772     PetscCheck(found,comm, PETSC_ERR_ARG_WRONG, "Invalid hex crossface");
2773     if (posOriented) *posOriented = PETSC_TRUE;
2774     PetscFunctionReturn(0);
2775   } else if (cellDim == 3 && numCorners == 10) {
2776     /* Quadratic tet */
2777     /* Faces are determined by the first 3 vertices (corners of faces) */
2778     const PetscInt faceSizeTet = 6;
2779     PetscInt       sortedIndices[6], i, iFace;
2780     PetscBool      found                     = PETSC_FALSE;
2781     PetscInt       faceVerticesTetSorted[24] = {
2782       0, 1, 2,  6, 7, 8, /* bottom */
2783       0, 3, 4,  6, 7, 9,  /* front */
2784       1, 4, 5,  7, 8, 9,  /* right */
2785       2, 3, 5,  6, 8, 9,  /* left */
2786     };
2787     PetscInt       faceVerticesTet[24] = {
2788       0, 1, 2,  6, 7, 8, /* bottom */
2789       0, 4, 3,  6, 7, 9,  /* front */
2790       1, 5, 4,  7, 8, 9,  /* right */
2791       2, 3, 5,  8, 6, 9,  /* left */
2792     };
2793 
2794     for (i = 0; i < faceSizeTet; ++i) sortedIndices[i] = indices[i];
2795     PetscCall(PetscSortInt(faceSizeTet, sortedIndices));
2796     for (iFace=0; iFace < 4; ++iFace) {
2797       const PetscInt ii = iFace*faceSizeTet;
2798       PetscInt       fVertex, cVertex;
2799 
2800       if ((sortedIndices[0] == faceVerticesTetSorted[ii+0]) &&
2801           (sortedIndices[1] == faceVerticesTetSorted[ii+1]) &&
2802           (sortedIndices[2] == faceVerticesTetSorted[ii+2]) &&
2803           (sortedIndices[3] == faceVerticesTetSorted[ii+3])) {
2804         for (fVertex = 0; fVertex < faceSizeTet; ++fVertex) {
2805           for (cVertex = 0; cVertex < faceSizeTet; ++cVertex) {
2806             if (indices[cVertex] == faceVerticesTet[ii+fVertex]) {
2807               faceVertices[fVertex] = origVertices[cVertex];
2808               break;
2809             }
2810           }
2811         }
2812         found = PETSC_TRUE;
2813         break;
2814       }
2815     }
2816     PetscCheck(found,comm, PETSC_ERR_ARG_WRONG, "Invalid tet crossface");
2817     if (posOriented) *posOriented = PETSC_TRUE;
2818     PetscFunctionReturn(0);
2819   } else if (cellDim == 3 && numCorners == 27) {
2820     /* Quadratic hexes (I hate this)
2821        A hex is two oriented quads with the normal of the first
2822        pointing up at the second.
2823 
2824          7---6
2825         /|  /|
2826        4---5 |
2827        | 3-|-2
2828        |/  |/
2829        0---1
2830 
2831        Faces are determined by the first 4 vertices (corners of faces) */
2832     const PetscInt faceSizeQuadHex = 9;
2833     PetscInt       sortedIndices[9], i, iFace;
2834     PetscBool      found                         = PETSC_FALSE;
2835     PetscInt       faceVerticesQuadHexSorted[54] = {
2836       0, 1, 2, 3,  8, 9, 10, 11,  24, /* bottom */
2837       4, 5, 6, 7,  12, 13, 14, 15,  25, /* top */
2838       0, 1, 4, 5,  8, 12, 16, 17,  22, /* front */
2839       1, 2, 5, 6,  9, 13, 17, 18,  21, /* right */
2840       2, 3, 6, 7,  10, 14, 18, 19,  23, /* back */
2841       0, 3, 4, 7,  11, 15, 16, 19,  20, /* left */
2842     };
2843     PetscInt       faceVerticesQuadHex[54] = {
2844       3, 2, 1, 0,  10, 9, 8, 11,  24, /* bottom */
2845       4, 5, 6, 7,  12, 13, 14, 15,  25, /* top */
2846       0, 1, 5, 4,  8, 17, 12, 16,  22, /* front */
2847       1, 2, 6, 5,  9, 18, 13, 17,  21, /* right */
2848       2, 3, 7, 6,  10, 19, 14, 18,  23, /* back */
2849       3, 0, 4, 7,  11, 16, 15, 19,  20 /* left */
2850     };
2851 
2852     for (i = 0; i < faceSizeQuadHex; ++i) sortedIndices[i] = indices[i];
2853     PetscCall(PetscSortInt(faceSizeQuadHex, sortedIndices));
2854     for (iFace = 0; iFace < 6; ++iFace) {
2855       const PetscInt ii = iFace*faceSizeQuadHex;
2856       PetscInt       fVertex, cVertex;
2857 
2858       if ((sortedIndices[0] == faceVerticesQuadHexSorted[ii+0]) &&
2859           (sortedIndices[1] == faceVerticesQuadHexSorted[ii+1]) &&
2860           (sortedIndices[2] == faceVerticesQuadHexSorted[ii+2]) &&
2861           (sortedIndices[3] == faceVerticesQuadHexSorted[ii+3])) {
2862         for (fVertex = 0; fVertex < faceSizeQuadHex; ++fVertex) {
2863           for (cVertex = 0; cVertex < faceSizeQuadHex; ++cVertex) {
2864             if (indices[cVertex] == faceVerticesQuadHex[ii+fVertex]) {
2865               faceVertices[fVertex] = origVertices[cVertex];
2866               break;
2867             }
2868           }
2869         }
2870         found = PETSC_TRUE;
2871         break;
2872       }
2873     }
2874     PetscCheck(found,comm, PETSC_ERR_ARG_WRONG, "Invalid hex crossface");
2875     if (posOriented) *posOriented = PETSC_TRUE;
2876     PetscFunctionReturn(0);
2877   } else SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Unknown cell type for faceOrientation().");
2878   if (!posOrient) {
2879     if (debug) PetscCall(PetscPrintf(comm, "  Reversing initial face orientation\n"));
2880     for (f = 0; f < faceSize; ++f) faceVertices[f] = origVertices[faceSize-1 - f];
2881   } else {
2882     if (debug) PetscCall(PetscPrintf(comm, "  Keeping initial face orientation\n"));
2883     for (f = 0; f < faceSize; ++f) faceVertices[f] = origVertices[f];
2884   }
2885   if (posOriented) *posOriented = posOrient;
2886   PetscFunctionReturn(0);
2887 }
2888 
2889 /*@
2890   DMPlexGetOrientedFace - Given a cell and a face, as a set of vertices, return the oriented face, as a set of vertices,
2891   in faceVertices. The orientation is such that the face normal points out of the cell
2892 
2893   Not collective
2894 
2895   Input Parameters:
2896 + dm           - The original mesh
2897 . cell         - The cell mesh point
2898 . faceSize     - The number of vertices on the face
2899 . face         - The face vertices
2900 . numCorners   - The number of vertices on the cell
2901 . indices      - Local numbering of face vertices in cell cone
2902 - origVertices - Original face vertices
2903 
2904   Output Parameters:
2905 + faceVertices - The face vertices properly oriented
2906 - posOriented  - PETSC_TRUE if the face was oriented with outward normal
2907 
2908   Level: developer
2909 
2910 .seealso: `DMPlexGetCone()`
2911 @*/
2912 PetscErrorCode DMPlexGetOrientedFace(DM dm, PetscInt cell, PetscInt faceSize, const PetscInt face[], PetscInt numCorners, PetscInt indices[], PetscInt origVertices[], PetscInt faceVertices[], PetscBool *posOriented)
2913 {
2914   const PetscInt *cone = NULL;
2915   PetscInt        coneSize, v, f, v2;
2916   PetscInt        oppositeVertex = -1;
2917 
2918   PetscFunctionBegin;
2919   PetscCall(DMPlexGetConeSize(dm, cell, &coneSize));
2920   PetscCall(DMPlexGetCone(dm, cell, &cone));
2921   for (v = 0, v2 = 0; v < coneSize; ++v) {
2922     PetscBool found = PETSC_FALSE;
2923 
2924     for (f = 0; f < faceSize; ++f) {
2925       if (face[f] == cone[v]) {
2926         found = PETSC_TRUE; break;
2927       }
2928     }
2929     if (found) {
2930       indices[v2]      = v;
2931       origVertices[v2] = cone[v];
2932       ++v2;
2933     } else {
2934       oppositeVertex = v;
2935     }
2936   }
2937   PetscCall(DMPlexGetFaceOrientation(dm, cell, numCorners, indices, oppositeVertex, origVertices, faceVertices, posOriented));
2938   PetscFunctionReturn(0);
2939 }
2940 
2941 /*
2942   DMPlexInsertFace_Internal - Puts a face into the mesh
2943 
2944   Not collective
2945 
2946   Input Parameters:
2947   + dm              - The DMPlex
2948   . numFaceVertex   - The number of vertices in the face
2949   . faceVertices    - The vertices in the face for dm
2950   . subfaceVertices - The vertices in the face for subdm
2951   . numCorners      - The number of vertices in the cell
2952   . cell            - A cell in dm containing the face
2953   . subcell         - A cell in subdm containing the face
2954   . firstFace       - First face in the mesh
2955   - newFacePoint    - Next face in the mesh
2956 
2957   Output Parameters:
2958   . newFacePoint - Contains next face point number on input, updated on output
2959 
2960   Level: developer
2961 */
2962 static PetscErrorCode DMPlexInsertFace_Internal(DM dm, DM subdm, PetscInt numFaceVertices, const PetscInt faceVertices[], const PetscInt subfaceVertices[], PetscInt numCorners, PetscInt cell, PetscInt subcell, PetscInt firstFace, PetscInt *newFacePoint)
2963 {
2964   MPI_Comm        comm;
2965   DM_Plex        *submesh = (DM_Plex*) subdm->data;
2966   const PetscInt *faces;
2967   PetscInt        numFaces, coneSize;
2968 
2969   PetscFunctionBegin;
2970   PetscCall(PetscObjectGetComm((PetscObject)dm,&comm));
2971   PetscCall(DMPlexGetConeSize(subdm, subcell, &coneSize));
2972   PetscCheck(coneSize == 1,comm, PETSC_ERR_ARG_OUTOFRANGE, "Cone size of cell %" PetscInt_FMT " is %" PetscInt_FMT " != 1", cell, coneSize);
2973 #if 0
2974   /* Cannot use this because support() has not been constructed yet */
2975   PetscCall(DMPlexGetJoin(subdm, numFaceVertices, subfaceVertices, &numFaces, &faces));
2976 #else
2977   {
2978     PetscInt f;
2979 
2980     numFaces = 0;
2981     PetscCall(DMGetWorkArray(subdm, 1, MPIU_INT, (void **) &faces));
2982     for (f = firstFace; f < *newFacePoint; ++f) {
2983       PetscInt dof, off, d;
2984 
2985       PetscCall(PetscSectionGetDof(submesh->coneSection, f, &dof));
2986       PetscCall(PetscSectionGetOffset(submesh->coneSection, f, &off));
2987       /* Yes, I know this is quadratic, but I expect the sizes to be <5 */
2988       for (d = 0; d < dof; ++d) {
2989         const PetscInt p = submesh->cones[off+d];
2990         PetscInt       v;
2991 
2992         for (v = 0; v < numFaceVertices; ++v) {
2993           if (subfaceVertices[v] == p) break;
2994         }
2995         if (v == numFaceVertices) break;
2996       }
2997       if (d == dof) {
2998         numFaces               = 1;
2999         ((PetscInt*) faces)[0] = f;
3000       }
3001     }
3002   }
3003 #endif
3004   PetscCheck(numFaces <= 1,comm, PETSC_ERR_ARG_WRONG, "Vertex set had %" PetscInt_FMT " faces, not one", numFaces);
3005   else if (numFaces == 1) {
3006     /* Add the other cell neighbor for this face */
3007     PetscCall(DMPlexSetCone(subdm, subcell, faces));
3008   } else {
3009     PetscInt *indices, *origVertices, *orientedVertices, *orientedSubVertices, v, ov;
3010     PetscBool posOriented;
3011 
3012     PetscCall(DMGetWorkArray(subdm, 4*numFaceVertices * sizeof(PetscInt), MPIU_INT, &orientedVertices));
3013     origVertices        = &orientedVertices[numFaceVertices];
3014     indices             = &orientedVertices[numFaceVertices*2];
3015     orientedSubVertices = &orientedVertices[numFaceVertices*3];
3016     PetscCall(DMPlexGetOrientedFace(dm, cell, numFaceVertices, faceVertices, numCorners, indices, origVertices, orientedVertices, &posOriented));
3017     /* TODO: I know that routine should return a permutation, not the indices */
3018     for (v = 0; v < numFaceVertices; ++v) {
3019       const PetscInt vertex = faceVertices[v], subvertex = subfaceVertices[v];
3020       for (ov = 0; ov < numFaceVertices; ++ov) {
3021         if (orientedVertices[ov] == vertex) {
3022           orientedSubVertices[ov] = subvertex;
3023           break;
3024         }
3025       }
3026       PetscCheck(ov != numFaceVertices,comm, PETSC_ERR_PLIB, "Could not find face vertex %" PetscInt_FMT " in orientated set", vertex);
3027     }
3028     PetscCall(DMPlexSetCone(subdm, *newFacePoint, orientedSubVertices));
3029     PetscCall(DMPlexSetCone(subdm, subcell, newFacePoint));
3030     PetscCall(DMRestoreWorkArray(subdm, 4*numFaceVertices * sizeof(PetscInt), MPIU_INT, &orientedVertices));
3031     ++(*newFacePoint);
3032   }
3033 #if 0
3034   PetscCall(DMPlexRestoreJoin(subdm, numFaceVertices, subfaceVertices, &numFaces, &faces));
3035 #else
3036   PetscCall(DMRestoreWorkArray(subdm, 1, MPIU_INT, (void **) &faces));
3037 #endif
3038   PetscFunctionReturn(0);
3039 }
3040 
3041 static PetscErrorCode DMPlexCreateSubmesh_Uninterpolated(DM dm, DMLabel vertexLabel, PetscInt value, DM subdm)
3042 {
3043   MPI_Comm        comm;
3044   DMLabel         subpointMap;
3045   IS              subvertexIS,  subcellIS;
3046   const PetscInt *subVertices, *subCells;
3047   PetscInt        numSubVertices, firstSubVertex, numSubCells;
3048   PetscInt       *subface, maxConeSize, numSubFaces = 0, firstSubFace, newFacePoint, nFV = 0;
3049   PetscInt        vStart, vEnd, c, f;
3050 
3051   PetscFunctionBegin;
3052   PetscCall(PetscObjectGetComm((PetscObject)dm,&comm));
3053   /* Create subpointMap which marks the submesh */
3054   PetscCall(DMLabelCreate(PETSC_COMM_SELF, "subpoint_map", &subpointMap));
3055   PetscCall(DMPlexSetSubpointMap(subdm, subpointMap));
3056   PetscCall(DMLabelDestroy(&subpointMap));
3057   if (vertexLabel) PetscCall(DMPlexMarkSubmesh_Uninterpolated(dm, vertexLabel, value, subpointMap, &numSubFaces, &nFV, subdm));
3058   /* Setup chart */
3059   PetscCall(DMLabelGetStratumSize(subpointMap, 0, &numSubVertices));
3060   PetscCall(DMLabelGetStratumSize(subpointMap, 2, &numSubCells));
3061   PetscCall(DMPlexSetChart(subdm, 0, numSubCells+numSubFaces+numSubVertices));
3062   PetscCall(DMPlexSetVTKCellHeight(subdm, 1));
3063   /* Set cone sizes */
3064   firstSubVertex = numSubCells;
3065   firstSubFace   = numSubCells+numSubVertices;
3066   newFacePoint   = firstSubFace;
3067   PetscCall(DMLabelGetStratumIS(subpointMap, 0, &subvertexIS));
3068   if (subvertexIS) PetscCall(ISGetIndices(subvertexIS, &subVertices));
3069   PetscCall(DMLabelGetStratumIS(subpointMap, 2, &subcellIS));
3070   if (subcellIS) PetscCall(ISGetIndices(subcellIS, &subCells));
3071   for (c = 0; c < numSubCells; ++c) {
3072     PetscCall(DMPlexSetConeSize(subdm, c, 1));
3073   }
3074   for (f = firstSubFace; f < firstSubFace+numSubFaces; ++f) {
3075     PetscCall(DMPlexSetConeSize(subdm, f, nFV));
3076   }
3077   PetscCall(DMSetUp(subdm));
3078   /* Create face cones */
3079   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
3080   PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, NULL));
3081   PetscCall(DMGetWorkArray(subdm, maxConeSize, MPIU_INT, (void**) &subface));
3082   for (c = 0; c < numSubCells; ++c) {
3083     const PetscInt cell    = subCells[c];
3084     const PetscInt subcell = c;
3085     PetscInt      *closure = NULL;
3086     PetscInt       closureSize, cl, numCorners = 0, faceSize = 0;
3087 
3088     PetscCall(DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
3089     for (cl = 0; cl < closureSize*2; cl += 2) {
3090       const PetscInt point = closure[cl];
3091       PetscInt       subVertex;
3092 
3093       if ((point >= vStart) && (point < vEnd)) {
3094         ++numCorners;
3095         PetscCall(PetscFindInt(point, numSubVertices, subVertices, &subVertex));
3096         if (subVertex >= 0) {
3097           closure[faceSize] = point;
3098           subface[faceSize] = firstSubVertex+subVertex;
3099           ++faceSize;
3100         }
3101       }
3102     }
3103     PetscCheck(faceSize <= nFV,comm, PETSC_ERR_ARG_WRONG, "Invalid submesh: Too many vertices %" PetscInt_FMT " of an element on the surface", faceSize);
3104     if (faceSize == nFV) {
3105       PetscCall(DMPlexInsertFace_Internal(dm, subdm, faceSize, closure, subface, numCorners, cell, subcell, firstSubFace, &newFacePoint));
3106     }
3107     PetscCall(DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
3108   }
3109   PetscCall(DMRestoreWorkArray(subdm, maxConeSize, MPIU_INT, (void**) &subface));
3110   PetscCall(DMPlexSymmetrize(subdm));
3111   PetscCall(DMPlexStratify(subdm));
3112   /* Build coordinates */
3113   {
3114     PetscSection coordSection, subCoordSection;
3115     Vec          coordinates, subCoordinates;
3116     PetscScalar *coords, *subCoords;
3117     PetscInt     numComp, coordSize, v;
3118     const char  *name;
3119 
3120     PetscCall(DMGetCoordinateSection(dm, &coordSection));
3121     PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
3122     PetscCall(DMGetCoordinateSection(subdm, &subCoordSection));
3123     PetscCall(PetscSectionSetNumFields(subCoordSection, 1));
3124     PetscCall(PetscSectionGetFieldComponents(coordSection, 0, &numComp));
3125     PetscCall(PetscSectionSetFieldComponents(subCoordSection, 0, numComp));
3126     PetscCall(PetscSectionSetChart(subCoordSection, firstSubVertex, firstSubVertex+numSubVertices));
3127     for (v = 0; v < numSubVertices; ++v) {
3128       const PetscInt vertex    = subVertices[v];
3129       const PetscInt subvertex = firstSubVertex+v;
3130       PetscInt       dof;
3131 
3132       PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3133       PetscCall(PetscSectionSetDof(subCoordSection, subvertex, dof));
3134       PetscCall(PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof));
3135     }
3136     PetscCall(PetscSectionSetUp(subCoordSection));
3137     PetscCall(PetscSectionGetStorageSize(subCoordSection, &coordSize));
3138     PetscCall(VecCreate(PETSC_COMM_SELF, &subCoordinates));
3139     PetscCall(PetscObjectGetName((PetscObject)coordinates,&name));
3140     PetscCall(PetscObjectSetName((PetscObject)subCoordinates,name));
3141     PetscCall(VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE));
3142     PetscCall(VecSetType(subCoordinates,VECSTANDARD));
3143     if (coordSize) {
3144       PetscCall(VecGetArray(coordinates,    &coords));
3145       PetscCall(VecGetArray(subCoordinates, &subCoords));
3146       for (v = 0; v < numSubVertices; ++v) {
3147         const PetscInt vertex    = subVertices[v];
3148         const PetscInt subvertex = firstSubVertex+v;
3149         PetscInt       dof, off, sdof, soff, d;
3150 
3151         PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3152         PetscCall(PetscSectionGetOffset(coordSection, vertex, &off));
3153         PetscCall(PetscSectionGetDof(subCoordSection, subvertex, &sdof));
3154         PetscCall(PetscSectionGetOffset(subCoordSection, subvertex, &soff));
3155         PetscCheck(dof == sdof,comm, PETSC_ERR_PLIB, "Coordinate dimension %" PetscInt_FMT " on subvertex %" PetscInt_FMT ", vertex %" PetscInt_FMT " should be %" PetscInt_FMT, sdof, subvertex, vertex, dof);
3156         for (d = 0; d < dof; ++d) subCoords[soff+d] = coords[off+d];
3157       }
3158       PetscCall(VecRestoreArray(coordinates,    &coords));
3159       PetscCall(VecRestoreArray(subCoordinates, &subCoords));
3160     }
3161     PetscCall(DMSetCoordinatesLocal(subdm, subCoordinates));
3162     PetscCall(VecDestroy(&subCoordinates));
3163   }
3164   /* Cleanup */
3165   if (subvertexIS) PetscCall(ISRestoreIndices(subvertexIS, &subVertices));
3166   PetscCall(ISDestroy(&subvertexIS));
3167   if (subcellIS) PetscCall(ISRestoreIndices(subcellIS, &subCells));
3168   PetscCall(ISDestroy(&subcellIS));
3169   PetscFunctionReturn(0);
3170 }
3171 
3172 /* TODO: Fix this to properly propogate up error conditions it may find */
3173 static inline PetscInt DMPlexFilterPoint_Internal(PetscInt point, PetscInt firstSubPoint, PetscInt numSubPoints, const PetscInt subPoints[])
3174 {
3175   PetscInt       subPoint;
3176   PetscErrorCode ierr;
3177 
3178   ierr = PetscFindInt(point, numSubPoints, subPoints, &subPoint); if (ierr) return -1;
3179   return subPoint < 0 ? subPoint : firstSubPoint+subPoint;
3180 }
3181 
3182 static PetscErrorCode DMPlexFilterLabels_Internal(DM dm, const PetscInt numSubPoints[], const PetscInt *subpoints[], const PetscInt firstSubPoint[], DM subdm)
3183 {
3184   PetscInt       Nl, l, d;
3185 
3186   PetscFunctionBegin;
3187   PetscCall(DMGetNumLabels(dm, &Nl));
3188   for (l = 0; l < Nl; ++l) {
3189     DMLabel         label, newlabel;
3190     const char     *lname;
3191     PetscBool       isDepth, isDim, isCelltype, isVTK;
3192     IS              valueIS;
3193     const PetscInt *values;
3194     PetscInt        Nv, v;
3195 
3196     PetscCall(DMGetLabelName(dm, l, &lname));
3197     PetscCall(PetscStrcmp(lname, "depth", &isDepth));
3198     PetscCall(PetscStrcmp(lname, "dim", &isDim));
3199     PetscCall(PetscStrcmp(lname, "celltype", &isCelltype));
3200     PetscCall(PetscStrcmp(lname, "vtk", &isVTK));
3201     if (isDepth || isDim || isCelltype || isVTK) continue;
3202     PetscCall(DMCreateLabel(subdm, lname));
3203     PetscCall(DMGetLabel(dm, lname, &label));
3204     PetscCall(DMGetLabel(subdm, lname, &newlabel));
3205     PetscCall(DMLabelGetDefaultValue(label, &v));
3206     PetscCall(DMLabelSetDefaultValue(newlabel, v));
3207     PetscCall(DMLabelGetValueIS(label, &valueIS));
3208     PetscCall(ISGetLocalSize(valueIS, &Nv));
3209     PetscCall(ISGetIndices(valueIS, &values));
3210     for (v = 0; v < Nv; ++v) {
3211       IS              pointIS;
3212       const PetscInt *points;
3213       PetscInt        Np, p;
3214 
3215       PetscCall(DMLabelGetStratumIS(label, values[v], &pointIS));
3216       PetscCall(ISGetLocalSize(pointIS, &Np));
3217       PetscCall(ISGetIndices(pointIS, &points));
3218       for (p = 0; p < Np; ++p) {
3219         const PetscInt point = points[p];
3220         PetscInt       subp;
3221 
3222         PetscCall(DMPlexGetPointDepth(dm, point, &d));
3223         subp = DMPlexFilterPoint_Internal(point, firstSubPoint[d], numSubPoints[d], subpoints[d]);
3224         if (subp >= 0) PetscCall(DMLabelSetValue(newlabel, subp, values[v]));
3225       }
3226       PetscCall(ISRestoreIndices(pointIS, &points));
3227       PetscCall(ISDestroy(&pointIS));
3228     }
3229     PetscCall(ISRestoreIndices(valueIS, &values));
3230     PetscCall(ISDestroy(&valueIS));
3231   }
3232   PetscFunctionReturn(0);
3233 }
3234 
3235 static PetscErrorCode DMPlexCreateSubmeshGeneric_Interpolated(DM dm, DMLabel label, PetscInt value, PetscBool markedFaces, PetscBool isCohesive, PetscInt cellHeight, DM subdm)
3236 {
3237   MPI_Comm         comm;
3238   DMLabel          subpointMap;
3239   IS              *subpointIS;
3240   const PetscInt **subpoints;
3241   PetscInt        *numSubPoints, *firstSubPoint, *coneNew, *orntNew;
3242   PetscInt         totSubPoints = 0, maxConeSize, dim, p, d, v;
3243   PetscMPIInt      rank;
3244 
3245   PetscFunctionBegin;
3246   PetscCall(PetscObjectGetComm((PetscObject)dm,&comm));
3247   PetscCallMPI(MPI_Comm_rank(comm, &rank));
3248   /* Create subpointMap which marks the submesh */
3249   PetscCall(DMLabelCreate(PETSC_COMM_SELF, "subpoint_map", &subpointMap));
3250   PetscCall(DMPlexSetSubpointMap(subdm, subpointMap));
3251   if (cellHeight) {
3252     if (isCohesive) PetscCall(DMPlexMarkCohesiveSubmesh_Interpolated(dm, label, value, subpointMap, subdm));
3253     else            PetscCall(DMPlexMarkSubmesh_Interpolated(dm, label, value, markedFaces, subpointMap, subdm));
3254   } else {
3255     DMLabel         depth;
3256     IS              pointIS;
3257     const PetscInt *points;
3258     PetscInt        numPoints=0;
3259 
3260     PetscCall(DMPlexGetDepthLabel(dm, &depth));
3261     PetscCall(DMLabelGetStratumIS(label, value, &pointIS));
3262     if (pointIS) {
3263       PetscCall(ISGetIndices(pointIS, &points));
3264       PetscCall(ISGetLocalSize(pointIS, &numPoints));
3265     }
3266     for (p = 0; p < numPoints; ++p) {
3267       PetscInt *closure = NULL;
3268       PetscInt  closureSize, c, pdim;
3269 
3270       PetscCall(DMPlexGetTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closure));
3271       for (c = 0; c < closureSize*2; c += 2) {
3272         PetscCall(DMLabelGetValue(depth, closure[c], &pdim));
3273         PetscCall(DMLabelSetValue(subpointMap, closure[c], pdim));
3274       }
3275       PetscCall(DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closure));
3276     }
3277     if (pointIS) PetscCall(ISRestoreIndices(pointIS, &points));
3278     PetscCall(ISDestroy(&pointIS));
3279   }
3280   /* Setup chart */
3281   PetscCall(DMGetDimension(dm, &dim));
3282   PetscCall(PetscMalloc4(dim+1,&numSubPoints,dim+1,&firstSubPoint,dim+1,&subpointIS,dim+1,&subpoints));
3283   for (d = 0; d <= dim; ++d) {
3284     PetscCall(DMLabelGetStratumSize(subpointMap, d, &numSubPoints[d]));
3285     totSubPoints += numSubPoints[d];
3286   }
3287   PetscCall(DMPlexSetChart(subdm, 0, totSubPoints));
3288   PetscCall(DMPlexSetVTKCellHeight(subdm, cellHeight));
3289   /* Set cone sizes */
3290   firstSubPoint[dim] = 0;
3291   firstSubPoint[0]   = firstSubPoint[dim] + numSubPoints[dim];
3292   if (dim > 1) {firstSubPoint[dim-1] = firstSubPoint[0]     + numSubPoints[0];}
3293   if (dim > 2) {firstSubPoint[dim-2] = firstSubPoint[dim-1] + numSubPoints[dim-1];}
3294   for (d = 0; d <= dim; ++d) {
3295     PetscCall(DMLabelGetStratumIS(subpointMap, d, &subpointIS[d]));
3296     if (subpointIS[d]) PetscCall(ISGetIndices(subpointIS[d], &subpoints[d]));
3297   }
3298   /* We do not want this label automatically computed, instead we compute it here */
3299   PetscCall(DMCreateLabel(subdm, "celltype"));
3300   for (d = 0; d <= dim; ++d) {
3301     for (p = 0; p < numSubPoints[d]; ++p) {
3302       const PetscInt  point    = subpoints[d][p];
3303       const PetscInt  subpoint = firstSubPoint[d] + p;
3304       const PetscInt *cone;
3305       PetscInt        coneSize;
3306 
3307       PetscCall(DMPlexGetConeSize(dm, point, &coneSize));
3308       if (cellHeight && (d == dim)) {
3309         PetscInt coneSizeNew, c, val;
3310 
3311         PetscCall(DMPlexGetCone(dm, point, &cone));
3312         for (c = 0, coneSizeNew = 0; c < coneSize; ++c) {
3313           PetscCall(DMLabelGetValue(subpointMap, cone[c], &val));
3314           if (val >= 0) coneSizeNew++;
3315         }
3316         PetscCall(DMPlexSetConeSize(subdm, subpoint, coneSizeNew));
3317         PetscCall(DMPlexSetCellType(subdm, subpoint, DM_POLYTOPE_FV_GHOST));
3318       } else {
3319         DMPolytopeType  ct;
3320 
3321         PetscCall(DMPlexSetConeSize(subdm, subpoint, coneSize));
3322         PetscCall(DMPlexGetCellType(dm, point, &ct));
3323         PetscCall(DMPlexSetCellType(subdm, subpoint, ct));
3324       }
3325     }
3326   }
3327   PetscCall(DMLabelDestroy(&subpointMap));
3328   PetscCall(DMSetUp(subdm));
3329   /* Set cones */
3330   PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, NULL));
3331   PetscCall(PetscMalloc2(maxConeSize,&coneNew,maxConeSize,&orntNew));
3332   for (d = 0; d <= dim; ++d) {
3333     for (p = 0; p < numSubPoints[d]; ++p) {
3334       const PetscInt  point    = subpoints[d][p];
3335       const PetscInt  subpoint = firstSubPoint[d] + p;
3336       const PetscInt *cone, *ornt;
3337       PetscInt        coneSize, subconeSize, coneSizeNew, c, subc, fornt = 0;
3338 
3339       if (d == dim-1) {
3340         const PetscInt *support, *cone, *ornt;
3341         PetscInt        supportSize, coneSize, s, subc;
3342 
3343         PetscCall(DMPlexGetSupport(dm, point, &support));
3344         PetscCall(DMPlexGetSupportSize(dm, point, &supportSize));
3345         for (s = 0; s < supportSize; ++s) {
3346           PetscBool isHybrid;
3347 
3348           PetscCall(DMPlexCellIsHybrid_Internal(dm, support[s], &isHybrid));
3349           if (!isHybrid) continue;
3350           PetscCall(PetscFindInt(support[s], numSubPoints[d+1], subpoints[d+1], &subc));
3351           if (subc >= 0) {
3352             const PetscInt ccell = subpoints[d+1][subc];
3353 
3354             PetscCall(DMPlexGetCone(dm, ccell, &cone));
3355             PetscCall(DMPlexGetConeSize(dm, ccell, &coneSize));
3356             PetscCall(DMPlexGetConeOrientation(dm, ccell, &ornt));
3357             for (c = 0; c < coneSize; ++c) {
3358               if (cone[c] == point) {
3359                 fornt = ornt[c];
3360                 break;
3361               }
3362             }
3363             break;
3364           }
3365         }
3366       }
3367       PetscCall(DMPlexGetConeSize(dm, point, &coneSize));
3368       PetscCall(DMPlexGetConeSize(subdm, subpoint, &subconeSize));
3369       PetscCall(DMPlexGetCone(dm, point, &cone));
3370       PetscCall(DMPlexGetConeOrientation(dm, point, &ornt));
3371       for (c = 0, coneSizeNew = 0; c < coneSize; ++c) {
3372         PetscCall(PetscFindInt(cone[c], numSubPoints[d-1], subpoints[d-1], &subc));
3373         if (subc >= 0) {
3374           coneNew[coneSizeNew] = firstSubPoint[d-1] + subc;
3375           orntNew[coneSizeNew] = ornt[c];
3376           ++coneSizeNew;
3377         }
3378       }
3379       PetscCheck(coneSizeNew == subconeSize,comm, PETSC_ERR_PLIB, "Number of cone points located %" PetscInt_FMT " does not match subcone size %" PetscInt_FMT, coneSizeNew, subconeSize);
3380       PetscCall(DMPlexSetCone(subdm, subpoint, coneNew));
3381       PetscCall(DMPlexSetConeOrientation(subdm, subpoint, orntNew));
3382       if (fornt < 0) PetscCall(DMPlexOrientPoint(subdm, subpoint, fornt));
3383     }
3384   }
3385   PetscCall(PetscFree2(coneNew,orntNew));
3386   PetscCall(DMPlexSymmetrize(subdm));
3387   PetscCall(DMPlexStratify(subdm));
3388   /* Build coordinates */
3389   {
3390     PetscSection coordSection, subCoordSection;
3391     Vec          coordinates, subCoordinates;
3392     PetscScalar *coords, *subCoords;
3393     PetscInt     cdim, numComp, coordSize;
3394     const char  *name;
3395 
3396     PetscCall(DMGetCoordinateDim(dm, &cdim));
3397     PetscCall(DMGetCoordinateSection(dm, &coordSection));
3398     PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
3399     PetscCall(DMGetCoordinateSection(subdm, &subCoordSection));
3400     PetscCall(PetscSectionSetNumFields(subCoordSection, 1));
3401     PetscCall(PetscSectionGetFieldComponents(coordSection, 0, &numComp));
3402     PetscCall(PetscSectionSetFieldComponents(subCoordSection, 0, numComp));
3403     PetscCall(PetscSectionSetChart(subCoordSection, firstSubPoint[0], firstSubPoint[0]+numSubPoints[0]));
3404     for (v = 0; v < numSubPoints[0]; ++v) {
3405       const PetscInt vertex    = subpoints[0][v];
3406       const PetscInt subvertex = firstSubPoint[0]+v;
3407       PetscInt       dof;
3408 
3409       PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3410       PetscCall(PetscSectionSetDof(subCoordSection, subvertex, dof));
3411       PetscCall(PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof));
3412     }
3413     PetscCall(PetscSectionSetUp(subCoordSection));
3414     PetscCall(PetscSectionGetStorageSize(subCoordSection, &coordSize));
3415     PetscCall(VecCreate(PETSC_COMM_SELF, &subCoordinates));
3416     PetscCall(PetscObjectGetName((PetscObject)coordinates,&name));
3417     PetscCall(PetscObjectSetName((PetscObject)subCoordinates,name));
3418     PetscCall(VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE));
3419     PetscCall(VecSetBlockSize(subCoordinates, cdim));
3420     PetscCall(VecSetType(subCoordinates,VECSTANDARD));
3421     PetscCall(VecGetArray(coordinates,    &coords));
3422     PetscCall(VecGetArray(subCoordinates, &subCoords));
3423     for (v = 0; v < numSubPoints[0]; ++v) {
3424       const PetscInt vertex    = subpoints[0][v];
3425       const PetscInt subvertex = firstSubPoint[0]+v;
3426       PetscInt dof, off, sdof, soff, d;
3427 
3428       PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3429       PetscCall(PetscSectionGetOffset(coordSection, vertex, &off));
3430       PetscCall(PetscSectionGetDof(subCoordSection, subvertex, &sdof));
3431       PetscCall(PetscSectionGetOffset(subCoordSection, subvertex, &soff));
3432       PetscCheck(dof == sdof,comm, PETSC_ERR_PLIB, "Coordinate dimension %" PetscInt_FMT " on subvertex %" PetscInt_FMT ", vertex %" PetscInt_FMT " should be %" PetscInt_FMT, sdof, subvertex, vertex, dof);
3433       for (d = 0; d < dof; ++d) subCoords[soff+d] = coords[off+d];
3434     }
3435     PetscCall(VecRestoreArray(coordinates,    &coords));
3436     PetscCall(VecRestoreArray(subCoordinates, &subCoords));
3437     PetscCall(DMSetCoordinatesLocal(subdm, subCoordinates));
3438     PetscCall(VecDestroy(&subCoordinates));
3439   }
3440   /* Build SF: We need this complexity because subpoints might not be selected on the owning process */
3441   {
3442     PetscSF            sfPoint, sfPointSub;
3443     IS                 subpIS;
3444     const PetscSFNode *remotePoints;
3445     PetscSFNode       *sremotePoints, *newLocalPoints, *newOwners;
3446     const PetscInt    *localPoints, *subpoints;
3447     PetscInt          *slocalPoints;
3448     PetscInt           numRoots, numLeaves, numSubpoints = 0, numSubroots, numSubleaves = 0, l, sl, ll, pStart, pEnd, p;
3449     PetscMPIInt        rank;
3450 
3451     PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank));
3452     PetscCall(DMGetPointSF(dm, &sfPoint));
3453     PetscCall(DMGetPointSF(subdm, &sfPointSub));
3454     PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
3455     PetscCall(DMPlexGetChart(subdm, NULL, &numSubroots));
3456     PetscCall(DMPlexGetSubpointIS(subdm, &subpIS));
3457     if (subpIS) {
3458       PetscCall(ISGetIndices(subpIS, &subpoints));
3459       PetscCall(ISGetLocalSize(subpIS, &numSubpoints));
3460     }
3461     PetscCall(PetscSFGetGraph(sfPoint, &numRoots, &numLeaves, &localPoints, &remotePoints));
3462     if (numRoots >= 0) {
3463       PetscCall(PetscMalloc2(pEnd-pStart,&newLocalPoints,numRoots,&newOwners));
3464       for (p = 0; p < pEnd-pStart; ++p) {
3465         newLocalPoints[p].rank  = -2;
3466         newLocalPoints[p].index = -2;
3467       }
3468       /* Set subleaves */
3469       for (l = 0; l < numLeaves; ++l) {
3470         const PetscInt point    = localPoints[l];
3471         const PetscInt subpoint = DMPlexFilterPoint_Internal(point, 0, numSubpoints, subpoints);
3472 
3473         if (subpoint < 0) continue;
3474         newLocalPoints[point-pStart].rank  = rank;
3475         newLocalPoints[point-pStart].index = subpoint;
3476         ++numSubleaves;
3477       }
3478       /* Must put in owned subpoints */
3479       for (p = pStart; p < pEnd; ++p) {
3480         const PetscInt subpoint = DMPlexFilterPoint_Internal(p, 0, numSubpoints, subpoints);
3481 
3482         if (subpoint < 0) {
3483           newOwners[p-pStart].rank  = -3;
3484           newOwners[p-pStart].index = -3;
3485         } else {
3486           newOwners[p-pStart].rank  = rank;
3487           newOwners[p-pStart].index = subpoint;
3488         }
3489       }
3490       PetscCall(PetscSFReduceBegin(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC));
3491       PetscCall(PetscSFReduceEnd(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC));
3492       PetscCall(PetscSFBcastBegin(sfPoint, MPIU_2INT, newOwners, newLocalPoints,MPI_REPLACE));
3493       PetscCall(PetscSFBcastEnd(sfPoint, MPIU_2INT, newOwners, newLocalPoints,MPI_REPLACE));
3494       PetscCall(PetscMalloc1(numSubleaves, &slocalPoints));
3495       PetscCall(PetscMalloc1(numSubleaves, &sremotePoints));
3496       for (l = 0, sl = 0, ll = 0; l < numLeaves; ++l) {
3497         const PetscInt point    = localPoints[l];
3498         const PetscInt subpoint = DMPlexFilterPoint_Internal(point, 0, numSubpoints, subpoints);
3499 
3500         if (subpoint < 0) continue;
3501         if (newLocalPoints[point].rank == rank) {++ll; continue;}
3502         slocalPoints[sl]        = subpoint;
3503         sremotePoints[sl].rank  = newLocalPoints[point].rank;
3504         sremotePoints[sl].index = newLocalPoints[point].index;
3505         PetscCheck(sremotePoints[sl].rank  >= 0,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote rank for local point %" PetscInt_FMT, point);
3506         PetscCheck(sremotePoints[sl].index >= 0,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote subpoint for local point %" PetscInt_FMT, point);
3507         ++sl;
3508       }
3509       PetscCheck(sl + ll == numSubleaves,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatch in number of subleaves %" PetscInt_FMT " + %" PetscInt_FMT " != %" PetscInt_FMT, sl, ll, numSubleaves);
3510       PetscCall(PetscFree2(newLocalPoints,newOwners));
3511       PetscCall(PetscSFSetGraph(sfPointSub, numSubroots, sl, slocalPoints, PETSC_OWN_POINTER, sremotePoints, PETSC_OWN_POINTER));
3512     }
3513     if (subpIS) {
3514       PetscCall(ISRestoreIndices(subpIS, &subpoints));
3515     }
3516   }
3517   /* Filter labels */
3518   PetscCall(DMPlexFilterLabels_Internal(dm, numSubPoints, subpoints, firstSubPoint, subdm));
3519   /* Cleanup */
3520   for (d = 0; d <= dim; ++d) {
3521     if (subpointIS[d]) PetscCall(ISRestoreIndices(subpointIS[d], &subpoints[d]));
3522     PetscCall(ISDestroy(&subpointIS[d]));
3523   }
3524   PetscCall(PetscFree4(numSubPoints,firstSubPoint,subpointIS,subpoints));
3525   PetscFunctionReturn(0);
3526 }
3527 
3528 static PetscErrorCode DMPlexCreateSubmesh_Interpolated(DM dm, DMLabel vertexLabel, PetscInt value, PetscBool markedFaces, DM subdm)
3529 {
3530   PetscFunctionBegin;
3531   PetscCall(DMPlexCreateSubmeshGeneric_Interpolated(dm, vertexLabel, value, markedFaces, PETSC_FALSE, 1, subdm));
3532   PetscFunctionReturn(0);
3533 }
3534 
3535 /*@
3536   DMPlexCreateSubmesh - Extract a hypersurface from the mesh using vertices defined by a label
3537 
3538   Input Parameters:
3539 + dm           - The original mesh
3540 . vertexLabel  - The DMLabel marking points contained in the surface
3541 . value        - The label value to use
3542 - markedFaces  - PETSC_TRUE if surface faces are marked in addition to vertices, PETSC_FALSE if only vertices are marked
3543 
3544   Output Parameter:
3545 . subdm - The surface mesh
3546 
3547   Note: This function produces a DMLabel mapping original points in the submesh to their depth. This can be obtained using DMPlexGetSubpointMap().
3548 
3549   Level: developer
3550 
3551 .seealso: `DMPlexGetSubpointMap()`, `DMGetLabel()`, `DMLabelSetValue()`
3552 @*/
3553 PetscErrorCode DMPlexCreateSubmesh(DM dm, DMLabel vertexLabel, PetscInt value, PetscBool markedFaces, DM *subdm)
3554 {
3555   DMPlexInterpolatedFlag interpolated;
3556   PetscInt       dim, cdim;
3557 
3558   PetscFunctionBegin;
3559   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3560   PetscValidPointer(subdm, 5);
3561   PetscCall(DMGetDimension(dm, &dim));
3562   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), subdm));
3563   PetscCall(DMSetType(*subdm, DMPLEX));
3564   PetscCall(DMSetDimension(*subdm, dim-1));
3565   PetscCall(DMGetCoordinateDim(dm, &cdim));
3566   PetscCall(DMSetCoordinateDim(*subdm, cdim));
3567   PetscCall(DMPlexIsInterpolated(dm, &interpolated));
3568   PetscCheck(interpolated != DMPLEX_INTERPOLATED_PARTIAL,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Not for partially interpolated meshes");
3569   if (interpolated) {
3570     PetscCall(DMPlexCreateSubmesh_Interpolated(dm, vertexLabel, value, markedFaces, *subdm));
3571   } else {
3572     PetscCall(DMPlexCreateSubmesh_Uninterpolated(dm, vertexLabel, value, *subdm));
3573   }
3574   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, *subdm));
3575   PetscFunctionReturn(0);
3576 }
3577 
3578 static PetscErrorCode DMPlexCreateCohesiveSubmesh_Uninterpolated(DM dm, PetscBool hasLagrange, const char label[], PetscInt value, DM subdm)
3579 {
3580   MPI_Comm        comm;
3581   DMLabel         subpointMap;
3582   IS              subvertexIS;
3583   const PetscInt *subVertices;
3584   PetscInt        numSubVertices, firstSubVertex, numSubCells, *subCells = NULL;
3585   PetscInt       *subface, maxConeSize, numSubFaces, firstSubFace, newFacePoint, nFV;
3586   PetscInt        c, f;
3587 
3588   PetscFunctionBegin;
3589   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
3590   /* Create subpointMap which marks the submesh */
3591   PetscCall(DMLabelCreate(PETSC_COMM_SELF, "subpoint_map", &subpointMap));
3592   PetscCall(DMPlexSetSubpointMap(subdm, subpointMap));
3593   PetscCall(DMLabelDestroy(&subpointMap));
3594   PetscCall(DMPlexMarkCohesiveSubmesh_Uninterpolated(dm, hasLagrange, label, value, subpointMap, &numSubFaces, &nFV, &subCells, subdm));
3595   /* Setup chart */
3596   PetscCall(DMLabelGetStratumSize(subpointMap, 0, &numSubVertices));
3597   PetscCall(DMLabelGetStratumSize(subpointMap, 2, &numSubCells));
3598   PetscCall(DMPlexSetChart(subdm, 0, numSubCells+numSubFaces+numSubVertices));
3599   PetscCall(DMPlexSetVTKCellHeight(subdm, 1));
3600   /* Set cone sizes */
3601   firstSubVertex = numSubCells;
3602   firstSubFace   = numSubCells+numSubVertices;
3603   newFacePoint   = firstSubFace;
3604   PetscCall(DMLabelGetStratumIS(subpointMap, 0, &subvertexIS));
3605   if (subvertexIS) PetscCall(ISGetIndices(subvertexIS, &subVertices));
3606   for (c = 0; c < numSubCells; ++c) {
3607     PetscCall(DMPlexSetConeSize(subdm, c, 1));
3608   }
3609   for (f = firstSubFace; f < firstSubFace+numSubFaces; ++f) {
3610     PetscCall(DMPlexSetConeSize(subdm, f, nFV));
3611   }
3612   PetscCall(DMSetUp(subdm));
3613   /* Create face cones */
3614   PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, NULL));
3615   PetscCall(DMGetWorkArray(subdm, maxConeSize, MPIU_INT, (void**) &subface));
3616   for (c = 0; c < numSubCells; ++c) {
3617     const PetscInt  cell    = subCells[c];
3618     const PetscInt  subcell = c;
3619     const PetscInt *cone, *cells;
3620     PetscBool       isHybrid;
3621     PetscInt        numCells, subVertex, p, v;
3622 
3623     PetscCall(DMPlexCellIsHybrid_Internal(dm, cell, &isHybrid));
3624     if (!isHybrid) continue;
3625     PetscCall(DMPlexGetCone(dm, cell, &cone));
3626     for (v = 0; v < nFV; ++v) {
3627       PetscCall(PetscFindInt(cone[v], numSubVertices, subVertices, &subVertex));
3628       subface[v] = firstSubVertex+subVertex;
3629     }
3630     PetscCall(DMPlexSetCone(subdm, newFacePoint, subface));
3631     PetscCall(DMPlexSetCone(subdm, subcell, &newFacePoint));
3632     PetscCall(DMPlexGetJoin(dm, nFV, cone, &numCells, &cells));
3633     /* Not true in parallel
3634     PetscCheck(numCells == 2,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive cells should separate two cells"); */
3635     for (p = 0; p < numCells; ++p) {
3636       PetscInt  negsubcell;
3637       PetscBool isHybrid;
3638 
3639       PetscCall(DMPlexCellIsHybrid_Internal(dm, cells[p], &isHybrid));
3640       if (isHybrid) continue;
3641       /* I know this is a crap search */
3642       for (negsubcell = 0; negsubcell < numSubCells; ++negsubcell) {
3643         if (subCells[negsubcell] == cells[p]) break;
3644       }
3645       PetscCheck(negsubcell != numSubCells,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not find negative face neighbor for cohesive cell %" PetscInt_FMT, cell);
3646       PetscCall(DMPlexSetCone(subdm, negsubcell, &newFacePoint));
3647     }
3648     PetscCall(DMPlexRestoreJoin(dm, nFV, cone, &numCells, &cells));
3649     ++newFacePoint;
3650   }
3651   PetscCall(DMRestoreWorkArray(subdm, maxConeSize, MPIU_INT, (void**) &subface));
3652   PetscCall(DMPlexSymmetrize(subdm));
3653   PetscCall(DMPlexStratify(subdm));
3654   /* Build coordinates */
3655   {
3656     PetscSection coordSection, subCoordSection;
3657     Vec          coordinates, subCoordinates;
3658     PetscScalar *coords, *subCoords;
3659     PetscInt     cdim, numComp, coordSize, v;
3660     const char  *name;
3661 
3662     PetscCall(DMGetCoordinateDim(dm, &cdim));
3663     PetscCall(DMGetCoordinateSection(dm, &coordSection));
3664     PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
3665     PetscCall(DMGetCoordinateSection(subdm, &subCoordSection));
3666     PetscCall(PetscSectionSetNumFields(subCoordSection, 1));
3667     PetscCall(PetscSectionGetFieldComponents(coordSection, 0, &numComp));
3668     PetscCall(PetscSectionSetFieldComponents(subCoordSection, 0, numComp));
3669     PetscCall(PetscSectionSetChart(subCoordSection, firstSubVertex, firstSubVertex+numSubVertices));
3670     for (v = 0; v < numSubVertices; ++v) {
3671       const PetscInt vertex    = subVertices[v];
3672       const PetscInt subvertex = firstSubVertex+v;
3673       PetscInt       dof;
3674 
3675       PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3676       PetscCall(PetscSectionSetDof(subCoordSection, subvertex, dof));
3677       PetscCall(PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof));
3678     }
3679     PetscCall(PetscSectionSetUp(subCoordSection));
3680     PetscCall(PetscSectionGetStorageSize(subCoordSection, &coordSize));
3681     PetscCall(VecCreate(PETSC_COMM_SELF, &subCoordinates));
3682     PetscCall(PetscObjectGetName((PetscObject)coordinates,&name));
3683     PetscCall(PetscObjectSetName((PetscObject)subCoordinates,name));
3684     PetscCall(VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE));
3685     PetscCall(VecSetBlockSize(subCoordinates, cdim));
3686     PetscCall(VecSetType(subCoordinates,VECSTANDARD));
3687     PetscCall(VecGetArray(coordinates,    &coords));
3688     PetscCall(VecGetArray(subCoordinates, &subCoords));
3689     for (v = 0; v < numSubVertices; ++v) {
3690       const PetscInt vertex    = subVertices[v];
3691       const PetscInt subvertex = firstSubVertex+v;
3692       PetscInt       dof, off, sdof, soff, d;
3693 
3694       PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3695       PetscCall(PetscSectionGetOffset(coordSection, vertex, &off));
3696       PetscCall(PetscSectionGetDof(subCoordSection, subvertex, &sdof));
3697       PetscCall(PetscSectionGetOffset(subCoordSection, subvertex, &soff));
3698       PetscCheck(dof == sdof,comm, PETSC_ERR_PLIB, "Coordinate dimension %" PetscInt_FMT " on subvertex %" PetscInt_FMT ", vertex %" PetscInt_FMT " should be %" PetscInt_FMT, sdof, subvertex, vertex, dof);
3699       for (d = 0; d < dof; ++d) subCoords[soff+d] = coords[off+d];
3700     }
3701     PetscCall(VecRestoreArray(coordinates,    &coords));
3702     PetscCall(VecRestoreArray(subCoordinates, &subCoords));
3703     PetscCall(DMSetCoordinatesLocal(subdm, subCoordinates));
3704     PetscCall(VecDestroy(&subCoordinates));
3705   }
3706   /* Build SF */
3707   CHKMEMQ;
3708   {
3709     PetscSF            sfPoint, sfPointSub;
3710     const PetscSFNode *remotePoints;
3711     PetscSFNode       *sremotePoints, *newLocalPoints, *newOwners;
3712     const PetscInt    *localPoints;
3713     PetscInt          *slocalPoints;
3714     PetscInt           numRoots, numLeaves, numSubRoots = numSubCells+numSubFaces+numSubVertices, numSubLeaves = 0, l, sl, ll, pStart, pEnd, p, vStart, vEnd;
3715     PetscMPIInt        rank;
3716 
3717     PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank));
3718     PetscCall(DMGetPointSF(dm, &sfPoint));
3719     PetscCall(DMGetPointSF(subdm, &sfPointSub));
3720     PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
3721     PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
3722     PetscCall(PetscSFGetGraph(sfPoint, &numRoots, &numLeaves, &localPoints, &remotePoints));
3723     if (numRoots >= 0) {
3724       /* Only vertices should be shared */
3725       PetscCall(PetscMalloc2(pEnd-pStart,&newLocalPoints,numRoots,&newOwners));
3726       for (p = 0; p < pEnd-pStart; ++p) {
3727         newLocalPoints[p].rank  = -2;
3728         newLocalPoints[p].index = -2;
3729       }
3730       /* Set subleaves */
3731       for (l = 0; l < numLeaves; ++l) {
3732         const PetscInt point    = localPoints[l];
3733         const PetscInt subPoint = DMPlexFilterPoint_Internal(point, firstSubVertex, numSubVertices, subVertices);
3734 
3735         PetscCheck(!(point < vStart) || !(point >= vEnd),PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Should not be mapping anything but vertices, %" PetscInt_FMT, point);
3736         if (subPoint < 0) continue;
3737         newLocalPoints[point-pStart].rank  = rank;
3738         newLocalPoints[point-pStart].index = subPoint;
3739         ++numSubLeaves;
3740       }
3741       /* Must put in owned subpoints */
3742       for (p = pStart; p < pEnd; ++p) {
3743         const PetscInt subPoint = DMPlexFilterPoint_Internal(p, firstSubVertex, numSubVertices, subVertices);
3744 
3745         if (subPoint < 0) {
3746           newOwners[p-pStart].rank  = -3;
3747           newOwners[p-pStart].index = -3;
3748         } else {
3749           newOwners[p-pStart].rank  = rank;
3750           newOwners[p-pStart].index = subPoint;
3751         }
3752       }
3753       PetscCall(PetscSFReduceBegin(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC));
3754       PetscCall(PetscSFReduceEnd(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC));
3755       PetscCall(PetscSFBcastBegin(sfPoint, MPIU_2INT, newOwners, newLocalPoints,MPI_REPLACE));
3756       PetscCall(PetscSFBcastEnd(sfPoint, MPIU_2INT, newOwners, newLocalPoints,MPI_REPLACE));
3757       PetscCall(PetscMalloc1(numSubLeaves,    &slocalPoints));
3758       PetscCall(PetscMalloc1(numSubLeaves, &sremotePoints));
3759       for (l = 0, sl = 0, ll = 0; l < numLeaves; ++l) {
3760         const PetscInt point    = localPoints[l];
3761         const PetscInt subPoint = DMPlexFilterPoint_Internal(point, firstSubVertex, numSubVertices, subVertices);
3762 
3763         if (subPoint < 0) continue;
3764         if (newLocalPoints[point].rank == rank) {++ll; continue;}
3765         slocalPoints[sl]        = subPoint;
3766         sremotePoints[sl].rank  = newLocalPoints[point].rank;
3767         sremotePoints[sl].index = newLocalPoints[point].index;
3768         PetscCheck(sremotePoints[sl].rank  >= 0,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote rank for local point %" PetscInt_FMT, point);
3769         PetscCheck(sremotePoints[sl].index >= 0,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote subpoint for local point %" PetscInt_FMT, point);
3770         ++sl;
3771       }
3772       PetscCall(PetscFree2(newLocalPoints,newOwners));
3773       PetscCheck(sl + ll == numSubLeaves,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatch in number of subleaves %" PetscInt_FMT " + %" PetscInt_FMT " != %" PetscInt_FMT, sl, ll, numSubLeaves);
3774       PetscCall(PetscSFSetGraph(sfPointSub, numSubRoots, sl, slocalPoints, PETSC_OWN_POINTER, sremotePoints, PETSC_OWN_POINTER));
3775     }
3776   }
3777   CHKMEMQ;
3778   /* Cleanup */
3779   if (subvertexIS) PetscCall(ISRestoreIndices(subvertexIS, &subVertices));
3780   PetscCall(ISDestroy(&subvertexIS));
3781   PetscCall(PetscFree(subCells));
3782   PetscFunctionReturn(0);
3783 }
3784 
3785 static PetscErrorCode DMPlexCreateCohesiveSubmesh_Interpolated(DM dm, const char labelname[], PetscInt value, DM subdm)
3786 {
3787   DMLabel        label = NULL;
3788 
3789   PetscFunctionBegin;
3790   if (labelname) PetscCall(DMGetLabel(dm, labelname, &label));
3791   PetscCall(DMPlexCreateSubmeshGeneric_Interpolated(dm, label, value, PETSC_FALSE, PETSC_TRUE, 1, subdm));
3792   PetscFunctionReturn(0);
3793 }
3794 
3795 /*@C
3796   DMPlexCreateCohesiveSubmesh - Extract from a mesh with cohesive cells the hypersurface defined by one face of the cells. Optionally, a Label can be given to restrict the cells.
3797 
3798   Input Parameters:
3799 + dm          - The original mesh
3800 . hasLagrange - The mesh has Lagrange unknowns in the cohesive cells
3801 . label       - A label name, or NULL
3802 - value  - A label value
3803 
3804   Output Parameter:
3805 . subdm - The surface mesh
3806 
3807   Note: This function produces a DMLabel mapping original points in the submesh to their depth. This can be obtained using DMPlexGetSubpointMap().
3808 
3809   Level: developer
3810 
3811 .seealso: `DMPlexGetSubpointMap()`, `DMPlexCreateSubmesh()`
3812 @*/
3813 PetscErrorCode DMPlexCreateCohesiveSubmesh(DM dm, PetscBool hasLagrange, const char label[], PetscInt value, DM *subdm)
3814 {
3815   PetscInt       dim, cdim, depth;
3816 
3817   PetscFunctionBegin;
3818   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3819   PetscValidPointer(subdm, 5);
3820   PetscCall(DMGetDimension(dm, &dim));
3821   PetscCall(DMPlexGetDepth(dm, &depth));
3822   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), subdm));
3823   PetscCall(DMSetType(*subdm, DMPLEX));
3824   PetscCall(DMSetDimension(*subdm, dim-1));
3825   PetscCall(DMGetCoordinateDim(dm, &cdim));
3826   PetscCall(DMSetCoordinateDim(*subdm, cdim));
3827   if (depth == dim) {
3828     PetscCall(DMPlexCreateCohesiveSubmesh_Interpolated(dm, label, value, *subdm));
3829   } else {
3830     PetscCall(DMPlexCreateCohesiveSubmesh_Uninterpolated(dm, hasLagrange, label, value, *subdm));
3831   }
3832   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, *subdm));
3833   PetscFunctionReturn(0);
3834 }
3835 
3836 /*@
3837   DMPlexFilter - Extract a subset of mesh cells defined by a label as a separate mesh
3838 
3839   Input Parameters:
3840 + dm        - The original mesh
3841 . cellLabel - The DMLabel marking cells contained in the new mesh
3842 - value     - The label value to use
3843 
3844   Output Parameter:
3845 . subdm - The new mesh
3846 
3847   Note: This function produces a DMLabel mapping original points in the submesh to their depth. This can be obtained using DMPlexGetSubpointMap().
3848 
3849   Level: developer
3850 
3851 .seealso: `DMPlexGetSubpointMap()`, `DMGetLabel()`, `DMLabelSetValue()`
3852 @*/
3853 PetscErrorCode DMPlexFilter(DM dm, DMLabel cellLabel, PetscInt value, DM *subdm)
3854 {
3855   PetscInt       dim;
3856 
3857   PetscFunctionBegin;
3858   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3859   PetscValidPointer(subdm, 4);
3860   PetscCall(DMGetDimension(dm, &dim));
3861   PetscCall(DMCreate(PetscObjectComm((PetscObject) dm), subdm));
3862   PetscCall(DMSetType(*subdm, DMPLEX));
3863   PetscCall(DMSetDimension(*subdm, dim));
3864   /* Extract submesh in place, could be empty on some procs, could have inconsistency if procs do not both extract a shared cell */
3865   PetscCall(DMPlexCreateSubmeshGeneric_Interpolated(dm, cellLabel, value, PETSC_FALSE, PETSC_FALSE, 0, *subdm));
3866   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, *subdm));
3867   PetscFunctionReturn(0);
3868 }
3869 
3870 /*@
3871   DMPlexGetSubpointMap - Returns a DMLabel with point dimension as values
3872 
3873   Input Parameter:
3874 . dm - The submesh DM
3875 
3876   Output Parameter:
3877 . subpointMap - The DMLabel of all the points from the original mesh in this submesh, or NULL if this is not a submesh
3878 
3879   Level: developer
3880 
3881 .seealso: `DMPlexCreateSubmesh()`, `DMPlexGetSubpointIS()`
3882 @*/
3883 PetscErrorCode DMPlexGetSubpointMap(DM dm, DMLabel *subpointMap)
3884 {
3885   PetscFunctionBegin;
3886   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3887   PetscValidPointer(subpointMap, 2);
3888   *subpointMap = ((DM_Plex*) dm->data)->subpointMap;
3889   PetscFunctionReturn(0);
3890 }
3891 
3892 /*@
3893   DMPlexSetSubpointMap - Sets the DMLabel with point dimension as values
3894 
3895   Input Parameters:
3896 + dm - The submesh DM
3897 - subpointMap - The DMLabel of all the points from the original mesh in this submesh
3898 
3899   Note: Should normally not be called by the user, since it is set in DMPlexCreateSubmesh()
3900 
3901   Level: developer
3902 
3903 .seealso: `DMPlexCreateSubmesh()`, `DMPlexGetSubpointIS()`
3904 @*/
3905 PetscErrorCode DMPlexSetSubpointMap(DM dm, DMLabel subpointMap)
3906 {
3907   DM_Plex       *mesh = (DM_Plex *) dm->data;
3908   DMLabel        tmp;
3909 
3910   PetscFunctionBegin;
3911   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3912   tmp  = mesh->subpointMap;
3913   mesh->subpointMap = subpointMap;
3914   PetscCall(PetscObjectReference((PetscObject) mesh->subpointMap));
3915   PetscCall(DMLabelDestroy(&tmp));
3916   PetscFunctionReturn(0);
3917 }
3918 
3919 static PetscErrorCode DMPlexCreateSubpointIS_Internal(DM dm, IS *subpointIS)
3920 {
3921   DMLabel        spmap;
3922   PetscInt       depth, d;
3923 
3924   PetscFunctionBegin;
3925   PetscCall(DMPlexGetSubpointMap(dm, &spmap));
3926   PetscCall(DMPlexGetDepth(dm, &depth));
3927   if (spmap && depth >= 0) {
3928     DM_Plex  *mesh = (DM_Plex *) dm->data;
3929     PetscInt *points, *depths;
3930     PetscInt  pStart, pEnd, p, off;
3931 
3932     PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
3933     PetscCheck(!pStart,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Submeshes must start the point numbering at 0, not %" PetscInt_FMT, pStart);
3934     PetscCall(PetscMalloc1(pEnd, &points));
3935     PetscCall(DMGetWorkArray(dm, depth+1, MPIU_INT, &depths));
3936     depths[0] = depth;
3937     depths[1] = 0;
3938     for (d = 2; d <= depth; ++d) {depths[d] = depth+1 - d;}
3939     for (d = 0, off = 0; d <= depth; ++d) {
3940       const PetscInt dep = depths[d];
3941       PetscInt       depStart, depEnd, n;
3942 
3943       PetscCall(DMPlexGetDepthStratum(dm, dep, &depStart, &depEnd));
3944       PetscCall(DMLabelGetStratumSize(spmap, dep, &n));
3945       if (((d < 2) && (depth > 1)) || (d == 1)) { /* Only check vertices and cells for now since the map is broken for others */
3946         PetscCheck(n == depEnd-depStart,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "The number of mapped submesh points %" PetscInt_FMT " at depth %" PetscInt_FMT " should be %" PetscInt_FMT, n, dep, depEnd-depStart);
3947       } else {
3948         if (!n) {
3949           if (d == 0) {
3950             /* Missing cells */
3951             for (p = 0; p < depEnd-depStart; ++p, ++off) points[off] = -1;
3952           } else {
3953             /* Missing faces */
3954             for (p = 0; p < depEnd-depStart; ++p, ++off) points[off] = PETSC_MAX_INT;
3955           }
3956         }
3957       }
3958       if (n) {
3959         IS              is;
3960         const PetscInt *opoints;
3961 
3962         PetscCall(DMLabelGetStratumIS(spmap, dep, &is));
3963         PetscCall(ISGetIndices(is, &opoints));
3964         for (p = 0; p < n; ++p, ++off) points[off] = opoints[p];
3965         PetscCall(ISRestoreIndices(is, &opoints));
3966         PetscCall(ISDestroy(&is));
3967       }
3968     }
3969     PetscCall(DMRestoreWorkArray(dm, depth+1, MPIU_INT, &depths));
3970     PetscCheck(off == pEnd,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "The number of mapped submesh points %" PetscInt_FMT " should be %" PetscInt_FMT, off, pEnd);
3971     PetscCall(ISCreateGeneral(PETSC_COMM_SELF, pEnd, points, PETSC_OWN_POINTER, subpointIS));
3972     PetscCall(PetscObjectStateGet((PetscObject) spmap, &mesh->subpointState));
3973   }
3974   PetscFunctionReturn(0);
3975 }
3976 
3977 /*@
3978   DMPlexGetSubpointIS - Returns an IS covering the entire subdm chart with the original points as data
3979 
3980   Input Parameter:
3981 . dm - The submesh DM
3982 
3983   Output Parameter:
3984 . subpointIS - The IS of all the points from the original mesh in this submesh, or NULL if this is not a submesh
3985 
3986   Note: This IS is guaranteed to be sorted by the construction of the submesh
3987 
3988   Level: developer
3989 
3990 .seealso: `DMPlexCreateSubmesh()`, `DMPlexGetSubpointMap()`
3991 @*/
3992 PetscErrorCode DMPlexGetSubpointIS(DM dm, IS *subpointIS)
3993 {
3994   DM_Plex         *mesh = (DM_Plex *) dm->data;
3995   DMLabel          spmap;
3996   PetscObjectState state;
3997 
3998   PetscFunctionBegin;
3999   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4000   PetscValidPointer(subpointIS, 2);
4001   PetscCall(DMPlexGetSubpointMap(dm, &spmap));
4002   PetscCall(PetscObjectStateGet((PetscObject) spmap, &state));
4003   if (state != mesh->subpointState || !mesh->subpointIS) PetscCall(DMPlexCreateSubpointIS_Internal(dm, &mesh->subpointIS));
4004   *subpointIS = mesh->subpointIS;
4005   PetscFunctionReturn(0);
4006 }
4007 
4008 /*@
4009   DMGetEnclosureRelation - Get the relationship between dmA and dmB
4010 
4011   Input Parameters:
4012 + dmA - The first DM
4013 - dmB - The second DM
4014 
4015   Output Parameter:
4016 . rel - The relation of dmA to dmB
4017 
4018   Level: intermediate
4019 
4020 .seealso: `DMGetEnclosurePoint()`
4021 @*/
4022 PetscErrorCode DMGetEnclosureRelation(DM dmA, DM dmB, DMEnclosureType *rel)
4023 {
4024   DM             plexA, plexB, sdm;
4025   DMLabel        spmap;
4026   PetscInt       pStartA, pEndA, pStartB, pEndB, NpA, NpB;
4027 
4028   PetscFunctionBegin;
4029   PetscValidPointer(rel, 3);
4030   *rel = DM_ENC_NONE;
4031   if (!dmA || !dmB) PetscFunctionReturn(0);
4032   PetscValidHeaderSpecific(dmA, DM_CLASSID, 1);
4033   PetscValidHeaderSpecific(dmB, DM_CLASSID, 2);
4034   if (dmA == dmB) {*rel = DM_ENC_EQUALITY; PetscFunctionReturn(0);}
4035   PetscCall(DMConvert(dmA, DMPLEX, &plexA));
4036   PetscCall(DMConvert(dmB, DMPLEX, &plexB));
4037   PetscCall(DMPlexGetChart(plexA, &pStartA, &pEndA));
4038   PetscCall(DMPlexGetChart(plexB, &pStartB, &pEndB));
4039   /* Assumption 1: subDMs have smaller charts than the DMs that they originate from
4040     - The degenerate case of a subdomain which includes all of the domain on some process can be treated as equality */
4041   if ((pStartA == pStartB) && (pEndA == pEndB)) {
4042     *rel = DM_ENC_EQUALITY;
4043     goto end;
4044   }
4045   NpA = pEndA - pStartA;
4046   NpB = pEndB - pStartB;
4047   if (NpA == NpB) goto end;
4048   sdm = NpA > NpB ? plexB : plexA; /* The other is the original, enclosing dm */
4049   PetscCall(DMPlexGetSubpointMap(sdm, &spmap));
4050   if (!spmap) goto end;
4051   /* TODO Check the space mapped to by subpointMap is same size as dm */
4052   if (NpA > NpB) {
4053     *rel = DM_ENC_SUPERMESH;
4054   } else {
4055     *rel = DM_ENC_SUBMESH;
4056   }
4057   end:
4058   PetscCall(DMDestroy(&plexA));
4059   PetscCall(DMDestroy(&plexB));
4060   PetscFunctionReturn(0);
4061 }
4062 
4063 /*@
4064   DMGetEnclosurePoint - Get the point pA in dmA which corresponds to the point pB in dmB
4065 
4066   Input Parameters:
4067 + dmA   - The first DM
4068 . dmB   - The second DM
4069 . etype - The type of enclosure relation that dmA has to dmB
4070 - pB    - A point of dmB
4071 
4072   Output Parameter:
4073 . pA    - The corresponding point of dmA
4074 
4075   Level: intermediate
4076 
4077 .seealso: `DMGetEnclosureRelation()`
4078 @*/
4079 PetscErrorCode DMGetEnclosurePoint(DM dmA, DM dmB, DMEnclosureType etype, PetscInt pB, PetscInt *pA)
4080 {
4081   DM              sdm;
4082   IS              subpointIS;
4083   const PetscInt *subpoints;
4084   PetscInt        numSubpoints;
4085 
4086   PetscFunctionBegin;
4087   /* TODO Cache the IS, making it look like an index */
4088   switch (etype) {
4089     case DM_ENC_SUPERMESH:
4090     sdm  = dmB;
4091     PetscCall(DMPlexGetSubpointIS(sdm, &subpointIS));
4092     PetscCall(ISGetIndices(subpointIS, &subpoints));
4093     *pA  = subpoints[pB];
4094     PetscCall(ISRestoreIndices(subpointIS, &subpoints));
4095     break;
4096     case DM_ENC_SUBMESH:
4097     sdm  = dmA;
4098     PetscCall(DMPlexGetSubpointIS(sdm, &subpointIS));
4099     PetscCall(ISGetLocalSize(subpointIS, &numSubpoints));
4100     PetscCall(ISGetIndices(subpointIS, &subpoints));
4101     PetscCall(PetscFindInt(pB, numSubpoints, subpoints, pA));
4102     if (*pA < 0) {
4103       PetscCall(DMViewFromOptions(dmA, NULL, "-dm_enc_A_view"));
4104       PetscCall(DMViewFromOptions(dmB, NULL, "-dm_enc_B_view"));
4105       SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Point %" PetscInt_FMT " not found in submesh", pB);
4106     }
4107     PetscCall(ISRestoreIndices(subpointIS, &subpoints));
4108     break;
4109     case DM_ENC_EQUALITY:
4110     case DM_ENC_NONE:
4111     *pA = pB;break;
4112     case DM_ENC_UNKNOWN:
4113     {
4114       DMEnclosureType enc;
4115 
4116       PetscCall(DMGetEnclosureRelation(dmA, dmB, &enc));
4117       PetscCall(DMGetEnclosurePoint(dmA, dmB, enc, pB, pA));
4118     }
4119     break;
4120     default: SETERRQ(PetscObjectComm((PetscObject) dmA), PETSC_ERR_ARG_OUTOFRANGE, "Invalid enclosure type %d", (int) etype);
4121   }
4122   PetscFunctionReturn(0);
4123 }
4124