xref: /petsc/src/dm/impls/plex/plexsubmesh.c (revision 76be6f4ff3bd4e251c19fc00ebbebfd58b6e7589)
1 #include <petsc/private/dmpleximpl.h>    /*I      "petscdmplex.h"    I*/
2 #include <petsc/private/dmlabelimpl.h>   /*I      "petscdmlabel.h"   I*/
3 #include <petscsf.h>
4 
5 static PetscErrorCode DMPlexCellIsHybrid_Internal(DM dm, PetscInt p, PetscBool *isHybrid)
6 {
7   DMPolytopeType ct;
8 
9   PetscFunctionBegin;
10   PetscCall(DMPlexGetCellType(dm, p, &ct));
11   switch (ct) {
12     case DM_POLYTOPE_POINT_PRISM_TENSOR:
13     case DM_POLYTOPE_SEG_PRISM_TENSOR:
14     case DM_POLYTOPE_TRI_PRISM_TENSOR:
15     case DM_POLYTOPE_QUAD_PRISM_TENSOR:
16       *isHybrid = PETSC_TRUE;
17     default: *isHybrid = PETSC_FALSE;
18   }
19   PetscFunctionReturn(0);
20 }
21 
22 static PetscErrorCode DMPlexGetTensorPrismBounds_Internal(DM dm, PetscInt dim, PetscInt *cStart, PetscInt *cEnd)
23 {
24   DMLabel        ctLabel;
25 
26   PetscFunctionBegin;
27   if (cStart) *cStart = -1;
28   if (cEnd)   *cEnd   = -1;
29   PetscCall(DMPlexGetCellTypeLabel(dm, &ctLabel));
30   switch (dim) {
31     case 1: PetscCall(DMLabelGetStratumBounds(ctLabel, DM_POLYTOPE_POINT_PRISM_TENSOR, cStart, cEnd));break;
32     case 2: PetscCall(DMLabelGetStratumBounds(ctLabel, DM_POLYTOPE_SEG_PRISM_TENSOR, cStart, cEnd));break;
33     case 3:
34       PetscCall(DMLabelGetStratumBounds(ctLabel, DM_POLYTOPE_TRI_PRISM_TENSOR, cStart, cEnd));
35       if (*cStart < 0) PetscCall(DMLabelGetStratumBounds(ctLabel, DM_POLYTOPE_QUAD_PRISM_TENSOR, cStart, cEnd));
36       break;
37     default: PetscFunctionReturn(0);
38   }
39   PetscFunctionReturn(0);
40 }
41 
42 static PetscErrorCode DMPlexMarkBoundaryFaces_Internal(DM dm, PetscInt val, PetscInt cellHeight, DMLabel label)
43 {
44   PetscInt       fStart, fEnd, f;
45 
46   PetscFunctionBegin;
47   PetscCall(DMPlexGetHeightStratum(dm, cellHeight+1, &fStart, &fEnd));
48   for (f = fStart; f < fEnd; ++f) {
49     PetscInt supportSize;
50 
51     PetscCall(DMPlexGetSupportSize(dm, f, &supportSize));
52     if (supportSize == 1) {
53       if (val < 0) {
54         PetscInt *closure = NULL;
55         PetscInt  clSize, cl, cval;
56 
57         PetscCall(DMPlexGetTransitiveClosure(dm, f, PETSC_TRUE, &clSize, &closure));
58         for (cl = 0; cl < clSize*2; cl += 2) {
59           PetscCall(DMLabelGetValue(label, closure[cl], &cval));
60           if (cval < 0) continue;
61           PetscCall(DMLabelSetValue(label, f, cval));
62           break;
63         }
64         if (cl == clSize*2) PetscCall(DMLabelSetValue(label, f, 1));
65         PetscCall(DMPlexRestoreTransitiveClosure(dm, f, PETSC_TRUE, &clSize, &closure));
66       } else {
67         PetscCall(DMLabelSetValue(label, f, val));
68       }
69     }
70   }
71   PetscFunctionReturn(0);
72 }
73 
74 /*@
75   DMPlexMarkBoundaryFaces - Mark all faces on the boundary
76 
77   Not Collective
78 
79   Input Parameters:
80 + dm - The original DM
81 - val - The marker value, or PETSC_DETERMINE to use some value in the closure (or 1 if none are found)
82 
83   Output Parameter:
84 . label - The DMLabel marking boundary faces with the given value
85 
86   Level: developer
87 
88 .seealso: `DMLabelCreate()`, `DMCreateLabel()`
89 @*/
90 PetscErrorCode DMPlexMarkBoundaryFaces(DM dm, PetscInt val, DMLabel label)
91 {
92   DMPlexInterpolatedFlag  flg;
93 
94   PetscFunctionBegin;
95   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
96   PetscCall(DMPlexIsInterpolated(dm, &flg));
97   PetscCheck(flg == DMPLEX_INTERPOLATED_FULL,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM is not fully interpolated on this rank");
98   PetscCall(DMPlexMarkBoundaryFaces_Internal(dm, val, 0, label));
99   PetscFunctionReturn(0);
100 }
101 
102 static PetscErrorCode DMPlexLabelComplete_Internal(DM dm, DMLabel label, PetscBool completeCells)
103 {
104   IS              valueIS;
105   PetscSF         sfPoint;
106   const PetscInt *values;
107   PetscInt        numValues, v, cStart, cEnd, nroots;
108 
109   PetscFunctionBegin;
110   PetscCall(DMLabelGetNumValues(label, &numValues));
111   PetscCall(DMLabelGetValueIS(label, &valueIS));
112   PetscCall(DMPlexGetHeightStratum(dm,0,&cStart,&cEnd));
113   PetscCall(ISGetIndices(valueIS, &values));
114   for (v = 0; v < numValues; ++v) {
115     IS              pointIS;
116     const PetscInt *points;
117     PetscInt        numPoints, p;
118 
119     PetscCall(DMLabelGetStratumSize(label, values[v], &numPoints));
120     PetscCall(DMLabelGetStratumIS(label, values[v], &pointIS));
121     PetscCall(ISGetIndices(pointIS, &points));
122     for (p = 0; p < numPoints; ++p) {
123       PetscInt  q = points[p];
124       PetscInt *closure = NULL;
125       PetscInt  closureSize, c;
126 
127       if (cStart <= q && q < cEnd && !completeCells) { /* skip cells */
128         continue;
129       }
130       PetscCall(DMPlexGetTransitiveClosure(dm, q, PETSC_TRUE, &closureSize, &closure));
131       for (c = 0; c < closureSize*2; c += 2) {
132         PetscCall(DMLabelSetValue(label, closure[c], values[v]));
133       }
134       PetscCall(DMPlexRestoreTransitiveClosure(dm, q, PETSC_TRUE, &closureSize, &closure));
135     }
136     PetscCall(ISRestoreIndices(pointIS, &points));
137     PetscCall(ISDestroy(&pointIS));
138   }
139   PetscCall(ISRestoreIndices(valueIS, &values));
140   PetscCall(ISDestroy(&valueIS));
141   PetscCall(DMGetPointSF(dm, &sfPoint));
142   PetscCall(PetscSFGetGraph(sfPoint, &nroots, NULL, NULL, NULL));
143   if (nroots >= 0) {
144     DMLabel         lblRoots, lblLeaves;
145     IS              valueIS, pointIS;
146     const PetscInt *values;
147     PetscInt        numValues, v;
148 
149     /* Pull point contributions from remote leaves into local roots */
150     PetscCall(DMLabelGather(label, sfPoint, &lblLeaves));
151     PetscCall(DMLabelGetValueIS(lblLeaves, &valueIS));
152     PetscCall(ISGetLocalSize(valueIS, &numValues));
153     PetscCall(ISGetIndices(valueIS, &values));
154     for (v = 0; v < numValues; ++v) {
155       const PetscInt value = values[v];
156 
157       PetscCall(DMLabelGetStratumIS(lblLeaves, value, &pointIS));
158       PetscCall(DMLabelInsertIS(label, pointIS, value));
159       PetscCall(ISDestroy(&pointIS));
160     }
161     PetscCall(ISRestoreIndices(valueIS, &values));
162     PetscCall(ISDestroy(&valueIS));
163     PetscCall(DMLabelDestroy(&lblLeaves));
164     /* Push point contributions from roots into remote leaves */
165     PetscCall(DMLabelDistribute(label, sfPoint, &lblRoots));
166     PetscCall(DMLabelGetValueIS(lblRoots, &valueIS));
167     PetscCall(ISGetLocalSize(valueIS, &numValues));
168     PetscCall(ISGetIndices(valueIS, &values));
169     for (v = 0; v < numValues; ++v) {
170       const PetscInt value = values[v];
171 
172       PetscCall(DMLabelGetStratumIS(lblRoots, value, &pointIS));
173       PetscCall(DMLabelInsertIS(label, pointIS, value));
174       PetscCall(ISDestroy(&pointIS));
175     }
176     PetscCall(ISRestoreIndices(valueIS, &values));
177     PetscCall(ISDestroy(&valueIS));
178     PetscCall(DMLabelDestroy(&lblRoots));
179   }
180   PetscFunctionReturn(0);
181 }
182 
183 /*@
184   DMPlexLabelComplete - Starting with a label marking points on a surface, we add the transitive closure to the surface
185 
186   Input Parameters:
187 + dm - The DM
188 - label - A DMLabel marking the surface points
189 
190   Output Parameter:
191 . label - A DMLabel marking all surface points in the transitive closure
192 
193   Level: developer
194 
195 .seealso: `DMPlexLabelCohesiveComplete()`
196 @*/
197 PetscErrorCode DMPlexLabelComplete(DM dm, DMLabel label)
198 {
199   PetscFunctionBegin;
200   PetscCall(DMPlexLabelComplete_Internal(dm, label, PETSC_TRUE));
201   PetscFunctionReturn(0);
202 }
203 
204 /*@
205   DMPlexLabelAddCells - Starting with a label marking points on a surface, we add a cell for each point
206 
207   Input Parameters:
208 + dm - The DM
209 - label - A DMLabel marking the surface points
210 
211   Output Parameter:
212 . label - A DMLabel incorporating cells
213 
214   Level: developer
215 
216   Note: The cells allow FEM boundary conditions to be applied using the cell geometry
217 
218 .seealso: `DMPlexLabelAddFaceCells()`, `DMPlexLabelComplete()`, `DMPlexLabelCohesiveComplete()`
219 @*/
220 PetscErrorCode DMPlexLabelAddCells(DM dm, DMLabel label)
221 {
222   IS              valueIS;
223   const PetscInt *values;
224   PetscInt        numValues, v, cStart, cEnd;
225 
226   PetscFunctionBegin;
227   PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd));
228   PetscCall(DMLabelGetNumValues(label, &numValues));
229   PetscCall(DMLabelGetValueIS(label, &valueIS));
230   PetscCall(ISGetIndices(valueIS, &values));
231   for (v = 0; v < numValues; ++v) {
232     IS              pointIS;
233     const PetscInt *points;
234     PetscInt        numPoints, p;
235 
236     PetscCall(DMLabelGetStratumSize(label, values[v], &numPoints));
237     PetscCall(DMLabelGetStratumIS(label, values[v], &pointIS));
238     PetscCall(ISGetIndices(pointIS, &points));
239     for (p = 0; p < numPoints; ++p) {
240       PetscInt *closure = NULL;
241       PetscInt  closureSize, cl;
242 
243       PetscCall(DMPlexGetTransitiveClosure(dm, points[p], PETSC_FALSE, &closureSize, &closure));
244       for (cl = closureSize-1; cl > 0; --cl) {
245         const PetscInt cell = closure[cl*2];
246         if ((cell >= cStart) && (cell < cEnd)) {PetscCall(DMLabelSetValue(label, cell, values[v])); break;}
247       }
248       PetscCall(DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_FALSE, &closureSize, &closure));
249     }
250     PetscCall(ISRestoreIndices(pointIS, &points));
251     PetscCall(ISDestroy(&pointIS));
252   }
253   PetscCall(ISRestoreIndices(valueIS, &values));
254   PetscCall(ISDestroy(&valueIS));
255   PetscFunctionReturn(0);
256 }
257 
258 /*@
259   DMPlexLabelAddFaceCells - Starting with a label marking faces on a surface, we add a cell for each face
260 
261   Input Parameters:
262 + dm - The DM
263 - label - A DMLabel marking the surface points
264 
265   Output Parameter:
266 . label - A DMLabel incorporating cells
267 
268   Level: developer
269 
270   Note: The cells allow FEM boundary conditions to be applied using the cell geometry
271 
272 .seealso: `DMPlexLabelAddCells()`, `DMPlexLabelComplete()`, `DMPlexLabelCohesiveComplete()`
273 @*/
274 PetscErrorCode DMPlexLabelAddFaceCells(DM dm, DMLabel label)
275 {
276   IS              valueIS;
277   const PetscInt *values;
278   PetscInt        numValues, v, cStart, cEnd, fStart, fEnd;
279 
280   PetscFunctionBegin;
281   PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd));
282   PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
283   PetscCall(DMLabelGetNumValues(label, &numValues));
284   PetscCall(DMLabelGetValueIS(label, &valueIS));
285   PetscCall(ISGetIndices(valueIS, &values));
286   for (v = 0; v < numValues; ++v) {
287     IS              pointIS;
288     const PetscInt *points;
289     PetscInt        numPoints, p;
290 
291     PetscCall(DMLabelGetStratumSize(label, values[v], &numPoints));
292     PetscCall(DMLabelGetStratumIS(label, values[v], &pointIS));
293     PetscCall(ISGetIndices(pointIS, &points));
294     for (p = 0; p < numPoints; ++p) {
295       const PetscInt face = points[p];
296       PetscInt      *closure = NULL;
297       PetscInt       closureSize, cl;
298 
299       if ((face < fStart) || (face >= fEnd)) continue;
300       PetscCall(DMPlexGetTransitiveClosure(dm, face, PETSC_FALSE, &closureSize, &closure));
301       for (cl = closureSize-1; cl > 0; --cl) {
302         const PetscInt cell = closure[cl*2];
303         if ((cell >= cStart) && (cell < cEnd)) {PetscCall(DMLabelSetValue(label, cell, values[v])); break;}
304       }
305       PetscCall(DMPlexRestoreTransitiveClosure(dm, face, PETSC_FALSE, &closureSize, &closure));
306     }
307     PetscCall(ISRestoreIndices(pointIS, &points));
308     PetscCall(ISDestroy(&pointIS));
309   }
310   PetscCall(ISRestoreIndices(valueIS, &values));
311   PetscCall(ISDestroy(&valueIS));
312   PetscFunctionReturn(0);
313 }
314 
315 /*@
316   DMPlexLabelClearCells - Remove cells from a label
317 
318   Input Parameters:
319 + dm - The DM
320 - label - A DMLabel marking surface points and their adjacent cells
321 
322   Output Parameter:
323 . label - A DMLabel without cells
324 
325   Level: developer
326 
327   Note: This undoes DMPlexLabelAddCells() or DMPlexLabelAddFaceCells()
328 
329 .seealso: `DMPlexLabelComplete()`, `DMPlexLabelCohesiveComplete()`, `DMPlexLabelAddCells()`
330 @*/
331 PetscErrorCode DMPlexLabelClearCells(DM dm, DMLabel label)
332 {
333   IS              valueIS;
334   const PetscInt *values;
335   PetscInt        numValues, v, cStart, cEnd;
336 
337   PetscFunctionBegin;
338   PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd));
339   PetscCall(DMLabelGetNumValues(label, &numValues));
340   PetscCall(DMLabelGetValueIS(label, &valueIS));
341   PetscCall(ISGetIndices(valueIS, &values));
342   for (v = 0; v < numValues; ++v) {
343     IS              pointIS;
344     const PetscInt *points;
345     PetscInt        numPoints, p;
346 
347     PetscCall(DMLabelGetStratumSize(label, values[v], &numPoints));
348     PetscCall(DMLabelGetStratumIS(label, values[v], &pointIS));
349     PetscCall(ISGetIndices(pointIS, &points));
350     for (p = 0; p < numPoints; ++p) {
351       PetscInt point = points[p];
352 
353       if (point >= cStart && point < cEnd) {
354         PetscCall(DMLabelClearValue(label,point,values[v]));
355       }
356     }
357     PetscCall(ISRestoreIndices(pointIS, &points));
358     PetscCall(ISDestroy(&pointIS));
359   }
360   PetscCall(ISRestoreIndices(valueIS, &values));
361   PetscCall(ISDestroy(&valueIS));
362   PetscFunctionReturn(0);
363 }
364 
365 /* take (oldEnd, added) pairs, ordered by height and convert them to (oldstart, newstart) pairs, ordered by ascending
366  * index (skipping first, which is (0,0)) */
367 static inline PetscErrorCode DMPlexShiftPointSetUp_Internal(PetscInt depth, PetscInt depthShift[])
368 {
369   PetscInt d, off = 0;
370 
371   PetscFunctionBegin;
372   /* sort by (oldend): yes this is an O(n^2) sort, we expect depth <= 3 */
373   for (d = 0; d < depth; d++) {
374     PetscInt firstd = d;
375     PetscInt firstStart = depthShift[2*d];
376     PetscInt e;
377 
378     for (e = d+1; e <= depth; e++) {
379       if (depthShift[2*e] < firstStart) {
380         firstd = e;
381         firstStart = depthShift[2*d];
382       }
383     }
384     if (firstd != d) {
385       PetscInt swap[2];
386 
387       e = firstd;
388       swap[0] = depthShift[2*d];
389       swap[1] = depthShift[2*d+1];
390       depthShift[2*d]   = depthShift[2*e];
391       depthShift[2*d+1] = depthShift[2*e+1];
392       depthShift[2*e]   = swap[0];
393       depthShift[2*e+1] = swap[1];
394     }
395   }
396   /* convert (oldstart, added) to (oldstart, newstart) */
397   for (d = 0; d <= depth; d++) {
398     off += depthShift[2*d+1];
399     depthShift[2*d+1] = depthShift[2*d] + off;
400   }
401   PetscFunctionReturn(0);
402 }
403 
404 /* depthShift is a list of (old, new) pairs */
405 static inline PetscInt DMPlexShiftPoint_Internal(PetscInt p, PetscInt depth, PetscInt depthShift[])
406 {
407   PetscInt d;
408   PetscInt newOff = 0;
409 
410   for (d = 0; d <= depth; d++) {
411     if (p < depthShift[2*d]) return p + newOff;
412     else newOff = depthShift[2*d+1] - depthShift[2*d];
413   }
414   return p + newOff;
415 }
416 
417 /* depthShift is a list of (old, new) pairs */
418 static inline PetscInt DMPlexShiftPointInverse_Internal(PetscInt p, PetscInt depth, PetscInt depthShift[])
419 {
420   PetscInt d;
421   PetscInt newOff = 0;
422 
423   for (d = 0; d <= depth; d++) {
424     if (p < depthShift[2*d+1]) return p + newOff;
425     else newOff = depthShift[2*d] - depthShift[2*d+1];
426   }
427   return p + newOff;
428 }
429 
430 static PetscErrorCode DMPlexShiftSizes_Internal(DM dm, PetscInt depthShift[], DM dmNew)
431 {
432   PetscInt       depth = 0, d, pStart, pEnd, p;
433   DMLabel        depthLabel;
434 
435   PetscFunctionBegin;
436   PetscCall(DMPlexGetDepth(dm, &depth));
437   if (depth < 0) PetscFunctionReturn(0);
438   /* Step 1: Expand chart */
439   PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
440   pEnd = DMPlexShiftPoint_Internal(pEnd,depth,depthShift);
441   PetscCall(DMPlexSetChart(dmNew, pStart, pEnd));
442   PetscCall(DMCreateLabel(dmNew,"depth"));
443   PetscCall(DMPlexGetDepthLabel(dmNew,&depthLabel));
444   PetscCall(DMCreateLabel(dmNew, "celltype"));
445   /* Step 2: Set cone and support sizes */
446   for (d = 0; d <= depth; ++d) {
447     PetscInt pStartNew, pEndNew;
448     IS pIS;
449 
450     PetscCall(DMPlexGetDepthStratum(dm, d, &pStart, &pEnd));
451     pStartNew = DMPlexShiftPoint_Internal(pStart, depth, depthShift);
452     pEndNew = DMPlexShiftPoint_Internal(pEnd, depth, depthShift);
453     PetscCall(ISCreateStride(PETSC_COMM_SELF, pEndNew - pStartNew, pStartNew, 1, &pIS));
454     PetscCall(DMLabelSetStratumIS(depthLabel, d, pIS));
455     PetscCall(ISDestroy(&pIS));
456     for (p = pStart; p < pEnd; ++p) {
457       PetscInt       newp = DMPlexShiftPoint_Internal(p, depth, depthShift);
458       PetscInt       size;
459       DMPolytopeType ct;
460 
461       PetscCall(DMPlexGetConeSize(dm, p, &size));
462       PetscCall(DMPlexSetConeSize(dmNew, newp, size));
463       PetscCall(DMPlexGetSupportSize(dm, p, &size));
464       PetscCall(DMPlexSetSupportSize(dmNew, newp, size));
465       PetscCall(DMPlexGetCellType(dm, p, &ct));
466       PetscCall(DMPlexSetCellType(dmNew, newp, ct));
467     }
468   }
469   PetscFunctionReturn(0);
470 }
471 
472 static PetscErrorCode DMPlexShiftPoints_Internal(DM dm, PetscInt depthShift[], DM dmNew)
473 {
474   PetscInt      *newpoints;
475   PetscInt       depth = 0, maxConeSize, maxSupportSize, maxConeSizeNew, maxSupportSizeNew, pStart, pEnd, p;
476 
477   PetscFunctionBegin;
478   PetscCall(DMPlexGetDepth(dm, &depth));
479   if (depth < 0) PetscFunctionReturn(0);
480   PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize));
481   PetscCall(DMPlexGetMaxSizes(dmNew, &maxConeSizeNew, &maxSupportSizeNew));
482   PetscCall(PetscMalloc1(PetscMax(PetscMax(maxConeSize, maxSupportSize), PetscMax(maxConeSizeNew, maxSupportSizeNew)),&newpoints));
483   /* Step 5: Set cones and supports */
484   PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
485   for (p = pStart; p < pEnd; ++p) {
486     const PetscInt *points = NULL, *orientations = NULL;
487     PetscInt        size,sizeNew, i, newp = DMPlexShiftPoint_Internal(p, depth, depthShift);
488 
489     PetscCall(DMPlexGetConeSize(dm, p, &size));
490     PetscCall(DMPlexGetCone(dm, p, &points));
491     PetscCall(DMPlexGetConeOrientation(dm, p, &orientations));
492     for (i = 0; i < size; ++i) {
493       newpoints[i] = DMPlexShiftPoint_Internal(points[i], depth, depthShift);
494     }
495     PetscCall(DMPlexSetCone(dmNew, newp, newpoints));
496     PetscCall(DMPlexSetConeOrientation(dmNew, newp, orientations));
497     PetscCall(DMPlexGetSupportSize(dm, p, &size));
498     PetscCall(DMPlexGetSupportSize(dmNew, newp, &sizeNew));
499     PetscCall(DMPlexGetSupport(dm, p, &points));
500     for (i = 0; i < size; ++i) {
501       newpoints[i] = DMPlexShiftPoint_Internal(points[i], depth, depthShift);
502     }
503     for (i = size; i < sizeNew; ++i) newpoints[i] = 0;
504     PetscCall(DMPlexSetSupport(dmNew, newp, newpoints));
505   }
506   PetscCall(PetscFree(newpoints));
507   PetscFunctionReturn(0);
508 }
509 
510 static PetscErrorCode DMPlexShiftCoordinates_Internal(DM dm, PetscInt depthShift[], DM dmNew)
511 {
512   PetscSection   coordSection, newCoordSection;
513   Vec            coordinates, newCoordinates;
514   PetscScalar   *coords, *newCoords;
515   PetscInt       coordSize, sStart, sEnd;
516   PetscInt       dim, depth = 0, cStart, cEnd, cStartNew, cEndNew, c, vStart, vEnd, vStartNew, vEndNew, v;
517   PetscBool      hasCells;
518 
519   PetscFunctionBegin;
520   PetscCall(DMGetCoordinateDim(dm, &dim));
521   PetscCall(DMSetCoordinateDim(dmNew, dim));
522   PetscCall(DMPlexGetDepth(dm, &depth));
523   /* Step 8: Convert coordinates */
524   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
525   PetscCall(DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd));
526   PetscCall(DMPlexGetDepthStratum(dmNew, 0, &vStartNew, &vEndNew));
527   PetscCall(DMPlexGetHeightStratum(dmNew, 0, &cStartNew, &cEndNew));
528   PetscCall(DMGetCoordinateSection(dm, &coordSection));
529   PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dm), &newCoordSection));
530   PetscCall(PetscSectionSetNumFields(newCoordSection, 1));
531   PetscCall(PetscSectionSetFieldComponents(newCoordSection, 0, dim));
532   PetscCall(PetscSectionGetChart(coordSection, &sStart, &sEnd));
533   hasCells = sStart == cStart ? PETSC_TRUE : PETSC_FALSE;
534   PetscCall(PetscSectionSetChart(newCoordSection, hasCells ? cStartNew : vStartNew, vEndNew));
535   if (hasCells) {
536     for (c = cStart; c < cEnd; ++c) {
537       PetscInt cNew = DMPlexShiftPoint_Internal(c, depth, depthShift), dof;
538 
539       PetscCall(PetscSectionGetDof(coordSection, c, &dof));
540       PetscCall(PetscSectionSetDof(newCoordSection, cNew, dof));
541       PetscCall(PetscSectionSetFieldDof(newCoordSection, cNew, 0, dof));
542     }
543   }
544   for (v = vStartNew; v < vEndNew; ++v) {
545     PetscCall(PetscSectionSetDof(newCoordSection, v, dim));
546     PetscCall(PetscSectionSetFieldDof(newCoordSection, v, 0, dim));
547   }
548   PetscCall(PetscSectionSetUp(newCoordSection));
549   PetscCall(DMSetCoordinateSection(dmNew, PETSC_DETERMINE, newCoordSection));
550   PetscCall(PetscSectionGetStorageSize(newCoordSection, &coordSize));
551   PetscCall(VecCreate(PETSC_COMM_SELF, &newCoordinates));
552   PetscCall(PetscObjectSetName((PetscObject) newCoordinates, "coordinates"));
553   PetscCall(VecSetSizes(newCoordinates, coordSize, PETSC_DETERMINE));
554   PetscCall(VecSetBlockSize(newCoordinates, dim));
555   PetscCall(VecSetType(newCoordinates,VECSTANDARD));
556   PetscCall(DMSetCoordinatesLocal(dmNew, newCoordinates));
557   PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
558   PetscCall(VecGetArray(coordinates, &coords));
559   PetscCall(VecGetArray(newCoordinates, &newCoords));
560   if (hasCells) {
561     for (c = cStart; c < cEnd; ++c) {
562       PetscInt cNew = DMPlexShiftPoint_Internal(c, depth, depthShift), dof, off, noff, d;
563 
564       PetscCall(PetscSectionGetDof(coordSection, c, &dof));
565       PetscCall(PetscSectionGetOffset(coordSection, c, &off));
566       PetscCall(PetscSectionGetOffset(newCoordSection, cNew, &noff));
567       for (d = 0; d < dof; ++d) newCoords[noff+d] = coords[off+d];
568     }
569   }
570   for (v = vStart; v < vEnd; ++v) {
571     PetscInt dof, off, noff, d;
572 
573     PetscCall(PetscSectionGetDof(coordSection, v, &dof));
574     PetscCall(PetscSectionGetOffset(coordSection, v, &off));
575     PetscCall(PetscSectionGetOffset(newCoordSection, DMPlexShiftPoint_Internal(v, depth, depthShift), &noff));
576     for (d = 0; d < dof; ++d) newCoords[noff+d] = coords[off+d];
577   }
578   PetscCall(VecRestoreArray(coordinates, &coords));
579   PetscCall(VecRestoreArray(newCoordinates, &newCoords));
580   PetscCall(VecDestroy(&newCoordinates));
581   PetscCall(PetscSectionDestroy(&newCoordSection));
582   PetscFunctionReturn(0);
583 }
584 
585 static PetscErrorCode DMPlexShiftSF_Single(DM dm, PetscInt depthShift[], PetscSF sf, PetscSF sfNew)
586 {
587   const PetscSFNode *remotePoints;
588   PetscSFNode       *gremotePoints;
589   const PetscInt    *localPoints;
590   PetscInt          *glocalPoints, *newLocation, *newRemoteLocation;
591   PetscInt           numRoots, numLeaves, l, pStart, pEnd, depth = 0, totShift = 0;
592 
593   PetscFunctionBegin;
594   PetscCall(DMPlexGetDepth(dm, &depth));
595   PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
596   PetscCall(PetscSFGetGraph(sf, &numRoots, &numLeaves, &localPoints, &remotePoints));
597   totShift = DMPlexShiftPoint_Internal(pEnd, depth, depthShift) - pEnd;
598   if (numRoots >= 0) {
599     PetscCall(PetscMalloc2(numRoots, &newLocation, pEnd-pStart, &newRemoteLocation));
600     for (l = 0; l < numRoots; ++l) newLocation[l] = DMPlexShiftPoint_Internal(l, depth, depthShift);
601     PetscCall(PetscSFBcastBegin(sf, MPIU_INT, newLocation, newRemoteLocation, MPI_REPLACE));
602     PetscCall(PetscSFBcastEnd(sf, MPIU_INT, newLocation, newRemoteLocation, MPI_REPLACE));
603     PetscCall(PetscMalloc1(numLeaves, &glocalPoints));
604     PetscCall(PetscMalloc1(numLeaves, &gremotePoints));
605     for (l = 0; l < numLeaves; ++l) {
606       glocalPoints[l]        = DMPlexShiftPoint_Internal(localPoints[l], depth, depthShift);
607       gremotePoints[l].rank  = remotePoints[l].rank;
608       gremotePoints[l].index = newRemoteLocation[localPoints[l]];
609     }
610     PetscCall(PetscFree2(newLocation, newRemoteLocation));
611     PetscCall(PetscSFSetGraph(sfNew, numRoots + totShift, numLeaves, glocalPoints, PETSC_OWN_POINTER, gremotePoints, PETSC_OWN_POINTER));
612   }
613   PetscFunctionReturn(0);
614 }
615 
616 static PetscErrorCode DMPlexShiftSF_Internal(DM dm, PetscInt depthShift[], DM dmNew)
617 {
618   PetscSF        sfPoint, sfPointNew;
619   PetscBool      useNatural;
620 
621   PetscFunctionBegin;
622   /* Step 9: Convert pointSF */
623   PetscCall(DMGetPointSF(dm, &sfPoint));
624   PetscCall(DMGetPointSF(dmNew, &sfPointNew));
625   PetscCall(DMPlexShiftSF_Single(dm, depthShift, sfPoint, sfPointNew));
626   /* Step 9b: Convert naturalSF */
627   PetscCall(DMGetUseNatural(dm, &useNatural));
628   if (useNatural) {
629     PetscSF sfNat, sfNatNew;
630 
631     PetscCall(DMSetUseNatural(dmNew, useNatural));
632     PetscCall(DMGetNaturalSF(dm, &sfNat));
633     PetscCall(DMGetNaturalSF(dmNew, &sfNatNew));
634     PetscCall(DMPlexShiftSF_Single(dm, depthShift, sfNat, sfNatNew));
635   }
636   PetscFunctionReturn(0);
637 }
638 
639 static PetscErrorCode DMPlexShiftLabels_Internal(DM dm, PetscInt depthShift[], DM dmNew)
640 {
641   PetscInt           depth = 0, numLabels, l;
642 
643   PetscFunctionBegin;
644   PetscCall(DMPlexGetDepth(dm, &depth));
645   /* Step 10: Convert labels */
646   PetscCall(DMGetNumLabels(dm, &numLabels));
647   for (l = 0; l < numLabels; ++l) {
648     DMLabel         label, newlabel;
649     const char     *lname;
650     PetscBool       isDepth, isDim;
651     IS              valueIS;
652     const PetscInt *values;
653     PetscInt        numValues, val;
654 
655     PetscCall(DMGetLabelName(dm, l, &lname));
656     PetscCall(PetscStrcmp(lname, "depth", &isDepth));
657     if (isDepth) continue;
658     PetscCall(PetscStrcmp(lname, "dim", &isDim));
659     if (isDim) continue;
660     PetscCall(DMCreateLabel(dmNew, lname));
661     PetscCall(DMGetLabel(dm, lname, &label));
662     PetscCall(DMGetLabel(dmNew, lname, &newlabel));
663     PetscCall(DMLabelGetDefaultValue(label,&val));
664     PetscCall(DMLabelSetDefaultValue(newlabel,val));
665     PetscCall(DMLabelGetValueIS(label, &valueIS));
666     PetscCall(ISGetLocalSize(valueIS, &numValues));
667     PetscCall(ISGetIndices(valueIS, &values));
668     for (val = 0; val < numValues; ++val) {
669       IS              pointIS;
670       const PetscInt *points;
671       PetscInt        numPoints, p;
672 
673       PetscCall(DMLabelGetStratumIS(label, values[val], &pointIS));
674       PetscCall(ISGetLocalSize(pointIS, &numPoints));
675       PetscCall(ISGetIndices(pointIS, &points));
676       for (p = 0; p < numPoints; ++p) {
677         const PetscInt newpoint = DMPlexShiftPoint_Internal(points[p], depth, depthShift);
678 
679         PetscCall(DMLabelSetValue(newlabel, newpoint, values[val]));
680       }
681       PetscCall(ISRestoreIndices(pointIS, &points));
682       PetscCall(ISDestroy(&pointIS));
683     }
684     PetscCall(ISRestoreIndices(valueIS, &values));
685     PetscCall(ISDestroy(&valueIS));
686   }
687   PetscFunctionReturn(0);
688 }
689 
690 static PetscErrorCode DMPlexCreateVTKLabel_Internal(DM dm, PetscBool createGhostLabel, DM dmNew)
691 {
692   PetscSF            sfPoint;
693   DMLabel            vtkLabel, ghostLabel = NULL;
694   const PetscSFNode *leafRemote;
695   const PetscInt    *leafLocal;
696   PetscInt           cellHeight, cStart, cEnd, c, fStart, fEnd, f, numLeaves, l;
697   PetscMPIInt        rank;
698 
699   PetscFunctionBegin;
700    /* Step 11: Make label for output (vtk) and to mark ghost points (ghost) */
701   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
702   PetscCall(DMGetPointSF(dm, &sfPoint));
703   PetscCall(DMPlexGetVTKCellHeight(dmNew, &cellHeight));
704   PetscCall(DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd));
705   PetscCall(PetscSFGetGraph(sfPoint, NULL, &numLeaves, &leafLocal, &leafRemote));
706   PetscCall(DMCreateLabel(dmNew, "vtk"));
707   PetscCall(DMGetLabel(dmNew, "vtk", &vtkLabel));
708   if (createGhostLabel) {
709     PetscCall(DMCreateLabel(dmNew, "ghost"));
710     PetscCall(DMGetLabel(dmNew, "ghost", &ghostLabel));
711   }
712   for (l = 0, c = cStart; l < numLeaves && c < cEnd; ++l, ++c) {
713     for (; c < leafLocal[l] && c < cEnd; ++c) {
714       PetscCall(DMLabelSetValue(vtkLabel, c, 1));
715     }
716     if (leafLocal[l] >= cEnd) break;
717     if (leafRemote[l].rank == rank) {
718       PetscCall(DMLabelSetValue(vtkLabel, c, 1));
719     } else if (ghostLabel) {
720       PetscCall(DMLabelSetValue(ghostLabel, c, 2));
721     }
722   }
723   for (; c < cEnd; ++c) {
724     PetscCall(DMLabelSetValue(vtkLabel, c, 1));
725   }
726   if (ghostLabel) {
727     PetscCall(DMPlexGetHeightStratum(dmNew, 1, &fStart, &fEnd));
728     for (f = fStart; f < fEnd; ++f) {
729       PetscInt numCells;
730 
731       PetscCall(DMPlexGetSupportSize(dmNew, f, &numCells));
732       if (numCells < 2) {
733         PetscCall(DMLabelSetValue(ghostLabel, f, 1));
734       } else {
735         const PetscInt *cells = NULL;
736         PetscInt        vA, vB;
737 
738         PetscCall(DMPlexGetSupport(dmNew, f, &cells));
739         PetscCall(DMLabelGetValue(vtkLabel, cells[0], &vA));
740         PetscCall(DMLabelGetValue(vtkLabel, cells[1], &vB));
741         if (vA != 1 && vB != 1) PetscCall(DMLabelSetValue(ghostLabel, f, 1));
742       }
743     }
744   }
745   PetscFunctionReturn(0);
746 }
747 
748 static PetscErrorCode DMPlexShiftTree_Internal(DM dm, PetscInt depthShift[], DM dmNew)
749 {
750   DM             refTree;
751   PetscSection   pSec;
752   PetscInt       *parents, *childIDs;
753 
754   PetscFunctionBegin;
755   PetscCall(DMPlexGetReferenceTree(dm,&refTree));
756   PetscCall(DMPlexSetReferenceTree(dmNew,refTree));
757   PetscCall(DMPlexGetTree(dm,&pSec,&parents,&childIDs,NULL,NULL));
758   if (pSec) {
759     PetscInt p, pStart, pEnd, *parentsShifted, pStartShifted, pEndShifted, depth;
760     PetscInt *childIDsShifted;
761     PetscSection pSecShifted;
762 
763     PetscCall(PetscSectionGetChart(pSec,&pStart,&pEnd));
764     PetscCall(DMPlexGetDepth(dm,&depth));
765     pStartShifted = DMPlexShiftPoint_Internal(pStart,depth,depthShift);
766     pEndShifted   = DMPlexShiftPoint_Internal(pEnd,depth,depthShift);
767     PetscCall(PetscMalloc2(pEndShifted - pStartShifted,&parentsShifted,pEndShifted-pStartShifted,&childIDsShifted));
768     PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dmNew),&pSecShifted));
769     PetscCall(PetscSectionSetChart(pSecShifted,pStartShifted,pEndShifted));
770     for (p = pStartShifted; p < pEndShifted; p++) {
771       /* start off assuming no children */
772       PetscCall(PetscSectionSetDof(pSecShifted,p,0));
773     }
774     for (p = pStart; p < pEnd; p++) {
775       PetscInt dof;
776       PetscInt pNew = DMPlexShiftPoint_Internal(p,depth,depthShift);
777 
778       PetscCall(PetscSectionGetDof(pSec,p,&dof));
779       PetscCall(PetscSectionSetDof(pSecShifted,pNew,dof));
780     }
781     PetscCall(PetscSectionSetUp(pSecShifted));
782     for (p = pStart; p < pEnd; p++) {
783       PetscInt dof;
784       PetscInt pNew = DMPlexShiftPoint_Internal(p,depth,depthShift);
785 
786       PetscCall(PetscSectionGetDof(pSec,p,&dof));
787       if (dof) {
788         PetscInt off, offNew;
789 
790         PetscCall(PetscSectionGetOffset(pSec,p,&off));
791         PetscCall(PetscSectionGetOffset(pSecShifted,pNew,&offNew));
792         parentsShifted[offNew] = DMPlexShiftPoint_Internal(parents[off],depth,depthShift);
793         childIDsShifted[offNew] = childIDs[off];
794       }
795     }
796     PetscCall(DMPlexSetTree(dmNew,pSecShifted,parentsShifted,childIDsShifted));
797     PetscCall(PetscFree2(parentsShifted,childIDsShifted));
798     PetscCall(PetscSectionDestroy(&pSecShifted));
799   }
800   PetscFunctionReturn(0);
801 }
802 
803 static PetscErrorCode DMPlexConstructGhostCells_Internal(DM dm, DMLabel label, PetscInt *numGhostCells, DM gdm)
804 {
805   PetscSF               sf;
806   IS                    valueIS;
807   const PetscInt       *values, *leaves;
808   PetscInt             *depthShift;
809   PetscInt              d, depth = 0, nleaves, loc, Ng, numFS, fs, fStart, fEnd, ghostCell, cEnd, c;
810   PetscBool             isper;
811   const PetscReal      *maxCell, *L;
812   const DMBoundaryType *bd;
813 
814   PetscFunctionBegin;
815   PetscCall(DMGetPointSF(dm, &sf));
816   PetscCall(PetscSFGetGraph(sf, NULL, &nleaves, &leaves, NULL));
817   nleaves = PetscMax(0, nleaves);
818   PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
819   /* Count ghost cells */
820   PetscCall(DMLabelGetValueIS(label, &valueIS));
821   PetscCall(ISGetLocalSize(valueIS, &numFS));
822   PetscCall(ISGetIndices(valueIS, &values));
823   Ng   = 0;
824   for (fs = 0; fs < numFS; ++fs) {
825     IS              faceIS;
826     const PetscInt *faces;
827     PetscInt        numFaces, f, numBdFaces = 0;
828 
829     PetscCall(DMLabelGetStratumIS(label, values[fs], &faceIS));
830     PetscCall(ISGetLocalSize(faceIS, &numFaces));
831     PetscCall(ISGetIndices(faceIS, &faces));
832     for (f = 0; f < numFaces; ++f) {
833       PetscInt numChildren;
834 
835       PetscCall(PetscFindInt(faces[f], nleaves, leaves, &loc));
836       PetscCall(DMPlexGetTreeChildren(dm,faces[f],&numChildren,NULL));
837       /* non-local and ancestors points don't get to register ghosts */
838       if (loc >= 0 || numChildren) continue;
839       if ((faces[f] >= fStart) && (faces[f] < fEnd)) ++numBdFaces;
840     }
841     Ng += numBdFaces;
842     PetscCall(ISRestoreIndices(faceIS, &faces));
843     PetscCall(ISDestroy(&faceIS));
844   }
845   PetscCall(DMPlexGetDepth(dm, &depth));
846   PetscCall(PetscMalloc1(2*(depth+1), &depthShift));
847   for (d = 0; d <= depth; d++) {
848     PetscInt dEnd;
849 
850     PetscCall(DMPlexGetDepthStratum(dm,d,NULL,&dEnd));
851     depthShift[2*d]   = dEnd;
852     depthShift[2*d+1] = 0;
853   }
854   if (depth >= 0) depthShift[2*depth+1] = Ng;
855   PetscCall(DMPlexShiftPointSetUp_Internal(depth,depthShift));
856   PetscCall(DMPlexShiftSizes_Internal(dm, depthShift, gdm));
857   /* Step 3: Set cone/support sizes for new points */
858   PetscCall(DMPlexGetHeightStratum(dm, 0, NULL, &cEnd));
859   for (c = cEnd; c < cEnd + Ng; ++c) {
860     PetscCall(DMPlexSetConeSize(gdm, c, 1));
861   }
862   for (fs = 0; fs < numFS; ++fs) {
863     IS              faceIS;
864     const PetscInt *faces;
865     PetscInt        numFaces, f;
866 
867     PetscCall(DMLabelGetStratumIS(label, values[fs], &faceIS));
868     PetscCall(ISGetLocalSize(faceIS, &numFaces));
869     PetscCall(ISGetIndices(faceIS, &faces));
870     for (f = 0; f < numFaces; ++f) {
871       PetscInt size, numChildren;
872 
873       PetscCall(PetscFindInt(faces[f], nleaves, leaves, &loc));
874       PetscCall(DMPlexGetTreeChildren(dm,faces[f],&numChildren,NULL));
875       if (loc >= 0 || numChildren) continue;
876       if ((faces[f] < fStart) || (faces[f] >= fEnd)) continue;
877       PetscCall(DMPlexGetSupportSize(dm, faces[f], &size));
878       PetscCheck(size == 1,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM has boundary face %" PetscInt_FMT " with %" PetscInt_FMT " support cells", faces[f], size);
879       PetscCall(DMPlexSetSupportSize(gdm, faces[f] + Ng, 2));
880     }
881     PetscCall(ISRestoreIndices(faceIS, &faces));
882     PetscCall(ISDestroy(&faceIS));
883   }
884   /* Step 4: Setup ghosted DM */
885   PetscCall(DMSetUp(gdm));
886   PetscCall(DMPlexShiftPoints_Internal(dm, depthShift, gdm));
887   /* Step 6: Set cones and supports for new points */
888   ghostCell = cEnd;
889   for (fs = 0; fs < numFS; ++fs) {
890     IS              faceIS;
891     const PetscInt *faces;
892     PetscInt        numFaces, f;
893 
894     PetscCall(DMLabelGetStratumIS(label, values[fs], &faceIS));
895     PetscCall(ISGetLocalSize(faceIS, &numFaces));
896     PetscCall(ISGetIndices(faceIS, &faces));
897     for (f = 0; f < numFaces; ++f) {
898       PetscInt newFace = faces[f] + Ng, numChildren;
899 
900       PetscCall(PetscFindInt(faces[f], nleaves, leaves, &loc));
901       PetscCall(DMPlexGetTreeChildren(dm,faces[f],&numChildren,NULL));
902       if (loc >= 0 || numChildren) continue;
903       if ((faces[f] < fStart) || (faces[f] >= fEnd)) continue;
904       PetscCall(DMPlexSetCone(gdm, ghostCell, &newFace));
905       PetscCall(DMPlexInsertSupport(gdm, newFace, 1, ghostCell));
906       ++ghostCell;
907     }
908     PetscCall(ISRestoreIndices(faceIS, &faces));
909     PetscCall(ISDestroy(&faceIS));
910   }
911   PetscCall(ISRestoreIndices(valueIS, &values));
912   PetscCall(ISDestroy(&valueIS));
913   PetscCall(DMPlexShiftCoordinates_Internal(dm, depthShift, gdm));
914   PetscCall(DMPlexShiftSF_Internal(dm, depthShift, gdm));
915   PetscCall(DMPlexShiftLabels_Internal(dm, depthShift, gdm));
916   PetscCall(DMPlexCreateVTKLabel_Internal(dm, PETSC_TRUE, gdm));
917   PetscCall(DMPlexShiftTree_Internal(dm, depthShift, gdm));
918   PetscCall(PetscFree(depthShift));
919   for (c = cEnd; c < cEnd + Ng; ++c) {
920     PetscCall(DMPlexSetCellType(gdm, c, DM_POLYTOPE_FV_GHOST));
921   }
922   /* Step 7: Periodicity */
923   PetscCall(DMGetPeriodicity(dm, &isper, &maxCell, &L, &bd));
924   PetscCall(DMSetPeriodicity(gdm, isper, maxCell,  L,  bd));
925   if (numGhostCells) *numGhostCells = Ng;
926   PetscFunctionReturn(0);
927 }
928 
929 /*@C
930   DMPlexConstructGhostCells - Construct ghost cells which connect to every boundary face
931 
932   Collective on dm
933 
934   Input Parameters:
935 + dm - The original DM
936 - labelName - The label specifying the boundary faces, or "Face Sets" if this is NULL
937 
938   Output Parameters:
939 + numGhostCells - The number of ghost cells added to the DM
940 - dmGhosted - The new DM
941 
942   Note: If no label exists of that name, one will be created marking all boundary faces
943 
944   Level: developer
945 
946 .seealso: `DMCreate()`
947 @*/
948 PetscErrorCode DMPlexConstructGhostCells(DM dm, const char labelName[], PetscInt *numGhostCells, DM *dmGhosted)
949 {
950   DM             gdm;
951   DMLabel        label;
952   const char    *name = labelName ? labelName : "Face Sets";
953   PetscInt       dim, Ng = 0;
954   PetscBool      useCone, useClosure;
955 
956   PetscFunctionBegin;
957   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
958   if (numGhostCells) PetscValidIntPointer(numGhostCells, 3);
959   PetscValidPointer(dmGhosted, 4);
960   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &gdm));
961   PetscCall(DMSetType(gdm, DMPLEX));
962   PetscCall(DMGetDimension(dm, &dim));
963   PetscCall(DMSetDimension(gdm, dim));
964   PetscCall(DMGetBasicAdjacency(dm, &useCone, &useClosure));
965   PetscCall(DMSetBasicAdjacency(gdm, useCone,  useClosure));
966   PetscCall(DMGetLabel(dm, name, &label));
967   if (!label) {
968     /* Get label for boundary faces */
969     PetscCall(DMCreateLabel(dm, name));
970     PetscCall(DMGetLabel(dm, name, &label));
971     PetscCall(DMPlexMarkBoundaryFaces(dm, 1, label));
972   }
973   PetscCall(DMPlexConstructGhostCells_Internal(dm, label, &Ng, gdm));
974   PetscCall(DMCopyDisc(dm, gdm));
975   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_TRUE, gdm));
976   gdm->setfromoptionscalled = dm->setfromoptionscalled;
977   if (numGhostCells) *numGhostCells = Ng;
978   *dmGhosted = gdm;
979   PetscFunctionReturn(0);
980 }
981 
982 static PetscErrorCode DivideCells_Private(DM dm, DMLabel label, DMPlexPointQueue queue)
983 {
984   PetscInt dim, d, shift = 100, *pStart, *pEnd;
985 
986   PetscFunctionBegin;
987   PetscCall(DMGetDimension(dm, &dim));
988   PetscCall(PetscMalloc2(dim, &pStart, dim, &pEnd));
989   for (d = 0; d < dim; ++d) {
990     PetscCall(DMPlexGetDepthStratum(dm, d, &pStart[d], &pEnd[d]));
991   }
992   while (!DMPlexPointQueueEmpty(queue)) {
993     PetscInt  cell    = -1;
994     PetscInt *closure = NULL;
995     PetscInt  closureSize, cl, cval;
996 
997     PetscCall(DMPlexPointQueueDequeue(queue, &cell));
998     PetscCall(DMLabelGetValue(label, cell, &cval));
999     PetscCall(DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
1000     /* Mark points in the cell closure that touch the fault */
1001     for (d = 0; d < dim; ++d) {
1002       for (cl = 0; cl < closureSize*2; cl += 2) {
1003         const PetscInt clp = closure[cl];
1004         PetscInt       clval;
1005 
1006         if ((clp < pStart[d]) || (clp >= pEnd[d])) continue;
1007         PetscCall(DMLabelGetValue(label, clp, &clval));
1008         if (clval == -1) {
1009           const PetscInt *cone;
1010           PetscInt        coneSize, c;
1011 
1012           /* If a cone point touches the fault, then this point touches the fault */
1013           PetscCall(DMPlexGetCone(dm, clp, &cone));
1014           PetscCall(DMPlexGetConeSize(dm, clp, &coneSize));
1015           for (c = 0; c < coneSize; ++c) {
1016             PetscInt cpval;
1017 
1018             PetscCall(DMLabelGetValue(label, cone[c], &cpval));
1019             if (cpval != -1) {
1020               PetscInt dep;
1021 
1022               PetscCall(DMPlexGetPointDepth(dm, clp, &dep));
1023               clval = cval < 0 ? -(shift+dep) : shift+dep;
1024               PetscCall(DMLabelSetValue(label, clp, clval));
1025               break;
1026             }
1027           }
1028         }
1029         /* Mark neighbor cells through marked faces (these cells must also touch the fault) */
1030         if (d == dim-1 && clval != -1) {
1031           const PetscInt *support;
1032           PetscInt        supportSize, s, nval;
1033 
1034           PetscCall(DMPlexGetSupport(dm, clp, &support));
1035           PetscCall(DMPlexGetSupportSize(dm, clp, &supportSize));
1036           for (s = 0; s < supportSize; ++s) {
1037             PetscCall(DMLabelGetValue(label, support[s], &nval));
1038             if (nval == -1) {
1039               PetscCall(DMLabelSetValue(label, support[s], clval < 0 ? clval-1 : clval+1));
1040               PetscCall(DMPlexPointQueueEnqueue(queue, support[s]));
1041             }
1042           }
1043         }
1044       }
1045     }
1046     PetscCall(DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
1047   }
1048   PetscCall(PetscFree2(pStart, pEnd));
1049   PetscFunctionReturn(0);
1050 }
1051 
1052 typedef struct {
1053   DM               dm;
1054   DMPlexPointQueue queue;
1055 } PointDivision;
1056 
1057 static PetscErrorCode divideCell(DMLabel label, PetscInt p, PetscInt val, void *ctx)
1058 {
1059   PointDivision  *div  = (PointDivision *) ctx;
1060   PetscInt        cval = val < 0 ? val-1 : val+1;
1061   const PetscInt *support;
1062   PetscInt        supportSize, s;
1063 
1064   PetscFunctionBegin;
1065   PetscCall(DMPlexGetSupport(div->dm, p, &support));
1066   PetscCall(DMPlexGetSupportSize(div->dm, p, &supportSize));
1067   for (s = 0; s < supportSize; ++s) {
1068     PetscCall(DMLabelSetValue(label, support[s], cval));
1069     PetscCall(DMPlexPointQueueEnqueue(div->queue, support[s]));
1070   }
1071   PetscFunctionReturn(0);
1072 }
1073 
1074 /* Mark cells by label propagation */
1075 static PetscErrorCode DMPlexLabelFaultHalo(DM dm, DMLabel faultLabel)
1076 {
1077   DMPlexPointQueue queue = NULL;
1078   PointDivision    div;
1079   PetscSF          pointSF;
1080   IS               pointIS;
1081   const PetscInt  *points;
1082   PetscBool        empty;
1083   PetscInt         dim, shift = 100, n, i;
1084 
1085   PetscFunctionBegin;
1086   PetscCall(DMGetDimension(dm, &dim));
1087   PetscCall(DMPlexPointQueueCreate(1024, &queue));
1088   div.dm    = dm;
1089   div.queue = queue;
1090   /* Enqueue cells on fault */
1091   PetscCall(DMLabelGetStratumIS(faultLabel, shift+dim, &pointIS));
1092   if (pointIS) {
1093     PetscCall(ISGetLocalSize(pointIS, &n));
1094     PetscCall(ISGetIndices(pointIS, &points));
1095     for (i = 0; i < n; ++i) {PetscCall(DMPlexPointQueueEnqueue(queue, points[i]));}
1096     PetscCall(ISRestoreIndices(pointIS, &points));
1097     PetscCall(ISDestroy(&pointIS));
1098   }
1099   PetscCall(DMLabelGetStratumIS(faultLabel, -(shift+dim), &pointIS));
1100   if (pointIS) {
1101     PetscCall(ISGetLocalSize(pointIS, &n));
1102     PetscCall(ISGetIndices(pointIS, &points));
1103     for (i = 0; i < n; ++i) {PetscCall(DMPlexPointQueueEnqueue(queue, points[i]));}
1104     PetscCall(ISRestoreIndices(pointIS, &points));
1105     PetscCall(ISDestroy(&pointIS));
1106   }
1107 
1108   PetscCall(DMGetPointSF(dm, &pointSF));
1109   PetscCall(DMLabelPropagateBegin(faultLabel, pointSF));
1110   /* While edge queue is not empty: */
1111   PetscCall(DMPlexPointQueueEmptyCollective((PetscObject) dm, queue, &empty));
1112   while (!empty) {
1113     PetscCall(DivideCells_Private(dm, faultLabel, queue));
1114     PetscCall(DMLabelPropagatePush(faultLabel, pointSF, divideCell, &div));
1115     PetscCall(DMPlexPointQueueEmptyCollective((PetscObject) dm, queue, &empty));
1116   }
1117   PetscCall(DMLabelPropagateEnd(faultLabel, pointSF));
1118   PetscCall(DMPlexPointQueueDestroy(&queue));
1119   PetscFunctionReturn(0);
1120 }
1121 
1122 /*
1123   We are adding three kinds of points here:
1124     Replicated:     Copies of points which exist in the mesh, such as vertices identified across a fault
1125     Non-replicated: Points which exist on the fault, but are not replicated
1126     Ghost:          These are shared fault faces which are not owned by this process. These do not produce hybrid cells and do not replicate
1127     Hybrid:         Entirely new points, such as cohesive cells
1128 
1129   When creating subsequent cohesive cells, we shift the old hybrid cells to the end of the numbering at
1130   each depth so that the new split/hybrid points can be inserted as a block.
1131 */
1132 static PetscErrorCode DMPlexConstructCohesiveCells_Internal(DM dm, DMLabel label, DMLabel splitLabel, DM sdm)
1133 {
1134   MPI_Comm         comm;
1135   IS               valueIS;
1136   PetscInt         numSP = 0;          /* The number of depths for which we have replicated points */
1137   const PetscInt  *values;             /* List of depths for which we have replicated points */
1138   IS              *splitIS;
1139   IS              *unsplitIS;
1140   IS               ghostIS;
1141   PetscInt        *numSplitPoints;     /* The number of replicated points at each depth */
1142   PetscInt        *numUnsplitPoints;   /* The number of non-replicated points at each depth which still give rise to hybrid points */
1143   PetscInt        *numHybridPoints;    /* The number of new hybrid points at each depth */
1144   PetscInt        *numHybridPointsOld; /* The number of existing hybrid points at each depth */
1145   PetscInt         numGhostPoints;     /* The number of unowned, shared fault faces */
1146   const PetscInt **splitPoints;        /* Replicated points for each depth */
1147   const PetscInt **unsplitPoints;      /* Non-replicated points for each depth */
1148   const PetscInt  *ghostPoints;        /* Ghost fault faces */
1149   PetscSection     coordSection;
1150   Vec              coordinates;
1151   PetscScalar     *coords;
1152   PetscInt        *depthMax;           /* The first hybrid point at each depth in the original mesh */
1153   PetscInt        *depthEnd;           /* The point limit at each depth in the original mesh */
1154   PetscInt        *depthShift;         /* Number of replicated+hybrid points at each depth */
1155   PetscInt        *pMaxNew;            /* The first replicated point at each depth in the new mesh, hybrids come after this */
1156   PetscInt        *coneNew, *coneONew, *supportNew;
1157   PetscInt         shift = 100, shift2 = 200, depth = 0, dep, dim, d, sp, maxConeSize, maxSupportSize, maxConeSizeNew, maxSupportSizeNew, numLabels, vStart, vEnd, pEnd, p, v;
1158 
1159   PetscFunctionBegin;
1160   PetscCall(PetscObjectGetComm((PetscObject)dm,&comm));
1161   PetscCall(DMGetDimension(dm, &dim));
1162   PetscCall(DMPlexGetDepth(dm, &depth));
1163   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
1164   /* We do not want this label automatically computed, instead we compute it here */
1165   PetscCall(DMCreateLabel(sdm, "celltype"));
1166   /* Count split points and add cohesive cells */
1167   PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize));
1168   PetscCall(PetscMalloc5(depth+1,&depthMax,depth+1,&depthEnd,2*(depth+1),&depthShift,depth+1,&pMaxNew,depth+1,&numHybridPointsOld));
1169   PetscCall(PetscMalloc7(depth+1,&splitIS,depth+1,&unsplitIS,depth+1,&numSplitPoints,depth+1,&numUnsplitPoints,depth+1,&numHybridPoints,depth+1,&splitPoints,depth+1,&unsplitPoints));
1170   for (d = 0; d <= depth; ++d) {
1171     PetscCall(DMPlexGetDepthStratum(dm, d, NULL, &pMaxNew[d]));
1172     PetscCall(DMPlexGetTensorPrismBounds_Internal(dm, d, &depthMax[d], NULL));
1173     depthEnd[d]           = pMaxNew[d];
1174     depthMax[d]           = depthMax[d] < 0 ? depthEnd[d] : depthMax[d];
1175     numSplitPoints[d]     = 0;
1176     numUnsplitPoints[d]   = 0;
1177     numHybridPoints[d]    = 0;
1178     numHybridPointsOld[d] = depthMax[d] < 0 ? 0 : depthEnd[d] - depthMax[d];
1179     splitPoints[d]        = NULL;
1180     unsplitPoints[d]      = NULL;
1181     splitIS[d]            = NULL;
1182     unsplitIS[d]          = NULL;
1183     /* we are shifting the existing hybrid points with the stratum behind them, so
1184      * the split comes at the end of the normal points, i.e., at depthMax[d] */
1185     depthShift[2*d]       = depthMax[d];
1186     depthShift[2*d+1]     = 0;
1187   }
1188   numGhostPoints = 0;
1189   ghostPoints    = NULL;
1190   if (label) {
1191     PetscCall(DMLabelGetValueIS(label, &valueIS));
1192     PetscCall(ISGetLocalSize(valueIS, &numSP));
1193     PetscCall(ISGetIndices(valueIS, &values));
1194   }
1195   for (sp = 0; sp < numSP; ++sp) {
1196     const PetscInt dep = values[sp];
1197 
1198     if ((dep < 0) || (dep > depth)) continue;
1199     PetscCall(DMLabelGetStratumIS(label, dep, &splitIS[dep]));
1200     if (splitIS[dep]) {
1201       PetscCall(ISGetLocalSize(splitIS[dep], &numSplitPoints[dep]));
1202       PetscCall(ISGetIndices(splitIS[dep], &splitPoints[dep]));
1203     }
1204     PetscCall(DMLabelGetStratumIS(label, shift2+dep, &unsplitIS[dep]));
1205     if (unsplitIS[dep]) {
1206       PetscCall(ISGetLocalSize(unsplitIS[dep], &numUnsplitPoints[dep]));
1207       PetscCall(ISGetIndices(unsplitIS[dep], &unsplitPoints[dep]));
1208     }
1209   }
1210   PetscCall(DMLabelGetStratumIS(label, shift2+dim-1, &ghostIS));
1211   if (ghostIS) {
1212     PetscCall(ISGetLocalSize(ghostIS, &numGhostPoints));
1213     PetscCall(ISGetIndices(ghostIS, &ghostPoints));
1214   }
1215   /* Calculate number of hybrid points */
1216   for (d = 1; d <= depth; ++d) numHybridPoints[d]     = numSplitPoints[d-1] + numUnsplitPoints[d-1]; /* There is a hybrid cell/face/edge for every split face/edge/vertex   */
1217   for (d = 0; d <= depth; ++d) depthShift[2*d+1]      = numSplitPoints[d] + numHybridPoints[d];
1218   PetscCall(DMPlexShiftPointSetUp_Internal(depth,depthShift));
1219   /* the end of the points in this stratum that come before the new points:
1220    * shifting pMaxNew[d] gets the new start of the next stratum, then count back the old hybrid points and the newly
1221    * added points */
1222   for (d = 0; d <= depth; ++d) pMaxNew[d]             = DMPlexShiftPoint_Internal(pMaxNew[d],depth,depthShift) - (numHybridPointsOld[d] + numSplitPoints[d] + numHybridPoints[d]);
1223   PetscCall(DMPlexShiftSizes_Internal(dm, depthShift, sdm));
1224   /* Step 3: Set cone/support sizes for new points */
1225   for (dep = 0; dep <= depth; ++dep) {
1226     for (p = 0; p < numSplitPoints[dep]; ++p) {
1227       const PetscInt  oldp   = splitPoints[dep][p];
1228       const PetscInt  newp   = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*oldp + depthOffset[dep]*/;
1229       const PetscInt  splitp = p    + pMaxNew[dep];
1230       const PetscInt *support;
1231       DMPolytopeType  ct;
1232       PetscInt        coneSize, supportSize, qf, qn, qp, e;
1233 
1234       PetscCall(DMPlexGetConeSize(dm, oldp, &coneSize));
1235       PetscCall(DMPlexSetConeSize(sdm, splitp, coneSize));
1236       PetscCall(DMPlexGetSupportSize(dm, oldp, &supportSize));
1237       PetscCall(DMPlexSetSupportSize(sdm, splitp, supportSize));
1238       PetscCall(DMPlexGetCellType(dm, oldp, &ct));
1239       PetscCall(DMPlexSetCellType(sdm, splitp, ct));
1240       if (dep == depth-1) {
1241         const PetscInt hybcell = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1242 
1243         /* Add cohesive cells, they are prisms */
1244         PetscCall(DMPlexSetConeSize(sdm, hybcell, 2 + coneSize));
1245         switch (coneSize) {
1246           case 2: PetscCall(DMPlexSetCellType(sdm, hybcell, DM_POLYTOPE_SEG_PRISM_TENSOR));break;
1247           case 3: PetscCall(DMPlexSetCellType(sdm, hybcell, DM_POLYTOPE_TRI_PRISM_TENSOR));break;
1248           case 4: PetscCall(DMPlexSetCellType(sdm, hybcell, DM_POLYTOPE_QUAD_PRISM_TENSOR));break;
1249         }
1250         /* Shared fault faces with only one support cell now have two with the cohesive cell */
1251         /*   TODO Check thaat oldp has rootdegree == 1 */
1252         if (supportSize == 1) {
1253           const PetscInt *support;
1254           PetscInt        val;
1255 
1256           PetscCall(DMPlexGetSupport(dm, oldp, &support));
1257           PetscCall(DMLabelGetValue(label, support[0], &val));
1258           if (val < 0) PetscCall(DMPlexSetSupportSize(sdm, splitp, 2));
1259           else         PetscCall(DMPlexSetSupportSize(sdm, newp,   2));
1260         }
1261       } else if (dep == 0) {
1262         const PetscInt hybedge = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1263 
1264         PetscCall(DMPlexGetSupport(dm, oldp, &support));
1265         for (e = 0, qn = 0, qp = 0, qf = 0; e < supportSize; ++e) {
1266           PetscInt val;
1267 
1268           PetscCall(DMLabelGetValue(label, support[e], &val));
1269           if (val == 1) ++qf;
1270           if ((val == 1) || (val ==  (shift + 1))) ++qn;
1271           if ((val == 1) || (val == -(shift + 1))) ++qp;
1272         }
1273         /* Split old vertex: Edges into original vertex and new cohesive edge */
1274         PetscCall(DMPlexSetSupportSize(sdm, newp, qn+1));
1275         /* Split new vertex: Edges into split vertex and new cohesive edge */
1276         PetscCall(DMPlexSetSupportSize(sdm, splitp, qp+1));
1277         /* Add hybrid edge */
1278         PetscCall(DMPlexSetConeSize(sdm, hybedge, 2));
1279         PetscCall(DMPlexSetSupportSize(sdm, hybedge, qf));
1280         PetscCall(DMPlexSetCellType(sdm, hybedge, DM_POLYTOPE_POINT_PRISM_TENSOR));
1281       } else if (dep == dim-2) {
1282         const PetscInt hybface = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1283 
1284         PetscCall(DMPlexGetSupport(dm, oldp, &support));
1285         for (e = 0, qn = 0, qp = 0, qf = 0; e < supportSize; ++e) {
1286           PetscInt val;
1287 
1288           PetscCall(DMLabelGetValue(label, support[e], &val));
1289           if (val == dim-1) ++qf;
1290           if ((val == dim-1) || (val ==  (shift + dim-1))) ++qn;
1291           if ((val == dim-1) || (val == -(shift + dim-1))) ++qp;
1292         }
1293         /* Split old edge: Faces into original edge and cohesive face (positive side?) */
1294         PetscCall(DMPlexSetSupportSize(sdm, newp, qn+1));
1295         /* Split new edge: Faces into split edge and cohesive face (negative side?) */
1296         PetscCall(DMPlexSetSupportSize(sdm, splitp, qp+1));
1297         /* Add hybrid face */
1298         PetscCall(DMPlexSetConeSize(sdm, hybface, 4));
1299         PetscCall(DMPlexSetSupportSize(sdm, hybface, qf));
1300         PetscCall(DMPlexSetCellType(sdm, hybface, DM_POLYTOPE_SEG_PRISM_TENSOR));
1301       }
1302     }
1303   }
1304   for (dep = 0; dep <= depth; ++dep) {
1305     for (p = 0; p < numUnsplitPoints[dep]; ++p) {
1306       const PetscInt  oldp   = unsplitPoints[dep][p];
1307       const PetscInt  newp   = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*oldp + depthOffset[dep]*/;
1308       const PetscInt *support;
1309       PetscInt        coneSize, supportSize, qf, e, s;
1310 
1311       PetscCall(DMPlexGetConeSize(dm, oldp, &coneSize));
1312       PetscCall(DMPlexGetSupportSize(dm, oldp, &supportSize));
1313       PetscCall(DMPlexGetSupport(dm, oldp, &support));
1314       if (dep == 0) {
1315         const PetscInt hybedge = p + pMaxNew[dep+1] + numSplitPoints[dep+1] + numSplitPoints[dep];
1316 
1317         /* Unsplit vertex: Edges into original vertex, split edges, and new cohesive edge twice */
1318         for (s = 0, qf = 0; s < supportSize; ++s, ++qf) {
1319           PetscCall(PetscFindInt(support[s], numSplitPoints[dep+1], splitPoints[dep+1], &e));
1320           if (e >= 0) ++qf;
1321         }
1322         PetscCall(DMPlexSetSupportSize(sdm, newp, qf+2));
1323         /* Add hybrid edge */
1324         PetscCall(DMPlexSetConeSize(sdm, hybedge, 2));
1325         for (e = 0, qf = 0; e < supportSize; ++e) {
1326           PetscInt val;
1327 
1328           PetscCall(DMLabelGetValue(label, support[e], &val));
1329           /* Split and unsplit edges produce hybrid faces */
1330           if (val == 1) ++qf;
1331           if (val == (shift2 + 1)) ++qf;
1332         }
1333         PetscCall(DMPlexSetSupportSize(sdm, hybedge, qf));
1334         PetscCall(DMPlexSetCellType(sdm, hybedge, DM_POLYTOPE_POINT_PRISM_TENSOR));
1335       } else if (dep == dim-2) {
1336         const PetscInt hybface = p + pMaxNew[dep+1] + numSplitPoints[dep+1] + numSplitPoints[dep];
1337         PetscInt       val;
1338 
1339         for (e = 0, qf = 0; e < supportSize; ++e) {
1340           PetscCall(DMLabelGetValue(label, support[e], &val));
1341           if (val == dim-1) qf += 2;
1342           else              ++qf;
1343         }
1344         /* Unsplit edge: Faces into original edge, split face, and cohesive face twice */
1345         PetscCall(DMPlexSetSupportSize(sdm, newp, qf+2));
1346         /* Add hybrid face */
1347         for (e = 0, qf = 0; e < supportSize; ++e) {
1348           PetscCall(DMLabelGetValue(label, support[e], &val));
1349           if (val == dim-1) ++qf;
1350         }
1351         PetscCall(DMPlexSetConeSize(sdm, hybface, 4));
1352         PetscCall(DMPlexSetSupportSize(sdm, hybface, qf));
1353         PetscCall(DMPlexSetCellType(sdm, hybface, DM_POLYTOPE_SEG_PRISM_TENSOR));
1354       }
1355     }
1356   }
1357   /* Step 4: Setup split DM */
1358   PetscCall(DMSetUp(sdm));
1359   PetscCall(DMPlexShiftPoints_Internal(dm, depthShift, sdm));
1360   PetscCall(DMPlexGetMaxSizes(sdm, &maxConeSizeNew, &maxSupportSizeNew));
1361   PetscCall(PetscMalloc3(PetscMax(maxConeSize, maxConeSizeNew)*3,&coneNew,PetscMax(maxConeSize, maxConeSizeNew)*3,&coneONew,PetscMax(maxSupportSize, maxSupportSizeNew),&supportNew));
1362   /* Step 6: Set cones and supports for new points */
1363   for (dep = 0; dep <= depth; ++dep) {
1364     for (p = 0; p < numSplitPoints[dep]; ++p) {
1365       const PetscInt  oldp   = splitPoints[dep][p];
1366       const PetscInt  newp   = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*oldp + depthOffset[dep]*/;
1367       const PetscInt  splitp = p    + pMaxNew[dep];
1368       const PetscInt *cone, *support, *ornt;
1369       DMPolytopeType  ct;
1370       PetscInt        coneSize, supportSize, q, qf, qn, qp, v, e, s;
1371 
1372       PetscCall(DMPlexGetCellType(dm, oldp, &ct));
1373       PetscCall(DMPlexGetConeSize(dm, oldp, &coneSize));
1374       PetscCall(DMPlexGetCone(dm, oldp, &cone));
1375       PetscCall(DMPlexGetConeOrientation(dm, oldp, &ornt));
1376       PetscCall(DMPlexGetSupportSize(dm, oldp, &supportSize));
1377       PetscCall(DMPlexGetSupport(dm, oldp, &support));
1378       if (dep == depth-1) {
1379         PetscBool       hasUnsplit = PETSC_FALSE;
1380         const PetscInt  hybcell    = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1381         const PetscInt *supportF;
1382 
1383         coneONew[0] = coneONew[1] = -1000;
1384         /* Split face:       copy in old face to new face to start */
1385         PetscCall(DMPlexGetSupport(sdm, newp,  &supportF));
1386         PetscCall(DMPlexSetSupport(sdm, splitp, supportF));
1387         /* Split old face:   old vertices/edges in cone so no change */
1388         /* Split new face:   new vertices/edges in cone */
1389         for (q = 0; q < coneSize; ++q) {
1390           PetscCall(PetscFindInt(cone[q], numSplitPoints[dep-1], splitPoints[dep-1], &v));
1391           if (v < 0) {
1392             PetscCall(PetscFindInt(cone[q], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &v));
1393             PetscCheck(v >= 0,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %" PetscInt_FMT " in split or unsplit points of depth %" PetscInt_FMT, cone[q], dep-1);
1394             coneNew[2+q] = DMPlexShiftPoint_Internal(cone[q], depth, depthShift) /*cone[q] + depthOffset[dep-1]*/;
1395             hasUnsplit   = PETSC_TRUE;
1396           } else {
1397             coneNew[2+q] = v + pMaxNew[dep-1];
1398             if (dep > 1) {
1399               const PetscInt *econe;
1400               PetscInt        econeSize, r, vs, vu;
1401 
1402               PetscCall(DMPlexGetConeSize(dm, cone[q], &econeSize));
1403               PetscCall(DMPlexGetCone(dm, cone[q], &econe));
1404               for (r = 0; r < econeSize; ++r) {
1405                 PetscCall(PetscFindInt(econe[r], numSplitPoints[dep-2],   splitPoints[dep-2],   &vs));
1406                 PetscCall(PetscFindInt(econe[r], numUnsplitPoints[dep-2], unsplitPoints[dep-2], &vu));
1407                 if (vs >= 0) continue;
1408                 PetscCheck(vu >= 0,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %" PetscInt_FMT " in split or unsplit points of depth %" PetscInt_FMT, econe[r], dep-2);
1409                 hasUnsplit   = PETSC_TRUE;
1410               }
1411             }
1412           }
1413         }
1414         PetscCall(DMPlexSetCone(sdm, splitp, &coneNew[2]));
1415         PetscCall(DMPlexSetConeOrientation(sdm, splitp, ornt));
1416         /* Face support */
1417         PetscInt vals[2];
1418 
1419         PetscCall(DMLabelGetValue(label, support[0], &vals[0]));
1420         if (supportSize > 1) PetscCall(DMLabelGetValue(label, support[1], &vals[1]));
1421         else                 vals[1] = -vals[0];
1422         PetscCheck(vals[0]*vals[1] < 0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid support labels %" PetscInt_FMT " %" PetscInt_FMT, vals[0], vals[1]);
1423 
1424         for (s = 0; s < 2; ++s) {
1425           if (s >= supportSize) {
1426             if (vals[s] < 0) {
1427               /* Ghost old face:   Replace negative side cell with cohesive cell */
1428               PetscCall(DMPlexInsertSupport(sdm, newp,   s, hybcell));
1429             } else {
1430               /* Ghost new face:   Replace positive side cell with cohesive cell */
1431               PetscCall(DMPlexInsertSupport(sdm, splitp, s, hybcell));
1432             }
1433           } else {
1434             if (vals[s] < 0) {
1435               /* Split old face:   Replace negative side cell with cohesive cell */
1436               PetscCall(DMPlexInsertSupport(sdm, newp,   s, hybcell));
1437             } else {
1438               /* Split new face:   Replace positive side cell with cohesive cell */
1439               PetscCall(DMPlexInsertSupport(sdm, splitp, s, hybcell));
1440             }
1441           }
1442         }
1443         /* Get orientation for cohesive face using the positive side cell */
1444         {
1445           const PetscInt *ncone, *nconeO;
1446           PetscInt        nconeSize, nc, ocell;
1447           PetscBool       flip = PETSC_FALSE;
1448 
1449           if (supportSize > 1) {ocell = vals[0] < 0 ? support[1] : support[0];}
1450           else                 {ocell = support[0]; flip = vals[0] < 0 ? PETSC_TRUE : PETSC_FALSE;}
1451           PetscCall(DMPlexGetConeSize(dm, ocell, &nconeSize));
1452           PetscCall(DMPlexGetCone(dm, ocell, &ncone));
1453           PetscCall(DMPlexGetConeOrientation(dm, ocell, &nconeO));
1454           for (nc = 0; nc < nconeSize; ++nc) {
1455             if (ncone[nc] == oldp) {
1456               coneONew[0] = flip ? -(nconeO[nc]+1) : nconeO[nc];
1457               break;
1458             }
1459           }
1460           PetscCheck(nc < nconeSize, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate face %" PetscInt_FMT " in neighboring cell %" PetscInt_FMT, oldp, ocell);
1461         }
1462         /* Cohesive cell:    Old and new split face, then new cohesive faces */
1463         {
1464           const PetscInt No = DMPolytopeTypeGetNumArrangments(ct)/2;
1465           PetscCheck((coneONew[0] >= -No) && (coneONew[0] < No), PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid %s orientation %" PetscInt_FMT, DMPolytopeTypes[ct], coneONew[0]);
1466         }
1467         const PetscInt *arr = DMPolytopeTypeGetArrangment(ct, coneONew[0]);
1468 
1469         coneNew[0]  = newp;   /* Extracted negative side orientation above */
1470         coneNew[1]  = splitp;
1471         coneONew[1] = coneONew[0];
1472         for (q = 0; q < coneSize; ++q) {
1473           /* Hybrid faces must follow order from oriented end face */
1474           const PetscInt qa = arr[q*2+0];
1475           const PetscInt qo = arr[q*2+1];
1476           DMPolytopeType ft = dep == 2 ? DM_POLYTOPE_SEGMENT : DM_POLYTOPE_POINT;
1477 
1478           PetscCall(PetscFindInt(cone[qa], numSplitPoints[dep-1], splitPoints[dep-1], &v));
1479           if (v < 0) {
1480             PetscCall(PetscFindInt(cone[qa], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &v));
1481             coneNew[2+q]  = v + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep-1];
1482           } else {
1483             coneNew[2+q]  = v + pMaxNew[dep] + numSplitPoints[dep];
1484           }
1485           coneONew[2+q] = DMPolytopeTypeComposeOrientation(ft, qo, ornt[qa]);
1486         }
1487         PetscCall(DMPlexSetCone(sdm, hybcell, coneNew));
1488         PetscCall(DMPlexSetConeOrientation(sdm, hybcell, coneONew));
1489         /* Label the hybrid cells on the boundary of the split */
1490         if (hasUnsplit) PetscCall(DMLabelSetValue(label, -hybcell, dim));
1491       } else if (dep == 0) {
1492         const PetscInt hybedge = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1493 
1494         /* Split old vertex: Edges in old split faces and new cohesive edge */
1495         for (e = 0, qn = 0; e < supportSize; ++e) {
1496           PetscInt val;
1497 
1498           PetscCall(DMLabelGetValue(label, support[e], &val));
1499           if ((val == 1) || (val == (shift + 1))) {
1500             supportNew[qn++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1501           }
1502         }
1503         supportNew[qn] = hybedge;
1504         PetscCall(DMPlexSetSupport(sdm, newp, supportNew));
1505         /* Split new vertex: Edges in new split faces and new cohesive edge */
1506         for (e = 0, qp = 0; e < supportSize; ++e) {
1507           PetscInt val, edge;
1508 
1509           PetscCall(DMLabelGetValue(label, support[e], &val));
1510           if (val == 1) {
1511             PetscCall(PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &edge));
1512             PetscCheck(edge >= 0,comm, PETSC_ERR_ARG_WRONG, "Edge %" PetscInt_FMT " is not a split edge", support[e]);
1513             supportNew[qp++] = edge + pMaxNew[dep+1];
1514           } else if (val == -(shift + 1)) {
1515             supportNew[qp++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1516           }
1517         }
1518         supportNew[qp] = hybedge;
1519         PetscCall(DMPlexSetSupport(sdm, splitp, supportNew));
1520         /* Hybrid edge:    Old and new split vertex */
1521         coneNew[0] = newp;
1522         coneNew[1] = splitp;
1523         PetscCall(DMPlexSetCone(sdm, hybedge, coneNew));
1524         for (e = 0, qf = 0; e < supportSize; ++e) {
1525           PetscInt val, edge;
1526 
1527           PetscCall(DMLabelGetValue(label, support[e], &val));
1528           if (val == 1) {
1529             PetscCall(PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &edge));
1530             PetscCheck(edge >= 0,comm, PETSC_ERR_ARG_WRONG, "Edge %" PetscInt_FMT " is not a split edge", support[e]);
1531             supportNew[qf++] = edge + pMaxNew[dep+2] + numSplitPoints[dep+2];
1532           }
1533         }
1534         PetscCall(DMPlexSetSupport(sdm, hybedge, supportNew));
1535       } else if (dep == dim-2) {
1536         const PetscInt hybface = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1537 
1538         /* Split old edge:   old vertices in cone so no change */
1539         /* Split new edge:   new vertices in cone */
1540         for (q = 0; q < coneSize; ++q) {
1541           PetscCall(PetscFindInt(cone[q], numSplitPoints[dep-1], splitPoints[dep-1], &v));
1542           if (v < 0) {
1543             PetscCall(PetscFindInt(cone[q], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &v));
1544             PetscCheck(v >= 0,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %" PetscInt_FMT " in split or unsplit points of depth %" PetscInt_FMT, cone[q], dep-1);
1545             coneNew[q] = DMPlexShiftPoint_Internal(cone[q], depth, depthShift) /*cone[q] + depthOffset[dep-1]*/;
1546           } else {
1547             coneNew[q] = v + pMaxNew[dep-1];
1548           }
1549         }
1550         PetscCall(DMPlexSetCone(sdm, splitp, coneNew));
1551         /* Split old edge: Faces in positive side cells and old split faces */
1552         for (e = 0, q = 0; e < supportSize; ++e) {
1553           PetscInt val;
1554 
1555           PetscCall(DMLabelGetValue(label, support[e], &val));
1556           if (val == dim-1) {
1557             supportNew[q++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1558           } else if (val == (shift + dim-1)) {
1559             supportNew[q++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1560           }
1561         }
1562         supportNew[q++] = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1563         PetscCall(DMPlexSetSupport(sdm, newp, supportNew));
1564         /* Split new edge: Faces in negative side cells and new split faces */
1565         for (e = 0, q = 0; e < supportSize; ++e) {
1566           PetscInt val, face;
1567 
1568           PetscCall(DMLabelGetValue(label, support[e], &val));
1569           if (val == dim-1) {
1570             PetscCall(PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &face));
1571             PetscCheck(face >= 0,comm, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " is not a split face", support[e]);
1572             supportNew[q++] = face + pMaxNew[dep+1];
1573           } else if (val == -(shift + dim-1)) {
1574             supportNew[q++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1575           }
1576         }
1577         supportNew[q++] = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1578         PetscCall(DMPlexSetSupport(sdm, splitp, supportNew));
1579         /* Hybrid face */
1580         coneNew[0] = newp;
1581         coneNew[1] = splitp;
1582         for (v = 0; v < coneSize; ++v) {
1583           PetscInt vertex;
1584           PetscCall(PetscFindInt(cone[v], numSplitPoints[dep-1], splitPoints[dep-1], &vertex));
1585           if (vertex < 0) {
1586             PetscCall(PetscFindInt(cone[v], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &vertex));
1587             PetscCheck(vertex >= 0,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %" PetscInt_FMT " in split or unsplit points of depth %" PetscInt_FMT, cone[v], dep-1);
1588             coneNew[2+v] = vertex + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep-1];
1589           } else {
1590             coneNew[2+v] = vertex + pMaxNew[dep] + numSplitPoints[dep];
1591           }
1592         }
1593         PetscCall(DMPlexSetCone(sdm, hybface, coneNew));
1594         for (e = 0, qf = 0; e < supportSize; ++e) {
1595           PetscInt val, face;
1596 
1597           PetscCall(DMLabelGetValue(label, support[e], &val));
1598           if (val == dim-1) {
1599             PetscCall(PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &face));
1600             PetscCheck(face >= 0,comm, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " is not a split face", support[e]);
1601             supportNew[qf++] = face + pMaxNew[dep+2] + numSplitPoints[dep+2];
1602           }
1603         }
1604         PetscCall(DMPlexSetSupport(sdm, hybface, supportNew));
1605       }
1606     }
1607   }
1608   for (dep = 0; dep <= depth; ++dep) {
1609     for (p = 0; p < numUnsplitPoints[dep]; ++p) {
1610       const PetscInt  oldp   = unsplitPoints[dep][p];
1611       const PetscInt  newp   = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*oldp + depthOffset[dep]*/;
1612       const PetscInt *cone, *support;
1613       PetscInt        coneSize, supportSize, supportSizeNew, q, qf, e, f, s;
1614 
1615       PetscCall(DMPlexGetConeSize(dm, oldp, &coneSize));
1616       PetscCall(DMPlexGetCone(dm, oldp, &cone));
1617       PetscCall(DMPlexGetSupportSize(dm, oldp, &supportSize));
1618       PetscCall(DMPlexGetSupport(dm, oldp, &support));
1619       if (dep == 0) {
1620         const PetscInt hybedge = p + pMaxNew[dep+1] + numSplitPoints[dep+1] + numSplitPoints[dep];
1621 
1622         /* Unsplit vertex */
1623         PetscCall(DMPlexGetSupportSize(sdm, newp, &supportSizeNew));
1624         for (s = 0, q = 0; s < supportSize; ++s) {
1625           supportNew[q++] = DMPlexShiftPoint_Internal(support[s], depth, depthShift) /*support[s] + depthOffset[dep+1]*/;
1626           PetscCall(PetscFindInt(support[s], numSplitPoints[dep+1], splitPoints[dep+1], &e));
1627           if (e >= 0) {
1628             supportNew[q++] = e + pMaxNew[dep+1];
1629           }
1630         }
1631         supportNew[q++] = hybedge;
1632         supportNew[q++] = hybedge;
1633         PetscCheck(q == supportSizeNew,comm, PETSC_ERR_ARG_WRONG, "Support size %" PetscInt_FMT " != %" PetscInt_FMT " for vertex %" PetscInt_FMT, q, supportSizeNew, newp);
1634         PetscCall(DMPlexSetSupport(sdm, newp, supportNew));
1635         /* Hybrid edge */
1636         coneNew[0] = newp;
1637         coneNew[1] = newp;
1638         PetscCall(DMPlexSetCone(sdm, hybedge, coneNew));
1639         for (e = 0, qf = 0; e < supportSize; ++e) {
1640           PetscInt val, edge;
1641 
1642           PetscCall(DMLabelGetValue(label, support[e], &val));
1643           if (val == 1) {
1644             PetscCall(PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &edge));
1645             PetscCheck(edge >= 0,comm, PETSC_ERR_ARG_WRONG, "Edge %" PetscInt_FMT " is not a split edge", support[e]);
1646             supportNew[qf++] = edge + pMaxNew[dep+2] + numSplitPoints[dep+2];
1647           } else if  (val ==  (shift2 + 1)) {
1648             PetscCall(PetscFindInt(support[e], numUnsplitPoints[dep+1], unsplitPoints[dep+1], &edge));
1649             PetscCheck(edge >= 0,comm, PETSC_ERR_ARG_WRONG, "Edge %" PetscInt_FMT " is not a unsplit edge", support[e]);
1650             supportNew[qf++] = edge + pMaxNew[dep+2] + numSplitPoints[dep+2] + numSplitPoints[dep+1];
1651           }
1652         }
1653         PetscCall(DMPlexSetSupport(sdm, hybedge, supportNew));
1654       } else if (dep == dim-2) {
1655         const PetscInt hybface = p + pMaxNew[dep+1] + numSplitPoints[dep+1] + numSplitPoints[dep];
1656 
1657         /* Unsplit edge: Faces into original edge, split face, and hybrid face twice */
1658         for (f = 0, qf = 0; f < supportSize; ++f) {
1659           PetscInt val, face;
1660 
1661           PetscCall(DMLabelGetValue(label, support[f], &val));
1662           if (val == dim-1) {
1663             PetscCall(PetscFindInt(support[f], numSplitPoints[dep+1], splitPoints[dep+1], &face));
1664             PetscCheck(face >= 0,comm, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " is not a split face", support[f]);
1665             supportNew[qf++] = DMPlexShiftPoint_Internal(support[f], depth, depthShift) /*support[f] + depthOffset[dep+1]*/;
1666             supportNew[qf++] = face + pMaxNew[dep+1];
1667           } else {
1668             supportNew[qf++] = DMPlexShiftPoint_Internal(support[f], depth, depthShift) /*support[f] + depthOffset[dep+1]*/;
1669           }
1670         }
1671         supportNew[qf++] = hybface;
1672         supportNew[qf++] = hybface;
1673         PetscCall(DMPlexGetSupportSize(sdm, newp, &supportSizeNew));
1674         PetscCheck(qf == supportSizeNew,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Support size for unsplit edge %" PetscInt_FMT " is %" PetscInt_FMT " != %" PetscInt_FMT, newp, qf, supportSizeNew);
1675         PetscCall(DMPlexSetSupport(sdm, newp, supportNew));
1676         /* Add hybrid face */
1677         coneNew[0] = newp;
1678         coneNew[1] = newp;
1679         PetscCall(PetscFindInt(cone[0], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &v));
1680         PetscCheck(v >= 0,comm, PETSC_ERR_ARG_WRONG, "Vertex %" PetscInt_FMT " is not an unsplit vertex", cone[0]);
1681         coneNew[2] = v + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep-1];
1682         PetscCall(PetscFindInt(cone[1], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &v));
1683         PetscCheck(v >= 0,comm, PETSC_ERR_ARG_WRONG, "Vertex %" PetscInt_FMT " is not an unsplit vertex", cone[1]);
1684         coneNew[3] = v + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep-1];
1685         PetscCall(DMPlexSetCone(sdm, hybface, coneNew));
1686         for (f = 0, qf = 0; f < supportSize; ++f) {
1687           PetscInt val, face;
1688 
1689           PetscCall(DMLabelGetValue(label, support[f], &val));
1690           if (val == dim-1) {
1691             PetscCall(PetscFindInt(support[f], numSplitPoints[dep+1], splitPoints[dep+1], &face));
1692             supportNew[qf++] = face + pMaxNew[dep+2] + numSplitPoints[dep+2];
1693           }
1694         }
1695         PetscCall(DMPlexGetSupportSize(sdm, hybface, &supportSizeNew));
1696         PetscCheck(qf == supportSizeNew,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Support size for hybrid face %" PetscInt_FMT " is %" PetscInt_FMT " != %" PetscInt_FMT, hybface, qf, supportSizeNew);
1697         PetscCall(DMPlexSetSupport(sdm, hybface, supportNew));
1698       }
1699     }
1700   }
1701   /* Step 6b: Replace split points in negative side cones */
1702   for (sp = 0; sp < numSP; ++sp) {
1703     PetscInt        dep = values[sp];
1704     IS              pIS;
1705     PetscInt        numPoints;
1706     const PetscInt *points;
1707 
1708     if (dep >= 0) continue;
1709     PetscCall(DMLabelGetStratumIS(label, dep, &pIS));
1710     if (!pIS) continue;
1711     dep  = -dep - shift;
1712     PetscCall(ISGetLocalSize(pIS, &numPoints));
1713     PetscCall(ISGetIndices(pIS, &points));
1714     for (p = 0; p < numPoints; ++p) {
1715       const PetscInt  oldp = points[p];
1716       const PetscInt  newp = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*depthOffset[dep] + oldp*/;
1717       const PetscInt *cone;
1718       PetscInt        coneSize, c;
1719       /* PetscBool       replaced = PETSC_FALSE; */
1720 
1721       /* Negative edge: replace split vertex */
1722       /* Negative cell: replace split face */
1723       PetscCall(DMPlexGetConeSize(sdm, newp, &coneSize));
1724       PetscCall(DMPlexGetCone(sdm, newp, &cone));
1725       for (c = 0; c < coneSize; ++c) {
1726         const PetscInt coldp = DMPlexShiftPointInverse_Internal(cone[c],depth,depthShift);
1727         PetscInt       csplitp, cp, val;
1728 
1729         PetscCall(DMLabelGetValue(label, coldp, &val));
1730         if (val == dep-1) {
1731           PetscCall(PetscFindInt(coldp, numSplitPoints[dep-1], splitPoints[dep-1], &cp));
1732           PetscCheck(cp >= 0,comm, PETSC_ERR_ARG_WRONG, "Point %" PetscInt_FMT " is not a split point of dimension %" PetscInt_FMT, oldp, dep-1);
1733           csplitp  = pMaxNew[dep-1] + cp;
1734           PetscCall(DMPlexInsertCone(sdm, newp, c, csplitp));
1735           /* replaced = PETSC_TRUE; */
1736         }
1737       }
1738       /* Cells with only a vertex or edge on the submesh have no replacement */
1739       /* PetscCheck(replaced,comm, PETSC_ERR_ARG_WRONG, "The cone of point %d does not contain split points", oldp); */
1740     }
1741     PetscCall(ISRestoreIndices(pIS, &points));
1742     PetscCall(ISDestroy(&pIS));
1743   }
1744   /* Step 7: Coordinates */
1745   PetscCall(DMPlexShiftCoordinates_Internal(dm, depthShift, sdm));
1746   PetscCall(DMGetCoordinateSection(sdm, &coordSection));
1747   PetscCall(DMGetCoordinatesLocal(sdm, &coordinates));
1748   PetscCall(VecGetArray(coordinates, &coords));
1749   for (v = 0; v < (numSplitPoints ? numSplitPoints[0] : 0); ++v) {
1750     const PetscInt newp   = DMPlexShiftPoint_Internal(splitPoints[0][v], depth, depthShift) /*depthOffset[0] + splitPoints[0][v]*/;
1751     const PetscInt splitp = pMaxNew[0] + v;
1752     PetscInt       dof, off, soff, d;
1753 
1754     PetscCall(PetscSectionGetDof(coordSection, newp, &dof));
1755     PetscCall(PetscSectionGetOffset(coordSection, newp, &off));
1756     PetscCall(PetscSectionGetOffset(coordSection, splitp, &soff));
1757     for (d = 0; d < dof; ++d) coords[soff+d] = coords[off+d];
1758   }
1759   PetscCall(VecRestoreArray(coordinates, &coords));
1760   /* Step 8: SF, if I can figure this out we can split the mesh in parallel */
1761   PetscCall(DMPlexShiftSF_Internal(dm, depthShift, sdm));
1762   /*   TODO We need to associate the ghost points with the correct replica */
1763   /* Step 9: Labels */
1764   PetscCall(DMPlexShiftLabels_Internal(dm, depthShift, sdm));
1765   PetscCall(DMPlexCreateVTKLabel_Internal(dm, PETSC_FALSE, sdm));
1766   PetscCall(DMGetNumLabels(sdm, &numLabels));
1767   for (dep = 0; dep <= depth; ++dep) {
1768     for (p = 0; p < numSplitPoints[dep]; ++p) {
1769       const PetscInt newp   = DMPlexShiftPoint_Internal(splitPoints[dep][p], depth, depthShift) /*depthOffset[dep] + splitPoints[dep][p]*/;
1770       const PetscInt splitp = pMaxNew[dep] + p;
1771       PetscInt       l;
1772 
1773       if (splitLabel) {
1774         const PetscInt val = 100 + dep;
1775 
1776         PetscCall(DMLabelSetValue(splitLabel, newp,    val));
1777         PetscCall(DMLabelSetValue(splitLabel, splitp, -val));
1778       }
1779       for (l = 0; l < numLabels; ++l) {
1780         DMLabel     mlabel;
1781         const char *lname;
1782         PetscInt    val;
1783         PetscBool   isDepth;
1784 
1785         PetscCall(DMGetLabelName(sdm, l, &lname));
1786         PetscCall(PetscStrcmp(lname, "depth", &isDepth));
1787         if (isDepth) continue;
1788         PetscCall(DMGetLabel(sdm, lname, &mlabel));
1789         PetscCall(DMLabelGetValue(mlabel, newp, &val));
1790         if (val >= 0) {
1791           PetscCall(DMLabelSetValue(mlabel, splitp, val));
1792         }
1793       }
1794     }
1795   }
1796   for (sp = 0; sp < numSP; ++sp) {
1797     const PetscInt dep = values[sp];
1798 
1799     if ((dep < 0) || (dep > depth)) continue;
1800     if (splitIS[dep]) PetscCall(ISRestoreIndices(splitIS[dep], &splitPoints[dep]));
1801     PetscCall(ISDestroy(&splitIS[dep]));
1802     if (unsplitIS[dep]) PetscCall(ISRestoreIndices(unsplitIS[dep], &unsplitPoints[dep]));
1803     PetscCall(ISDestroy(&unsplitIS[dep]));
1804   }
1805   if (ghostIS) PetscCall(ISRestoreIndices(ghostIS, &ghostPoints));
1806   PetscCall(ISDestroy(&ghostIS));
1807   if (label) {
1808     PetscCall(ISRestoreIndices(valueIS, &values));
1809     PetscCall(ISDestroy(&valueIS));
1810   }
1811   for (d = 0; d <= depth; ++d) {
1812     PetscCall(DMPlexGetDepthStratum(sdm, d, NULL, &pEnd));
1813     pMaxNew[d] = pEnd - numHybridPoints[d] - numHybridPointsOld[d];
1814   }
1815   PetscCall(PetscFree3(coneNew, coneONew, supportNew));
1816   PetscCall(PetscFree5(depthMax, depthEnd, depthShift, pMaxNew, numHybridPointsOld));
1817   PetscCall(PetscFree7(splitIS, unsplitIS, numSplitPoints, numUnsplitPoints, numHybridPoints, splitPoints, unsplitPoints));
1818   PetscFunctionReturn(0);
1819 }
1820 
1821 /*@C
1822   DMPlexConstructCohesiveCells - Construct cohesive cells which split the face along an internal interface
1823 
1824   Collective on dm
1825 
1826   Input Parameters:
1827 + dm - The original DM
1828 - label - The label specifying the boundary faces (this could be auto-generated)
1829 
1830   Output Parameters:
1831 + splitLabel - The label containing the split points, or NULL if no output is desired
1832 - dmSplit - The new DM
1833 
1834   Level: developer
1835 
1836 .seealso: `DMCreate()`, `DMPlexLabelCohesiveComplete()`
1837 @*/
1838 PetscErrorCode DMPlexConstructCohesiveCells(DM dm, DMLabel label, DMLabel splitLabel, DM *dmSplit)
1839 {
1840   DM             sdm;
1841   PetscInt       dim;
1842 
1843   PetscFunctionBegin;
1844   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1845   PetscValidPointer(dmSplit, 4);
1846   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &sdm));
1847   PetscCall(DMSetType(sdm, DMPLEX));
1848   PetscCall(DMGetDimension(dm, &dim));
1849   PetscCall(DMSetDimension(sdm, dim));
1850   switch (dim) {
1851   case 2:
1852   case 3:
1853     PetscCall(DMPlexConstructCohesiveCells_Internal(dm, label, splitLabel, sdm));
1854     break;
1855   default:
1856     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cannot construct cohesive cells for dimension %" PetscInt_FMT, dim);
1857   }
1858   *dmSplit = sdm;
1859   PetscFunctionReturn(0);
1860 }
1861 
1862 /* Returns the side of the surface for a given cell with a face on the surface */
1863 static PetscErrorCode GetSurfaceSide_Static(DM dm, DM subdm, PetscInt numSubpoints, const PetscInt *subpoints, PetscInt cell, PetscInt face, PetscBool *pos)
1864 {
1865   const PetscInt *cone, *ornt;
1866   PetscInt        dim, coneSize, c;
1867 
1868   PetscFunctionBegin;
1869   *pos = PETSC_TRUE;
1870   PetscCall(DMGetDimension(dm, &dim));
1871   PetscCall(DMPlexGetConeSize(dm, cell, &coneSize));
1872   PetscCall(DMPlexGetCone(dm, cell, &cone));
1873   PetscCall(DMPlexGetConeOrientation(dm, cell, &ornt));
1874   for (c = 0; c < coneSize; ++c) {
1875     if (cone[c] == face) {
1876       PetscInt o = ornt[c];
1877 
1878       if (subdm) {
1879         const PetscInt *subcone, *subornt;
1880         PetscInt        subpoint, subface, subconeSize, sc;
1881 
1882         PetscCall(PetscFindInt(cell, numSubpoints, subpoints, &subpoint));
1883         PetscCall(PetscFindInt(face, numSubpoints, subpoints, &subface));
1884         PetscCall(DMPlexGetConeSize(subdm, subpoint, &subconeSize));
1885         PetscCall(DMPlexGetCone(subdm, subpoint, &subcone));
1886         PetscCall(DMPlexGetConeOrientation(subdm, subpoint, &subornt));
1887         for (sc = 0; sc < subconeSize; ++sc) {
1888           if (subcone[sc] == subface) {
1889             o = subornt[0];
1890             break;
1891           }
1892         }
1893         PetscCheck(sc < subconeSize,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not find subpoint %" PetscInt_FMT " (%" PetscInt_FMT ") in cone for subpoint %" PetscInt_FMT " (%" PetscInt_FMT ")", subface, face, subpoint, cell);
1894       }
1895       if (o >= 0) *pos = PETSC_TRUE;
1896       else        *pos = PETSC_FALSE;
1897       break;
1898     }
1899   }
1900   PetscCheck(c != coneSize,PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Cell %" PetscInt_FMT " in split face %" PetscInt_FMT " support does not have it in the cone", cell, face);
1901   PetscFunctionReturn(0);
1902 }
1903 
1904 static PetscErrorCode CheckFaultEdge_Private(DM dm, DMLabel label)
1905 {
1906   IS              facePosIS, faceNegIS, dimIS;
1907   const PetscInt *points;
1908   PetscInt        dim, numPoints, p, shift = 100, shift2 = 200;
1909 
1910   PetscFunctionBegin;
1911   PetscCall(DMGetDimension(dm, &dim));
1912   /* If any faces touching the fault divide cells on either side, split them */
1913   PetscCall(DMLabelGetStratumIS(label,   shift+dim-1,  &facePosIS));
1914   PetscCall(DMLabelGetStratumIS(label, -(shift+dim-1), &faceNegIS));
1915   if (!facePosIS || !faceNegIS) {
1916     PetscCall(ISDestroy(&facePosIS));
1917     PetscCall(ISDestroy(&faceNegIS));
1918     PetscFunctionReturn(0);
1919   }
1920   PetscCall(ISExpand(facePosIS, faceNegIS, &dimIS));
1921   PetscCall(ISDestroy(&facePosIS));
1922   PetscCall(ISDestroy(&faceNegIS));
1923   PetscCall(ISGetLocalSize(dimIS, &numPoints));
1924   PetscCall(ISGetIndices(dimIS, &points));
1925   for (p = 0; p < numPoints; ++p) {
1926     const PetscInt  point = points[p];
1927     const PetscInt *support;
1928     PetscInt        supportSize, valA, valB;
1929 
1930     PetscCall(DMPlexGetSupportSize(dm, point, &supportSize));
1931     if (supportSize != 2) continue;
1932     PetscCall(DMPlexGetSupport(dm, point, &support));
1933     PetscCall(DMLabelGetValue(label, support[0], &valA));
1934     PetscCall(DMLabelGetValue(label, support[1], &valB));
1935     if ((valA == -1) || (valB == -1)) continue;
1936     if (valA*valB > 0) continue;
1937     /* Check that this face is not incident on only unsplit faces, meaning has at least one split face */
1938     {
1939       PetscInt *closure = NULL;
1940       PetscBool split   = PETSC_FALSE;
1941       PetscInt  closureSize, cl;
1942 
1943       PetscCall(DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
1944       for (cl = 0; cl < closureSize*2; cl += 2) {
1945         PetscCall(DMLabelGetValue(label, closure[cl], &valA));
1946         if ((valA >= 0) && (valA <= dim)) {split = PETSC_TRUE; break;}
1947       }
1948       PetscCall(DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
1949       if (!split) continue;
1950     }
1951     /* Split the face */
1952     PetscCall(DMLabelGetValue(label, point, &valA));
1953     PetscCall(DMLabelClearValue(label, point, valA));
1954     PetscCall(DMLabelSetValue(label, point, dim-1));
1955     /* Label its closure:
1956       unmarked: label as unsplit
1957       incident: relabel as split
1958       split:    do nothing
1959     */
1960     {
1961       PetscInt *closure = NULL;
1962       PetscInt  closureSize, cl, dep;
1963 
1964       PetscCall(DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
1965       for (cl = 0; cl < closureSize*2; cl += 2) {
1966         PetscCall(DMLabelGetValue(label, closure[cl], &valA));
1967         if (valA == -1) { /* Mark as unsplit */
1968           PetscCall(DMPlexGetPointDepth(dm, closure[cl], &dep));
1969           PetscCall(DMLabelSetValue(label, closure[cl], shift2+dep));
1970         } else if (((valA >= shift) && (valA < shift2)) || ((valA <= -shift) && (valA > -shift2))) {
1971           PetscCall(DMPlexGetPointDepth(dm, closure[cl], &dep));
1972           PetscCall(DMLabelClearValue(label, closure[cl], valA));
1973           PetscCall(DMLabelSetValue(label, closure[cl], dep));
1974         }
1975       }
1976       PetscCall(DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
1977     }
1978   }
1979   PetscCall(ISRestoreIndices(dimIS, &points));
1980   PetscCall(ISDestroy(&dimIS));
1981   PetscFunctionReturn(0);
1982 }
1983 
1984 /*@
1985   DMPlexLabelCohesiveComplete - Starting with a label marking points on an internal surface, we add all other mesh pieces
1986   to complete the surface
1987 
1988   Input Parameters:
1989 + dm     - The DM
1990 . label  - A DMLabel marking the surface
1991 . blabel - A DMLabel marking the vertices on the boundary which will not be duplicated, or NULL to find them automatically
1992 . bvalue - Value of DMLabel marking the vertices on the boundary
1993 . flip   - Flag to flip the submesh normal and replace points on the other side
1994 - subdm  - The subDM associated with the label, or NULL
1995 
1996   Output Parameter:
1997 . label - A DMLabel marking all surface points
1998 
1999   Note: The vertices in blabel are called "unsplit" in the terminology from hybrid cell creation.
2000 
2001   Level: developer
2002 
2003 .seealso: `DMPlexConstructCohesiveCells()`, `DMPlexLabelComplete()`
2004 @*/
2005 PetscErrorCode DMPlexLabelCohesiveComplete(DM dm, DMLabel label, DMLabel blabel, PetscInt bvalue, PetscBool flip, DM subdm)
2006 {
2007   DMLabel         depthLabel;
2008   IS              dimIS, subpointIS = NULL;
2009   const PetscInt *points, *subpoints;
2010   const PetscInt  rev   = flip ? -1 : 1;
2011   PetscInt        shift = 100, shift2 = 200, shift3 = 300, dim, depth, numPoints, numSubpoints, p, val;
2012 
2013   PetscFunctionBegin;
2014   PetscCall(DMPlexGetDepth(dm, &depth));
2015   PetscCall(DMGetDimension(dm, &dim));
2016   PetscCall(DMPlexGetDepthLabel(dm, &depthLabel));
2017   if (subdm) {
2018     PetscCall(DMPlexGetSubpointIS(subdm, &subpointIS));
2019     if (subpointIS) {
2020       PetscCall(ISGetLocalSize(subpointIS, &numSubpoints));
2021       PetscCall(ISGetIndices(subpointIS, &subpoints));
2022     }
2023   }
2024   /* Mark cell on the fault, and its faces which touch the fault: cell orientation for face gives the side of the fault */
2025   PetscCall(DMLabelGetStratumIS(label, dim-1, &dimIS));
2026   if (!dimIS) goto divide;
2027   PetscCall(ISGetLocalSize(dimIS, &numPoints));
2028   PetscCall(ISGetIndices(dimIS, &points));
2029   for (p = 0; p < numPoints; ++p) { /* Loop over fault faces */
2030     const PetscInt *support;
2031     PetscInt        supportSize, s;
2032 
2033     PetscCall(DMPlexGetSupportSize(dm, points[p], &supportSize));
2034 #if 0
2035     if (supportSize != 2) {
2036       const PetscInt *lp;
2037       PetscInt        Nlp, pind;
2038 
2039       /* Check that for a cell with a single support face, that face is in the SF */
2040       /*   THis check only works for the remote side. We would need root side information */
2041       PetscCall(PetscSFGetGraph(dm->sf, NULL, &Nlp, &lp, NULL));
2042       PetscCall(PetscFindInt(points[p], Nlp, lp, &pind));
2043       PetscCheck(pind >= 0,PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Split face %" PetscInt_FMT " has %" PetscInt_FMT " != 2 supports, and the face is not shared with another process", points[p], supportSize);
2044     }
2045 #endif
2046     PetscCall(DMPlexGetSupport(dm, points[p], &support));
2047     for (s = 0; s < supportSize; ++s) {
2048       const PetscInt *cone;
2049       PetscInt        coneSize, c;
2050       PetscBool       pos;
2051 
2052       PetscCall(GetSurfaceSide_Static(dm, subdm, numSubpoints, subpoints, support[s], points[p], &pos));
2053       if (pos) PetscCall(DMLabelSetValue(label, support[s],  rev*(shift+dim)));
2054       else     PetscCall(DMLabelSetValue(label, support[s], -rev*(shift+dim)));
2055       if (rev < 0) pos = !pos ? PETSC_TRUE : PETSC_FALSE;
2056       /* Put faces touching the fault in the label */
2057       PetscCall(DMPlexGetConeSize(dm, support[s], &coneSize));
2058       PetscCall(DMPlexGetCone(dm, support[s], &cone));
2059       for (c = 0; c < coneSize; ++c) {
2060         const PetscInt point = cone[c];
2061 
2062         PetscCall(DMLabelGetValue(label, point, &val));
2063         if (val == -1) {
2064           PetscInt *closure = NULL;
2065           PetscInt  closureSize, cl;
2066 
2067           PetscCall(DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
2068           for (cl = 0; cl < closureSize*2; cl += 2) {
2069             const PetscInt clp  = closure[cl];
2070             PetscInt       bval = -1;
2071 
2072             PetscCall(DMLabelGetValue(label, clp, &val));
2073             if (blabel) PetscCall(DMLabelGetValue(blabel, clp, &bval));
2074             if ((val >= 0) && (val < dim-1) && (bval < 0)) {
2075               PetscCall(DMLabelSetValue(label, point, pos == PETSC_TRUE ? shift+dim-1 : -(shift+dim-1)));
2076               break;
2077             }
2078           }
2079           PetscCall(DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
2080         }
2081       }
2082     }
2083   }
2084   PetscCall(ISRestoreIndices(dimIS, &points));
2085   PetscCall(ISDestroy(&dimIS));
2086   /* Mark boundary points as unsplit */
2087   if (blabel) {
2088     IS bdIS;
2089 
2090     PetscCall(DMLabelGetStratumIS(blabel, bvalue, &bdIS));
2091     PetscCall(ISGetLocalSize(bdIS, &numPoints));
2092     PetscCall(ISGetIndices(bdIS, &points));
2093     for (p = 0; p < numPoints; ++p) {
2094       const PetscInt point = points[p];
2095       PetscInt       val, bval;
2096 
2097       PetscCall(DMLabelGetValue(blabel, point, &bval));
2098       if (bval >= 0) {
2099         PetscCall(DMLabelGetValue(label, point, &val));
2100         if ((val < 0) || (val > dim)) {
2101           /* This could be a point added from splitting a vertex on an adjacent fault, otherwise its just wrong */
2102           PetscCall(DMLabelClearValue(blabel, point, bval));
2103         }
2104       }
2105     }
2106     for (p = 0; p < numPoints; ++p) {
2107       const PetscInt point = points[p];
2108       PetscInt       val, bval;
2109 
2110       PetscCall(DMLabelGetValue(blabel, point, &bval));
2111       if (bval >= 0) {
2112         const PetscInt *cone,    *support;
2113         PetscInt        coneSize, supportSize, s, valA, valB, valE;
2114 
2115         /* Mark as unsplit */
2116         PetscCall(DMLabelGetValue(label, point, &val));
2117         PetscCheck(val >= 0 && val <= dim, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %" PetscInt_FMT " has label value %" PetscInt_FMT ", should be part of the fault", point, val);
2118         PetscCall(DMLabelClearValue(label, point, val));
2119         PetscCall(DMLabelSetValue(label, point, shift2+val));
2120         /* Check for cross-edge
2121              A cross-edge has endpoints which are both on the boundary of the surface, but the edge itself is not. */
2122         if (val != 0) continue;
2123         PetscCall(DMPlexGetSupport(dm, point, &support));
2124         PetscCall(DMPlexGetSupportSize(dm, point, &supportSize));
2125         for (s = 0; s < supportSize; ++s) {
2126           PetscCall(DMPlexGetCone(dm, support[s], &cone));
2127           PetscCall(DMPlexGetConeSize(dm, support[s], &coneSize));
2128           PetscCheck(coneSize == 2, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Edge %" PetscInt_FMT " has %" PetscInt_FMT " vertices != 2", support[s], coneSize);
2129           PetscCall(DMLabelGetValue(blabel, cone[0], &valA));
2130           PetscCall(DMLabelGetValue(blabel, cone[1], &valB));
2131           PetscCall(DMLabelGetValue(blabel, support[s], &valE));
2132           if ((valE < 0) && (valA >= 0) && (valB >= 0) && (cone[0] != cone[1])) PetscCall(DMLabelSetValue(blabel, support[s], 2));
2133         }
2134       }
2135     }
2136     PetscCall(ISRestoreIndices(bdIS, &points));
2137     PetscCall(ISDestroy(&bdIS));
2138   }
2139   /* Mark ghost fault cells */
2140   {
2141     PetscSF         sf;
2142     const PetscInt *leaves;
2143     PetscInt         Nl, l;
2144 
2145     PetscCall(DMGetPointSF(dm, &sf));
2146     PetscCall(PetscSFGetGraph(sf, NULL, &Nl, &leaves, NULL));
2147     PetscCall(DMLabelGetStratumIS(label, dim-1, &dimIS));
2148     if (!dimIS) goto divide;
2149     PetscCall(ISGetLocalSize(dimIS, &numPoints));
2150     PetscCall(ISGetIndices(dimIS, &points));
2151     if (Nl > 0) {
2152       for (p = 0; p < numPoints; ++p) {
2153         const PetscInt point = points[p];
2154         PetscInt       val;
2155 
2156         PetscCall(PetscFindInt(point, Nl, leaves, &l));
2157         if (l >= 0) {
2158           PetscInt *closure = NULL;
2159           PetscInt  closureSize, cl;
2160 
2161           PetscCall(DMLabelGetValue(label, point, &val));
2162           PetscCheck((val == dim-1) || (val == shift2+dim-1), PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %" PetscInt_FMT " has label value %" PetscInt_FMT ", should be a fault face", point, val);
2163           PetscCall(DMLabelClearValue(label, point, val));
2164           PetscCall(DMLabelSetValue(label, point, shift3+val));
2165           PetscCall(DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
2166           for (cl = 2; cl < closureSize*2; cl += 2) {
2167             const PetscInt clp  = closure[cl];
2168 
2169             PetscCall(DMLabelGetValue(label, clp, &val));
2170             PetscCheck(val != -1, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %" PetscInt_FMT " is missing from label, but is in the closure of a fault face", point);
2171             PetscCall(DMLabelClearValue(label, clp, val));
2172             PetscCall(DMLabelSetValue(label, clp, shift3+val));
2173           }
2174           PetscCall(DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
2175         }
2176       }
2177     }
2178     PetscCall(ISRestoreIndices(dimIS, &points));
2179     PetscCall(ISDestroy(&dimIS));
2180   }
2181   divide:
2182   if (subpointIS) PetscCall(ISRestoreIndices(subpointIS, &subpoints));
2183   PetscCall(DMPlexLabelFaultHalo(dm, label));
2184   PetscCall(CheckFaultEdge_Private(dm, label));
2185   PetscFunctionReturn(0);
2186 }
2187 
2188 /* Check that no cell have all vertices on the fault */
2189 PetscErrorCode DMPlexCheckValidSubmesh_Private(DM dm, DMLabel label, DM subdm)
2190 {
2191   IS              subpointIS;
2192   const PetscInt *dmpoints;
2193   PetscInt        defaultValue, cStart, cEnd, c, vStart, vEnd;
2194 
2195   PetscFunctionBegin;
2196   if (!label) PetscFunctionReturn(0);
2197   PetscCall(DMLabelGetDefaultValue(label, &defaultValue));
2198   PetscCall(DMPlexGetSubpointIS(subdm, &subpointIS));
2199   if (!subpointIS) PetscFunctionReturn(0);
2200   PetscCall(DMPlexGetHeightStratum(subdm, 0, &cStart, &cEnd));
2201   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
2202   PetscCall(ISGetIndices(subpointIS, &dmpoints));
2203   for (c = cStart; c < cEnd; ++c) {
2204     PetscBool invalidCell = PETSC_TRUE;
2205     PetscInt *closure     = NULL;
2206     PetscInt  closureSize, cl;
2207 
2208     PetscCall(DMPlexGetTransitiveClosure(dm, dmpoints[c], PETSC_TRUE, &closureSize, &closure));
2209     for (cl = 0; cl < closureSize*2; cl += 2) {
2210       PetscInt value = 0;
2211 
2212       if ((closure[cl] < vStart) || (closure[cl] >= vEnd)) continue;
2213       PetscCall(DMLabelGetValue(label, closure[cl], &value));
2214       if (value == defaultValue) {invalidCell = PETSC_FALSE; break;}
2215     }
2216     PetscCall(DMPlexRestoreTransitiveClosure(dm, dmpoints[c], PETSC_TRUE, &closureSize, &closure));
2217     if (invalidCell) {
2218       PetscCall(ISRestoreIndices(subpointIS, &dmpoints));
2219       PetscCall(ISDestroy(&subpointIS));
2220       PetscCall(DMDestroy(&subdm));
2221       SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Ambiguous submesh. Cell %" PetscInt_FMT " has all of its vertices on the submesh.", dmpoints[c]);
2222     }
2223   }
2224   PetscCall(ISRestoreIndices(subpointIS, &dmpoints));
2225   PetscFunctionReturn(0);
2226 }
2227 
2228 /*@
2229   DMPlexCreateHybridMesh - Create a mesh with hybrid cells along an internal interface
2230 
2231   Collective on dm
2232 
2233   Input Parameters:
2234 + dm - The original DM
2235 . label - The label specifying the interface vertices
2236 . bdlabel - The optional label specifying the interface boundary vertices
2237 - bdvalue - Value of optional label specifying the interface boundary vertices
2238 
2239   Output Parameters:
2240 + hybridLabel - The label fully marking the interface, or NULL if no output is desired
2241 . splitLabel - The label containing the split points, or NULL if no output is desired
2242 . dmInterface - The new interface DM, or NULL
2243 - dmHybrid - The new DM with cohesive cells
2244 
2245   Note: The hybridLabel indicates what parts of the original mesh impinged on the on division surface. For points
2246   directly on the division surface, they are labeled with their dimension, so an edge 7 on the division surface would be
2247   7 (1) in hybridLabel. For points that impinge from the positive side, they are labeled with 100+dim, so an edge 6 with
2248   one vertex 3 on the surface would be 6 (101) and 3 (0) in hybridLabel. If an edge 9 from the negative side of the
2249   surface also hits vertex 3, it would be 9 (-101) in hybridLabel.
2250 
2251   The splitLabel indicates what points in the new hybrid mesh were the result of splitting points in the original
2252   mesh. The label value is +=100+dim for each point. For example, if two edges 10 and 14 in the hybrid resulting from
2253   splitting an edge in the original mesh, you would have 10 (101) and 14 (-101) in the splitLabel.
2254 
2255   The dmInterface is a DM built from the original division surface. It has a label which can be retrieved using
2256   DMPlexGetSubpointMap() which maps each point back to the point in the surface of the original mesh.
2257 
2258   Level: developer
2259 
2260 .seealso: `DMPlexConstructCohesiveCells()`, `DMPlexLabelCohesiveComplete()`, `DMPlexGetSubpointMap()`, `DMCreate()`
2261 @*/
2262 PetscErrorCode DMPlexCreateHybridMesh(DM dm, DMLabel label, DMLabel bdlabel, PetscInt bdvalue, DMLabel *hybridLabel, DMLabel *splitLabel, DM *dmInterface, DM *dmHybrid)
2263 {
2264   DM             idm;
2265   DMLabel        subpointMap, hlabel, slabel = NULL;
2266   PetscInt       dim;
2267 
2268   PetscFunctionBegin;
2269   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2270   if (label) PetscValidPointer(label, 2);
2271   if (bdlabel) PetscValidPointer(bdlabel, 3);
2272   if (hybridLabel) PetscValidPointer(hybridLabel, 5);
2273   if (splitLabel)  PetscValidPointer(splitLabel, 6);
2274   if (dmInterface) PetscValidPointer(dmInterface, 7);
2275   PetscValidPointer(dmHybrid, 8);
2276   PetscCall(DMGetDimension(dm, &dim));
2277   PetscCall(DMPlexCreateSubmesh(dm, label, 1, PETSC_FALSE, &idm));
2278   PetscCall(DMPlexCheckValidSubmesh_Private(dm, label, idm));
2279   PetscCall(DMPlexOrient(idm));
2280   PetscCall(DMPlexGetSubpointMap(idm, &subpointMap));
2281   PetscCall(DMLabelDuplicate(subpointMap, &hlabel));
2282   PetscCall(DMLabelClearStratum(hlabel, dim));
2283   if (splitLabel) {
2284     const char *name;
2285     char        sname[PETSC_MAX_PATH_LEN];
2286 
2287     PetscCall(PetscObjectGetName((PetscObject) hlabel, &name));
2288     PetscCall(PetscStrncpy(sname, name, PETSC_MAX_PATH_LEN));
2289     PetscCall(PetscStrcat(sname, " split"));
2290     PetscCall(DMLabelCreate(PETSC_COMM_SELF, sname, &slabel));
2291   }
2292   PetscCall(DMPlexLabelCohesiveComplete(dm, hlabel, bdlabel, bdvalue, PETSC_FALSE, idm));
2293   if (dmInterface) {*dmInterface = idm;}
2294   else             PetscCall(DMDestroy(&idm));
2295   PetscCall(DMPlexConstructCohesiveCells(dm, hlabel, slabel, dmHybrid));
2296   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_TRUE, *dmHybrid));
2297   if (hybridLabel) *hybridLabel = hlabel;
2298   else             PetscCall(DMLabelDestroy(&hlabel));
2299   if (splitLabel)  *splitLabel  = slabel;
2300   {
2301     DM      cdm;
2302     DMLabel ctLabel;
2303 
2304     /* We need to somehow share the celltype label with the coordinate dm */
2305     PetscCall(DMGetCoordinateDM(*dmHybrid, &cdm));
2306     PetscCall(DMPlexGetCellTypeLabel(*dmHybrid, &ctLabel));
2307     PetscCall(DMSetLabel(cdm, ctLabel));
2308   }
2309   PetscFunctionReturn(0);
2310 }
2311 
2312 /* Here we need the explicit assumption that:
2313 
2314      For any marked cell, the marked vertices constitute a single face
2315 */
2316 static PetscErrorCode DMPlexMarkSubmesh_Uninterpolated(DM dm, DMLabel vertexLabel, PetscInt value, DMLabel subpointMap, PetscInt *numFaces, PetscInt *nFV, DM subdm)
2317 {
2318   IS               subvertexIS = NULL;
2319   const PetscInt  *subvertices;
2320   PetscInt        *pStart, *pEnd, pSize;
2321   PetscInt         depth, dim, d, numSubVerticesInitial = 0, v;
2322 
2323   PetscFunctionBegin;
2324   *numFaces = 0;
2325   *nFV      = 0;
2326   PetscCall(DMPlexGetDepth(dm, &depth));
2327   PetscCall(DMGetDimension(dm, &dim));
2328   pSize = PetscMax(depth, dim) + 1;
2329   PetscCall(PetscMalloc2(pSize, &pStart, pSize, &pEnd));
2330   for (d = 0; d <= depth; ++d) {
2331     PetscCall(DMPlexGetSimplexOrBoxCells(dm, depth-d, &pStart[d], &pEnd[d]));
2332   }
2333   /* Loop over initial vertices and mark all faces in the collective star() */
2334   if (vertexLabel) PetscCall(DMLabelGetStratumIS(vertexLabel, value, &subvertexIS));
2335   if (subvertexIS) {
2336     PetscCall(ISGetSize(subvertexIS, &numSubVerticesInitial));
2337     PetscCall(ISGetIndices(subvertexIS, &subvertices));
2338   }
2339   for (v = 0; v < numSubVerticesInitial; ++v) {
2340     const PetscInt vertex = subvertices[v];
2341     PetscInt      *star   = NULL;
2342     PetscInt       starSize, s, numCells = 0, c;
2343 
2344     PetscCall(DMPlexGetTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star));
2345     for (s = 0; s < starSize*2; s += 2) {
2346       const PetscInt point = star[s];
2347       if ((point >= pStart[depth]) && (point < pEnd[depth])) star[numCells++] = point;
2348     }
2349     for (c = 0; c < numCells; ++c) {
2350       const PetscInt cell    = star[c];
2351       PetscInt      *closure = NULL;
2352       PetscInt       closureSize, cl;
2353       PetscInt       cellLoc, numCorners = 0, faceSize = 0;
2354 
2355       PetscCall(DMLabelGetValue(subpointMap, cell, &cellLoc));
2356       if (cellLoc == 2) continue;
2357       PetscCheck(cellLoc < 0,PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Cell %" PetscInt_FMT " has dimension %" PetscInt_FMT " in the surface label", cell, cellLoc);
2358       PetscCall(DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
2359       for (cl = 0; cl < closureSize*2; cl += 2) {
2360         const PetscInt point = closure[cl];
2361         PetscInt       vertexLoc;
2362 
2363         if ((point >= pStart[0]) && (point < pEnd[0])) {
2364           ++numCorners;
2365           PetscCall(DMLabelGetValue(vertexLabel, point, &vertexLoc));
2366           if (vertexLoc == value) closure[faceSize++] = point;
2367         }
2368       }
2369       if (!(*nFV)) PetscCall(DMPlexGetNumFaceVertices(dm, dim, numCorners, nFV));
2370       PetscCheck(faceSize <= *nFV,PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Invalid submesh: Too many vertices %" PetscInt_FMT " of an element on the surface", faceSize);
2371       if (faceSize == *nFV) {
2372         const PetscInt *cells = NULL;
2373         PetscInt        numCells, nc;
2374 
2375         ++(*numFaces);
2376         for (cl = 0; cl < faceSize; ++cl) {
2377           PetscCall(DMLabelSetValue(subpointMap, closure[cl], 0));
2378         }
2379         PetscCall(DMPlexGetJoin(dm, faceSize, closure, &numCells, &cells));
2380         for (nc = 0; nc < numCells; ++nc) {
2381           PetscCall(DMLabelSetValue(subpointMap, cells[nc], 2));
2382         }
2383         PetscCall(DMPlexRestoreJoin(dm, faceSize, closure, &numCells, &cells));
2384       }
2385       PetscCall(DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
2386     }
2387     PetscCall(DMPlexRestoreTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star));
2388   }
2389   if (subvertexIS) {
2390     PetscCall(ISRestoreIndices(subvertexIS, &subvertices));
2391   }
2392   PetscCall(ISDestroy(&subvertexIS));
2393   PetscCall(PetscFree2(pStart, pEnd));
2394   PetscFunctionReturn(0);
2395 }
2396 
2397 static PetscErrorCode DMPlexMarkSubmesh_Interpolated(DM dm, DMLabel vertexLabel, PetscInt value, PetscBool markedFaces, DMLabel subpointMap, DM subdm)
2398 {
2399   IS               subvertexIS = NULL;
2400   const PetscInt  *subvertices;
2401   PetscInt        *pStart, *pEnd;
2402   PetscInt         dim, d, numSubVerticesInitial = 0, v;
2403 
2404   PetscFunctionBegin;
2405   PetscCall(DMGetDimension(dm, &dim));
2406   PetscCall(PetscMalloc2(dim+1, &pStart, dim+1, &pEnd));
2407   for (d = 0; d <= dim; ++d) {
2408     PetscCall(DMPlexGetSimplexOrBoxCells(dm, dim-d, &pStart[d], &pEnd[d]));
2409   }
2410   /* Loop over initial vertices and mark all faces in the collective star() */
2411   if (vertexLabel) {
2412     PetscCall(DMLabelGetStratumIS(vertexLabel, value, &subvertexIS));
2413     if (subvertexIS) {
2414       PetscCall(ISGetSize(subvertexIS, &numSubVerticesInitial));
2415       PetscCall(ISGetIndices(subvertexIS, &subvertices));
2416     }
2417   }
2418   for (v = 0; v < numSubVerticesInitial; ++v) {
2419     const PetscInt vertex = subvertices[v];
2420     PetscInt      *star   = NULL;
2421     PetscInt       starSize, s, numFaces = 0, f;
2422 
2423     PetscCall(DMPlexGetTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star));
2424     for (s = 0; s < starSize*2; s += 2) {
2425       const PetscInt point = star[s];
2426       PetscInt       faceLoc;
2427 
2428       if ((point >= pStart[dim-1]) && (point < pEnd[dim-1])) {
2429         if (markedFaces) {
2430           PetscCall(DMLabelGetValue(vertexLabel, point, &faceLoc));
2431           if (faceLoc < 0) continue;
2432         }
2433         star[numFaces++] = point;
2434       }
2435     }
2436     for (f = 0; f < numFaces; ++f) {
2437       const PetscInt face    = star[f];
2438       PetscInt      *closure = NULL;
2439       PetscInt       closureSize, c;
2440       PetscInt       faceLoc;
2441 
2442       PetscCall(DMLabelGetValue(subpointMap, face, &faceLoc));
2443       if (faceLoc == dim-1) continue;
2444       PetscCheck(faceLoc < 0,PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Face %" PetscInt_FMT " has dimension %" PetscInt_FMT " in the surface label", face, faceLoc);
2445       PetscCall(DMPlexGetTransitiveClosure(dm, face, PETSC_TRUE, &closureSize, &closure));
2446       for (c = 0; c < closureSize*2; c += 2) {
2447         const PetscInt point = closure[c];
2448         PetscInt       vertexLoc;
2449 
2450         if ((point >= pStart[0]) && (point < pEnd[0])) {
2451           PetscCall(DMLabelGetValue(vertexLabel, point, &vertexLoc));
2452           if (vertexLoc != value) break;
2453         }
2454       }
2455       if (c == closureSize*2) {
2456         const PetscInt *support;
2457         PetscInt        supportSize, s;
2458 
2459         for (c = 0; c < closureSize*2; c += 2) {
2460           const PetscInt point = closure[c];
2461 
2462           for (d = 0; d < dim; ++d) {
2463             if ((point >= pStart[d]) && (point < pEnd[d])) {
2464               PetscCall(DMLabelSetValue(subpointMap, point, d));
2465               break;
2466             }
2467           }
2468         }
2469         PetscCall(DMPlexGetSupportSize(dm, face, &supportSize));
2470         PetscCall(DMPlexGetSupport(dm, face, &support));
2471         for (s = 0; s < supportSize; ++s) {
2472           PetscCall(DMLabelSetValue(subpointMap, support[s], dim));
2473         }
2474       }
2475       PetscCall(DMPlexRestoreTransitiveClosure(dm, face, PETSC_TRUE, &closureSize, &closure));
2476     }
2477     PetscCall(DMPlexRestoreTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star));
2478   }
2479   if (subvertexIS) PetscCall(ISRestoreIndices(subvertexIS, &subvertices));
2480   PetscCall(ISDestroy(&subvertexIS));
2481   PetscCall(PetscFree2(pStart, pEnd));
2482   PetscFunctionReturn(0);
2483 }
2484 
2485 static PetscErrorCode DMPlexMarkCohesiveSubmesh_Uninterpolated(DM dm, PetscBool hasLagrange, const char labelname[], PetscInt value, DMLabel subpointMap, PetscInt *numFaces, PetscInt *nFV, PetscInt *subCells[], DM subdm)
2486 {
2487   DMLabel         label = NULL;
2488   const PetscInt *cone;
2489   PetscInt        dim, cMax, cEnd, c, subc = 0, p, coneSize = -1;
2490 
2491   PetscFunctionBegin;
2492   *numFaces = 0;
2493   *nFV = 0;
2494   if (labelname) PetscCall(DMGetLabel(dm, labelname, &label));
2495   *subCells = NULL;
2496   PetscCall(DMGetDimension(dm, &dim));
2497   PetscCall(DMPlexGetTensorPrismBounds_Internal(dm, dim, &cMax, &cEnd));
2498   if (cMax < 0) PetscFunctionReturn(0);
2499   if (label) {
2500     for (c = cMax; c < cEnd; ++c) {
2501       PetscInt val;
2502 
2503       PetscCall(DMLabelGetValue(label, c, &val));
2504       if (val == value) {
2505         ++(*numFaces);
2506         PetscCall(DMPlexGetConeSize(dm, c, &coneSize));
2507       }
2508     }
2509   } else {
2510     *numFaces = cEnd - cMax;
2511     PetscCall(DMPlexGetConeSize(dm, cMax, &coneSize));
2512   }
2513   PetscCall(PetscMalloc1(*numFaces *2, subCells));
2514   if (!(*numFaces)) PetscFunctionReturn(0);
2515   *nFV = hasLagrange ? coneSize/3 : coneSize/2;
2516   for (c = cMax; c < cEnd; ++c) {
2517     const PetscInt *cells;
2518     PetscInt        numCells;
2519 
2520     if (label) {
2521       PetscInt val;
2522 
2523       PetscCall(DMLabelGetValue(label, c, &val));
2524       if (val != value) continue;
2525     }
2526     PetscCall(DMPlexGetCone(dm, c, &cone));
2527     for (p = 0; p < *nFV; ++p) {
2528       PetscCall(DMLabelSetValue(subpointMap, cone[p], 0));
2529     }
2530     /* Negative face */
2531     PetscCall(DMPlexGetJoin(dm, *nFV, cone, &numCells, &cells));
2532     /* Not true in parallel
2533     PetscCheck(numCells == 2,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive cells should separate two cells"); */
2534     for (p = 0; p < numCells; ++p) {
2535       PetscCall(DMLabelSetValue(subpointMap, cells[p], 2));
2536       (*subCells)[subc++] = cells[p];
2537     }
2538     PetscCall(DMPlexRestoreJoin(dm, *nFV, cone, &numCells, &cells));
2539     /* Positive face is not included */
2540   }
2541   PetscFunctionReturn(0);
2542 }
2543 
2544 static PetscErrorCode DMPlexMarkCohesiveSubmesh_Interpolated(DM dm, DMLabel label, PetscInt value, DMLabel subpointMap, DM subdm)
2545 {
2546   PetscInt      *pStart, *pEnd;
2547   PetscInt       dim, cMax, cEnd, c, d;
2548 
2549   PetscFunctionBegin;
2550   PetscCall(DMGetDimension(dm, &dim));
2551   PetscCall(DMPlexGetTensorPrismBounds_Internal(dm, dim, &cMax, &cEnd));
2552   if (cMax < 0) PetscFunctionReturn(0);
2553   PetscCall(PetscMalloc2(dim+1,&pStart,dim+1,&pEnd));
2554   for (d = 0; d <= dim; ++d) PetscCall(DMPlexGetDepthStratum(dm, d, &pStart[d], &pEnd[d]));
2555   for (c = cMax; c < cEnd; ++c) {
2556     const PetscInt *cone;
2557     PetscInt       *closure = NULL;
2558     PetscInt        fconeSize, coneSize, closureSize, cl, val;
2559 
2560     if (label) {
2561       PetscCall(DMLabelGetValue(label, c, &val));
2562       if (val != value) continue;
2563     }
2564     PetscCall(DMPlexGetConeSize(dm, c, &coneSize));
2565     PetscCall(DMPlexGetCone(dm, c, &cone));
2566     PetscCall(DMPlexGetConeSize(dm, cone[0], &fconeSize));
2567     PetscCheck(coneSize == (fconeSize ? fconeSize : 1) + 2,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive cells should separate two cells");
2568     /* Negative face */
2569     PetscCall(DMPlexGetTransitiveClosure(dm, cone[0], PETSC_TRUE, &closureSize, &closure));
2570     for (cl = 0; cl < closureSize*2; cl += 2) {
2571       const PetscInt point = closure[cl];
2572 
2573       for (d = 0; d <= dim; ++d) {
2574         if ((point >= pStart[d]) && (point < pEnd[d])) {
2575           PetscCall(DMLabelSetValue(subpointMap, point, d));
2576           break;
2577         }
2578       }
2579     }
2580     PetscCall(DMPlexRestoreTransitiveClosure(dm, cone[0], PETSC_TRUE, &closureSize, &closure));
2581     /* Cells -- positive face is not included */
2582     for (cl = 0; cl < 1; ++cl) {
2583       const PetscInt *support;
2584       PetscInt        supportSize, s;
2585 
2586       PetscCall(DMPlexGetSupportSize(dm, cone[cl], &supportSize));
2587       /* PetscCheck(supportSize == 2,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive faces should separate two cells"); */
2588       PetscCall(DMPlexGetSupport(dm, cone[cl], &support));
2589       for (s = 0; s < supportSize; ++s) {
2590         PetscCall(DMLabelSetValue(subpointMap, support[s], dim));
2591       }
2592     }
2593   }
2594   PetscCall(PetscFree2(pStart, pEnd));
2595   PetscFunctionReturn(0);
2596 }
2597 
2598 static PetscErrorCode DMPlexGetFaceOrientation(DM dm, PetscInt cell, PetscInt numCorners, PetscInt indices[], PetscInt oppositeVertex, PetscInt origVertices[], PetscInt faceVertices[], PetscBool *posOriented)
2599 {
2600   MPI_Comm       comm;
2601   PetscBool      posOrient = PETSC_FALSE;
2602   const PetscInt debug     = 0;
2603   PetscInt       cellDim, faceSize, f;
2604 
2605   PetscFunctionBegin;
2606   PetscCall(PetscObjectGetComm((PetscObject)dm,&comm));
2607   PetscCall(DMGetDimension(dm, &cellDim));
2608   if (debug) PetscCall(PetscPrintf(comm, "cellDim: %" PetscInt_FMT " numCorners: %" PetscInt_FMT "\n", cellDim, numCorners));
2609 
2610   if (cellDim == 1 && numCorners == 2) {
2611     /* Triangle */
2612     faceSize  = numCorners-1;
2613     posOrient = !(oppositeVertex%2) ? PETSC_TRUE : PETSC_FALSE;
2614   } else if (cellDim == 2 && numCorners == 3) {
2615     /* Triangle */
2616     faceSize  = numCorners-1;
2617     posOrient = !(oppositeVertex%2) ? PETSC_TRUE : PETSC_FALSE;
2618   } else if (cellDim == 3 && numCorners == 4) {
2619     /* Tetrahedron */
2620     faceSize  = numCorners-1;
2621     posOrient = (oppositeVertex%2) ? PETSC_TRUE : PETSC_FALSE;
2622   } else if (cellDim == 1 && numCorners == 3) {
2623     /* Quadratic line */
2624     faceSize  = 1;
2625     posOrient = PETSC_TRUE;
2626   } else if (cellDim == 2 && numCorners == 4) {
2627     /* Quads */
2628     faceSize = 2;
2629     if ((indices[1] > indices[0]) && (indices[1] - indices[0] == 1)) {
2630       posOrient = PETSC_TRUE;
2631     } else if ((indices[0] == 3) && (indices[1] == 0)) {
2632       posOrient = PETSC_TRUE;
2633     } else {
2634       if (((indices[0] > indices[1]) && (indices[0] - indices[1] == 1)) || ((indices[0] == 0) && (indices[1] == 3))) {
2635         posOrient = PETSC_FALSE;
2636       } else SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid quad crossedge");
2637     }
2638   } else if (cellDim == 2 && numCorners == 6) {
2639     /* Quadratic triangle (I hate this) */
2640     /* Edges are determined by the first 2 vertices (corners of edges) */
2641     const PetscInt faceSizeTri = 3;
2642     PetscInt       sortedIndices[3], i, iFace;
2643     PetscBool      found                    = PETSC_FALSE;
2644     PetscInt       faceVerticesTriSorted[9] = {
2645       0, 3,  4, /* bottom */
2646       1, 4,  5, /* right */
2647       2, 3,  5, /* left */
2648     };
2649     PetscInt       faceVerticesTri[9] = {
2650       0, 3,  4, /* bottom */
2651       1, 4,  5, /* right */
2652       2, 5,  3, /* left */
2653     };
2654 
2655     for (i = 0; i < faceSizeTri; ++i) sortedIndices[i] = indices[i];
2656     PetscCall(PetscSortInt(faceSizeTri, sortedIndices));
2657     for (iFace = 0; iFace < 3; ++iFace) {
2658       const PetscInt ii = iFace*faceSizeTri;
2659       PetscInt       fVertex, cVertex;
2660 
2661       if ((sortedIndices[0] == faceVerticesTriSorted[ii+0]) &&
2662           (sortedIndices[1] == faceVerticesTriSorted[ii+1])) {
2663         for (fVertex = 0; fVertex < faceSizeTri; ++fVertex) {
2664           for (cVertex = 0; cVertex < faceSizeTri; ++cVertex) {
2665             if (indices[cVertex] == faceVerticesTri[ii+fVertex]) {
2666               faceVertices[fVertex] = origVertices[cVertex];
2667               break;
2668             }
2669           }
2670         }
2671         found = PETSC_TRUE;
2672         break;
2673       }
2674     }
2675     PetscCheck(found,comm, PETSC_ERR_ARG_WRONG, "Invalid tri crossface");
2676     if (posOriented) *posOriented = PETSC_TRUE;
2677     PetscFunctionReturn(0);
2678   } else if (cellDim == 2 && numCorners == 9) {
2679     /* Quadratic quad (I hate this) */
2680     /* Edges are determined by the first 2 vertices (corners of edges) */
2681     const PetscInt faceSizeQuad = 3;
2682     PetscInt       sortedIndices[3], i, iFace;
2683     PetscBool      found                      = PETSC_FALSE;
2684     PetscInt       faceVerticesQuadSorted[12] = {
2685       0, 1,  4, /* bottom */
2686       1, 2,  5, /* right */
2687       2, 3,  6, /* top */
2688       0, 3,  7, /* left */
2689     };
2690     PetscInt       faceVerticesQuad[12] = {
2691       0, 1,  4, /* bottom */
2692       1, 2,  5, /* right */
2693       2, 3,  6, /* top */
2694       3, 0,  7, /* left */
2695     };
2696 
2697     for (i = 0; i < faceSizeQuad; ++i) sortedIndices[i] = indices[i];
2698     PetscCall(PetscSortInt(faceSizeQuad, sortedIndices));
2699     for (iFace = 0; iFace < 4; ++iFace) {
2700       const PetscInt ii = iFace*faceSizeQuad;
2701       PetscInt       fVertex, cVertex;
2702 
2703       if ((sortedIndices[0] == faceVerticesQuadSorted[ii+0]) &&
2704           (sortedIndices[1] == faceVerticesQuadSorted[ii+1])) {
2705         for (fVertex = 0; fVertex < faceSizeQuad; ++fVertex) {
2706           for (cVertex = 0; cVertex < faceSizeQuad; ++cVertex) {
2707             if (indices[cVertex] == faceVerticesQuad[ii+fVertex]) {
2708               faceVertices[fVertex] = origVertices[cVertex];
2709               break;
2710             }
2711           }
2712         }
2713         found = PETSC_TRUE;
2714         break;
2715       }
2716     }
2717     PetscCheck(found,comm, PETSC_ERR_ARG_WRONG, "Invalid quad crossface");
2718     if (posOriented) *posOriented = PETSC_TRUE;
2719     PetscFunctionReturn(0);
2720   } else if (cellDim == 3 && numCorners == 8) {
2721     /* Hexes
2722        A hex is two oriented quads with the normal of the first
2723        pointing up at the second.
2724 
2725           7---6
2726          /|  /|
2727         4---5 |
2728         | 1-|-2
2729         |/  |/
2730         0---3
2731 
2732         Faces are determined by the first 4 vertices (corners of faces) */
2733     const PetscInt faceSizeHex = 4;
2734     PetscInt       sortedIndices[4], i, iFace;
2735     PetscBool      found                     = PETSC_FALSE;
2736     PetscInt       faceVerticesHexSorted[24] = {
2737       0, 1, 2, 3,  /* bottom */
2738       4, 5, 6, 7,  /* top */
2739       0, 3, 4, 5,  /* front */
2740       2, 3, 5, 6,  /* right */
2741       1, 2, 6, 7,  /* back */
2742       0, 1, 4, 7,  /* left */
2743     };
2744     PetscInt       faceVerticesHex[24] = {
2745       1, 2, 3, 0,  /* bottom */
2746       4, 5, 6, 7,  /* top */
2747       0, 3, 5, 4,  /* front */
2748       3, 2, 6, 5,  /* right */
2749       2, 1, 7, 6,  /* back */
2750       1, 0, 4, 7,  /* left */
2751     };
2752 
2753     for (i = 0; i < faceSizeHex; ++i) sortedIndices[i] = indices[i];
2754     PetscCall(PetscSortInt(faceSizeHex, sortedIndices));
2755     for (iFace = 0; iFace < 6; ++iFace) {
2756       const PetscInt ii = iFace*faceSizeHex;
2757       PetscInt       fVertex, cVertex;
2758 
2759       if ((sortedIndices[0] == faceVerticesHexSorted[ii+0]) &&
2760           (sortedIndices[1] == faceVerticesHexSorted[ii+1]) &&
2761           (sortedIndices[2] == faceVerticesHexSorted[ii+2]) &&
2762           (sortedIndices[3] == faceVerticesHexSorted[ii+3])) {
2763         for (fVertex = 0; fVertex < faceSizeHex; ++fVertex) {
2764           for (cVertex = 0; cVertex < faceSizeHex; ++cVertex) {
2765             if (indices[cVertex] == faceVerticesHex[ii+fVertex]) {
2766               faceVertices[fVertex] = origVertices[cVertex];
2767               break;
2768             }
2769           }
2770         }
2771         found = PETSC_TRUE;
2772         break;
2773       }
2774     }
2775     PetscCheck(found,comm, PETSC_ERR_ARG_WRONG, "Invalid hex crossface");
2776     if (posOriented) *posOriented = PETSC_TRUE;
2777     PetscFunctionReturn(0);
2778   } else if (cellDim == 3 && numCorners == 10) {
2779     /* Quadratic tet */
2780     /* Faces are determined by the first 3 vertices (corners of faces) */
2781     const PetscInt faceSizeTet = 6;
2782     PetscInt       sortedIndices[6], i, iFace;
2783     PetscBool      found                     = PETSC_FALSE;
2784     PetscInt       faceVerticesTetSorted[24] = {
2785       0, 1, 2,  6, 7, 8, /* bottom */
2786       0, 3, 4,  6, 7, 9,  /* front */
2787       1, 4, 5,  7, 8, 9,  /* right */
2788       2, 3, 5,  6, 8, 9,  /* left */
2789     };
2790     PetscInt       faceVerticesTet[24] = {
2791       0, 1, 2,  6, 7, 8, /* bottom */
2792       0, 4, 3,  6, 7, 9,  /* front */
2793       1, 5, 4,  7, 8, 9,  /* right */
2794       2, 3, 5,  8, 6, 9,  /* left */
2795     };
2796 
2797     for (i = 0; i < faceSizeTet; ++i) sortedIndices[i] = indices[i];
2798     PetscCall(PetscSortInt(faceSizeTet, sortedIndices));
2799     for (iFace=0; iFace < 4; ++iFace) {
2800       const PetscInt ii = iFace*faceSizeTet;
2801       PetscInt       fVertex, cVertex;
2802 
2803       if ((sortedIndices[0] == faceVerticesTetSorted[ii+0]) &&
2804           (sortedIndices[1] == faceVerticesTetSorted[ii+1]) &&
2805           (sortedIndices[2] == faceVerticesTetSorted[ii+2]) &&
2806           (sortedIndices[3] == faceVerticesTetSorted[ii+3])) {
2807         for (fVertex = 0; fVertex < faceSizeTet; ++fVertex) {
2808           for (cVertex = 0; cVertex < faceSizeTet; ++cVertex) {
2809             if (indices[cVertex] == faceVerticesTet[ii+fVertex]) {
2810               faceVertices[fVertex] = origVertices[cVertex];
2811               break;
2812             }
2813           }
2814         }
2815         found = PETSC_TRUE;
2816         break;
2817       }
2818     }
2819     PetscCheck(found,comm, PETSC_ERR_ARG_WRONG, "Invalid tet crossface");
2820     if (posOriented) *posOriented = PETSC_TRUE;
2821     PetscFunctionReturn(0);
2822   } else if (cellDim == 3 && numCorners == 27) {
2823     /* Quadratic hexes (I hate this)
2824        A hex is two oriented quads with the normal of the first
2825        pointing up at the second.
2826 
2827          7---6
2828         /|  /|
2829        4---5 |
2830        | 3-|-2
2831        |/  |/
2832        0---1
2833 
2834        Faces are determined by the first 4 vertices (corners of faces) */
2835     const PetscInt faceSizeQuadHex = 9;
2836     PetscInt       sortedIndices[9], i, iFace;
2837     PetscBool      found                         = PETSC_FALSE;
2838     PetscInt       faceVerticesQuadHexSorted[54] = {
2839       0, 1, 2, 3,  8, 9, 10, 11,  24, /* bottom */
2840       4, 5, 6, 7,  12, 13, 14, 15,  25, /* top */
2841       0, 1, 4, 5,  8, 12, 16, 17,  22, /* front */
2842       1, 2, 5, 6,  9, 13, 17, 18,  21, /* right */
2843       2, 3, 6, 7,  10, 14, 18, 19,  23, /* back */
2844       0, 3, 4, 7,  11, 15, 16, 19,  20, /* left */
2845     };
2846     PetscInt       faceVerticesQuadHex[54] = {
2847       3, 2, 1, 0,  10, 9, 8, 11,  24, /* bottom */
2848       4, 5, 6, 7,  12, 13, 14, 15,  25, /* top */
2849       0, 1, 5, 4,  8, 17, 12, 16,  22, /* front */
2850       1, 2, 6, 5,  9, 18, 13, 17,  21, /* right */
2851       2, 3, 7, 6,  10, 19, 14, 18,  23, /* back */
2852       3, 0, 4, 7,  11, 16, 15, 19,  20 /* left */
2853     };
2854 
2855     for (i = 0; i < faceSizeQuadHex; ++i) sortedIndices[i] = indices[i];
2856     PetscCall(PetscSortInt(faceSizeQuadHex, sortedIndices));
2857     for (iFace = 0; iFace < 6; ++iFace) {
2858       const PetscInt ii = iFace*faceSizeQuadHex;
2859       PetscInt       fVertex, cVertex;
2860 
2861       if ((sortedIndices[0] == faceVerticesQuadHexSorted[ii+0]) &&
2862           (sortedIndices[1] == faceVerticesQuadHexSorted[ii+1]) &&
2863           (sortedIndices[2] == faceVerticesQuadHexSorted[ii+2]) &&
2864           (sortedIndices[3] == faceVerticesQuadHexSorted[ii+3])) {
2865         for (fVertex = 0; fVertex < faceSizeQuadHex; ++fVertex) {
2866           for (cVertex = 0; cVertex < faceSizeQuadHex; ++cVertex) {
2867             if (indices[cVertex] == faceVerticesQuadHex[ii+fVertex]) {
2868               faceVertices[fVertex] = origVertices[cVertex];
2869               break;
2870             }
2871           }
2872         }
2873         found = PETSC_TRUE;
2874         break;
2875       }
2876     }
2877     PetscCheck(found,comm, PETSC_ERR_ARG_WRONG, "Invalid hex crossface");
2878     if (posOriented) *posOriented = PETSC_TRUE;
2879     PetscFunctionReturn(0);
2880   } else SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Unknown cell type for faceOrientation().");
2881   if (!posOrient) {
2882     if (debug) PetscCall(PetscPrintf(comm, "  Reversing initial face orientation\n"));
2883     for (f = 0; f < faceSize; ++f) faceVertices[f] = origVertices[faceSize-1 - f];
2884   } else {
2885     if (debug) PetscCall(PetscPrintf(comm, "  Keeping initial face orientation\n"));
2886     for (f = 0; f < faceSize; ++f) faceVertices[f] = origVertices[f];
2887   }
2888   if (posOriented) *posOriented = posOrient;
2889   PetscFunctionReturn(0);
2890 }
2891 
2892 /*@
2893   DMPlexGetOrientedFace - Given a cell and a face, as a set of vertices, return the oriented face, as a set of vertices,
2894   in faceVertices. The orientation is such that the face normal points out of the cell
2895 
2896   Not collective
2897 
2898   Input Parameters:
2899 + dm           - The original mesh
2900 . cell         - The cell mesh point
2901 . faceSize     - The number of vertices on the face
2902 . face         - The face vertices
2903 . numCorners   - The number of vertices on the cell
2904 . indices      - Local numbering of face vertices in cell cone
2905 - origVertices - Original face vertices
2906 
2907   Output Parameters:
2908 + faceVertices - The face vertices properly oriented
2909 - posOriented  - PETSC_TRUE if the face was oriented with outward normal
2910 
2911   Level: developer
2912 
2913 .seealso: `DMPlexGetCone()`
2914 @*/
2915 PetscErrorCode DMPlexGetOrientedFace(DM dm, PetscInt cell, PetscInt faceSize, const PetscInt face[], PetscInt numCorners, PetscInt indices[], PetscInt origVertices[], PetscInt faceVertices[], PetscBool *posOriented)
2916 {
2917   const PetscInt *cone = NULL;
2918   PetscInt        coneSize, v, f, v2;
2919   PetscInt        oppositeVertex = -1;
2920 
2921   PetscFunctionBegin;
2922   PetscCall(DMPlexGetConeSize(dm, cell, &coneSize));
2923   PetscCall(DMPlexGetCone(dm, cell, &cone));
2924   for (v = 0, v2 = 0; v < coneSize; ++v) {
2925     PetscBool found = PETSC_FALSE;
2926 
2927     for (f = 0; f < faceSize; ++f) {
2928       if (face[f] == cone[v]) {
2929         found = PETSC_TRUE; break;
2930       }
2931     }
2932     if (found) {
2933       indices[v2]      = v;
2934       origVertices[v2] = cone[v];
2935       ++v2;
2936     } else {
2937       oppositeVertex = v;
2938     }
2939   }
2940   PetscCall(DMPlexGetFaceOrientation(dm, cell, numCorners, indices, oppositeVertex, origVertices, faceVertices, posOriented));
2941   PetscFunctionReturn(0);
2942 }
2943 
2944 /*
2945   DMPlexInsertFace_Internal - Puts a face into the mesh
2946 
2947   Not collective
2948 
2949   Input Parameters:
2950   + dm              - The DMPlex
2951   . numFaceVertex   - The number of vertices in the face
2952   . faceVertices    - The vertices in the face for dm
2953   . subfaceVertices - The vertices in the face for subdm
2954   . numCorners      - The number of vertices in the cell
2955   . cell            - A cell in dm containing the face
2956   . subcell         - A cell in subdm containing the face
2957   . firstFace       - First face in the mesh
2958   - newFacePoint    - Next face in the mesh
2959 
2960   Output Parameters:
2961   . newFacePoint - Contains next face point number on input, updated on output
2962 
2963   Level: developer
2964 */
2965 static PetscErrorCode DMPlexInsertFace_Internal(DM dm, DM subdm, PetscInt numFaceVertices, const PetscInt faceVertices[], const PetscInt subfaceVertices[], PetscInt numCorners, PetscInt cell, PetscInt subcell, PetscInt firstFace, PetscInt *newFacePoint)
2966 {
2967   MPI_Comm        comm;
2968   DM_Plex        *submesh = (DM_Plex*) subdm->data;
2969   const PetscInt *faces;
2970   PetscInt        numFaces, coneSize;
2971 
2972   PetscFunctionBegin;
2973   PetscCall(PetscObjectGetComm((PetscObject)dm,&comm));
2974   PetscCall(DMPlexGetConeSize(subdm, subcell, &coneSize));
2975   PetscCheck(coneSize == 1,comm, PETSC_ERR_ARG_OUTOFRANGE, "Cone size of cell %" PetscInt_FMT " is %" PetscInt_FMT " != 1", cell, coneSize);
2976 #if 0
2977   /* Cannot use this because support() has not been constructed yet */
2978   PetscCall(DMPlexGetJoin(subdm, numFaceVertices, subfaceVertices, &numFaces, &faces));
2979 #else
2980   {
2981     PetscInt f;
2982 
2983     numFaces = 0;
2984     PetscCall(DMGetWorkArray(subdm, 1, MPIU_INT, (void **) &faces));
2985     for (f = firstFace; f < *newFacePoint; ++f) {
2986       PetscInt dof, off, d;
2987 
2988       PetscCall(PetscSectionGetDof(submesh->coneSection, f, &dof));
2989       PetscCall(PetscSectionGetOffset(submesh->coneSection, f, &off));
2990       /* Yes, I know this is quadratic, but I expect the sizes to be <5 */
2991       for (d = 0; d < dof; ++d) {
2992         const PetscInt p = submesh->cones[off+d];
2993         PetscInt       v;
2994 
2995         for (v = 0; v < numFaceVertices; ++v) {
2996           if (subfaceVertices[v] == p) break;
2997         }
2998         if (v == numFaceVertices) break;
2999       }
3000       if (d == dof) {
3001         numFaces               = 1;
3002         ((PetscInt*) faces)[0] = f;
3003       }
3004     }
3005   }
3006 #endif
3007   PetscCheck(numFaces <= 1,comm, PETSC_ERR_ARG_WRONG, "Vertex set had %" PetscInt_FMT " faces, not one", numFaces);
3008   else if (numFaces == 1) {
3009     /* Add the other cell neighbor for this face */
3010     PetscCall(DMPlexSetCone(subdm, subcell, faces));
3011   } else {
3012     PetscInt *indices, *origVertices, *orientedVertices, *orientedSubVertices, v, ov;
3013     PetscBool posOriented;
3014 
3015     PetscCall(DMGetWorkArray(subdm, 4*numFaceVertices * sizeof(PetscInt), MPIU_INT, &orientedVertices));
3016     origVertices        = &orientedVertices[numFaceVertices];
3017     indices             = &orientedVertices[numFaceVertices*2];
3018     orientedSubVertices = &orientedVertices[numFaceVertices*3];
3019     PetscCall(DMPlexGetOrientedFace(dm, cell, numFaceVertices, faceVertices, numCorners, indices, origVertices, orientedVertices, &posOriented));
3020     /* TODO: I know that routine should return a permutation, not the indices */
3021     for (v = 0; v < numFaceVertices; ++v) {
3022       const PetscInt vertex = faceVertices[v], subvertex = subfaceVertices[v];
3023       for (ov = 0; ov < numFaceVertices; ++ov) {
3024         if (orientedVertices[ov] == vertex) {
3025           orientedSubVertices[ov] = subvertex;
3026           break;
3027         }
3028       }
3029       PetscCheck(ov != numFaceVertices,comm, PETSC_ERR_PLIB, "Could not find face vertex %" PetscInt_FMT " in orientated set", vertex);
3030     }
3031     PetscCall(DMPlexSetCone(subdm, *newFacePoint, orientedSubVertices));
3032     PetscCall(DMPlexSetCone(subdm, subcell, newFacePoint));
3033     PetscCall(DMRestoreWorkArray(subdm, 4*numFaceVertices * sizeof(PetscInt), MPIU_INT, &orientedVertices));
3034     ++(*newFacePoint);
3035   }
3036 #if 0
3037   PetscCall(DMPlexRestoreJoin(subdm, numFaceVertices, subfaceVertices, &numFaces, &faces));
3038 #else
3039   PetscCall(DMRestoreWorkArray(subdm, 1, MPIU_INT, (void **) &faces));
3040 #endif
3041   PetscFunctionReturn(0);
3042 }
3043 
3044 static PetscErrorCode DMPlexCreateSubmesh_Uninterpolated(DM dm, DMLabel vertexLabel, PetscInt value, DM subdm)
3045 {
3046   MPI_Comm        comm;
3047   DMLabel         subpointMap;
3048   IS              subvertexIS,  subcellIS;
3049   const PetscInt *subVertices, *subCells;
3050   PetscInt        numSubVertices, firstSubVertex, numSubCells;
3051   PetscInt       *subface, maxConeSize, numSubFaces = 0, firstSubFace, newFacePoint, nFV = 0;
3052   PetscInt        vStart, vEnd, c, f;
3053 
3054   PetscFunctionBegin;
3055   PetscCall(PetscObjectGetComm((PetscObject)dm,&comm));
3056   /* Create subpointMap which marks the submesh */
3057   PetscCall(DMLabelCreate(PETSC_COMM_SELF, "subpoint_map", &subpointMap));
3058   PetscCall(DMPlexSetSubpointMap(subdm, subpointMap));
3059   PetscCall(DMLabelDestroy(&subpointMap));
3060   if (vertexLabel) PetscCall(DMPlexMarkSubmesh_Uninterpolated(dm, vertexLabel, value, subpointMap, &numSubFaces, &nFV, subdm));
3061   /* Setup chart */
3062   PetscCall(DMLabelGetStratumSize(subpointMap, 0, &numSubVertices));
3063   PetscCall(DMLabelGetStratumSize(subpointMap, 2, &numSubCells));
3064   PetscCall(DMPlexSetChart(subdm, 0, numSubCells+numSubFaces+numSubVertices));
3065   PetscCall(DMPlexSetVTKCellHeight(subdm, 1));
3066   /* Set cone sizes */
3067   firstSubVertex = numSubCells;
3068   firstSubFace   = numSubCells+numSubVertices;
3069   newFacePoint   = firstSubFace;
3070   PetscCall(DMLabelGetStratumIS(subpointMap, 0, &subvertexIS));
3071   if (subvertexIS) PetscCall(ISGetIndices(subvertexIS, &subVertices));
3072   PetscCall(DMLabelGetStratumIS(subpointMap, 2, &subcellIS));
3073   if (subcellIS) PetscCall(ISGetIndices(subcellIS, &subCells));
3074   for (c = 0; c < numSubCells; ++c) {
3075     PetscCall(DMPlexSetConeSize(subdm, c, 1));
3076   }
3077   for (f = firstSubFace; f < firstSubFace+numSubFaces; ++f) {
3078     PetscCall(DMPlexSetConeSize(subdm, f, nFV));
3079   }
3080   PetscCall(DMSetUp(subdm));
3081   /* Create face cones */
3082   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
3083   PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, NULL));
3084   PetscCall(DMGetWorkArray(subdm, maxConeSize, MPIU_INT, (void**) &subface));
3085   for (c = 0; c < numSubCells; ++c) {
3086     const PetscInt cell    = subCells[c];
3087     const PetscInt subcell = c;
3088     PetscInt      *closure = NULL;
3089     PetscInt       closureSize, cl, numCorners = 0, faceSize = 0;
3090 
3091     PetscCall(DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
3092     for (cl = 0; cl < closureSize*2; cl += 2) {
3093       const PetscInt point = closure[cl];
3094       PetscInt       subVertex;
3095 
3096       if ((point >= vStart) && (point < vEnd)) {
3097         ++numCorners;
3098         PetscCall(PetscFindInt(point, numSubVertices, subVertices, &subVertex));
3099         if (subVertex >= 0) {
3100           closure[faceSize] = point;
3101           subface[faceSize] = firstSubVertex+subVertex;
3102           ++faceSize;
3103         }
3104       }
3105     }
3106     PetscCheck(faceSize <= nFV,comm, PETSC_ERR_ARG_WRONG, "Invalid submesh: Too many vertices %" PetscInt_FMT " of an element on the surface", faceSize);
3107     if (faceSize == nFV) {
3108       PetscCall(DMPlexInsertFace_Internal(dm, subdm, faceSize, closure, subface, numCorners, cell, subcell, firstSubFace, &newFacePoint));
3109     }
3110     PetscCall(DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
3111   }
3112   PetscCall(DMRestoreWorkArray(subdm, maxConeSize, MPIU_INT, (void**) &subface));
3113   PetscCall(DMPlexSymmetrize(subdm));
3114   PetscCall(DMPlexStratify(subdm));
3115   /* Build coordinates */
3116   {
3117     PetscSection coordSection, subCoordSection;
3118     Vec          coordinates, subCoordinates;
3119     PetscScalar *coords, *subCoords;
3120     PetscInt     numComp, coordSize, v;
3121     const char  *name;
3122 
3123     PetscCall(DMGetCoordinateSection(dm, &coordSection));
3124     PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
3125     PetscCall(DMGetCoordinateSection(subdm, &subCoordSection));
3126     PetscCall(PetscSectionSetNumFields(subCoordSection, 1));
3127     PetscCall(PetscSectionGetFieldComponents(coordSection, 0, &numComp));
3128     PetscCall(PetscSectionSetFieldComponents(subCoordSection, 0, numComp));
3129     PetscCall(PetscSectionSetChart(subCoordSection, firstSubVertex, firstSubVertex+numSubVertices));
3130     for (v = 0; v < numSubVertices; ++v) {
3131       const PetscInt vertex    = subVertices[v];
3132       const PetscInt subvertex = firstSubVertex+v;
3133       PetscInt       dof;
3134 
3135       PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3136       PetscCall(PetscSectionSetDof(subCoordSection, subvertex, dof));
3137       PetscCall(PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof));
3138     }
3139     PetscCall(PetscSectionSetUp(subCoordSection));
3140     PetscCall(PetscSectionGetStorageSize(subCoordSection, &coordSize));
3141     PetscCall(VecCreate(PETSC_COMM_SELF, &subCoordinates));
3142     PetscCall(PetscObjectGetName((PetscObject)coordinates,&name));
3143     PetscCall(PetscObjectSetName((PetscObject)subCoordinates,name));
3144     PetscCall(VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE));
3145     PetscCall(VecSetType(subCoordinates,VECSTANDARD));
3146     if (coordSize) {
3147       PetscCall(VecGetArray(coordinates,    &coords));
3148       PetscCall(VecGetArray(subCoordinates, &subCoords));
3149       for (v = 0; v < numSubVertices; ++v) {
3150         const PetscInt vertex    = subVertices[v];
3151         const PetscInt subvertex = firstSubVertex+v;
3152         PetscInt       dof, off, sdof, soff, d;
3153 
3154         PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3155         PetscCall(PetscSectionGetOffset(coordSection, vertex, &off));
3156         PetscCall(PetscSectionGetDof(subCoordSection, subvertex, &sdof));
3157         PetscCall(PetscSectionGetOffset(subCoordSection, subvertex, &soff));
3158         PetscCheck(dof == sdof,comm, PETSC_ERR_PLIB, "Coordinate dimension %" PetscInt_FMT " on subvertex %" PetscInt_FMT ", vertex %" PetscInt_FMT " should be %" PetscInt_FMT, sdof, subvertex, vertex, dof);
3159         for (d = 0; d < dof; ++d) subCoords[soff+d] = coords[off+d];
3160       }
3161       PetscCall(VecRestoreArray(coordinates,    &coords));
3162       PetscCall(VecRestoreArray(subCoordinates, &subCoords));
3163     }
3164     PetscCall(DMSetCoordinatesLocal(subdm, subCoordinates));
3165     PetscCall(VecDestroy(&subCoordinates));
3166   }
3167   /* Cleanup */
3168   if (subvertexIS) PetscCall(ISRestoreIndices(subvertexIS, &subVertices));
3169   PetscCall(ISDestroy(&subvertexIS));
3170   if (subcellIS) PetscCall(ISRestoreIndices(subcellIS, &subCells));
3171   PetscCall(ISDestroy(&subcellIS));
3172   PetscFunctionReturn(0);
3173 }
3174 
3175 /* TODO: Fix this to properly propogate up error conditions it may find */
3176 static inline PetscInt DMPlexFilterPoint_Internal(PetscInt point, PetscInt firstSubPoint, PetscInt numSubPoints, const PetscInt subPoints[])
3177 {
3178   PetscInt       subPoint;
3179   PetscErrorCode ierr;
3180 
3181   ierr = PetscFindInt(point, numSubPoints, subPoints, &subPoint); if (ierr) return -1;
3182   return subPoint < 0 ? subPoint : firstSubPoint+subPoint;
3183 }
3184 
3185 static PetscErrorCode DMPlexFilterLabels_Internal(DM dm, const PetscInt numSubPoints[], const PetscInt *subpoints[], const PetscInt firstSubPoint[], DM subdm)
3186 {
3187   PetscInt       Nl, l, d;
3188 
3189   PetscFunctionBegin;
3190   PetscCall(DMGetNumLabels(dm, &Nl));
3191   for (l = 0; l < Nl; ++l) {
3192     DMLabel         label, newlabel;
3193     const char     *lname;
3194     PetscBool       isDepth, isDim, isCelltype, isVTK;
3195     IS              valueIS;
3196     const PetscInt *values;
3197     PetscInt        Nv, v;
3198 
3199     PetscCall(DMGetLabelName(dm, l, &lname));
3200     PetscCall(PetscStrcmp(lname, "depth", &isDepth));
3201     PetscCall(PetscStrcmp(lname, "dim", &isDim));
3202     PetscCall(PetscStrcmp(lname, "celltype", &isCelltype));
3203     PetscCall(PetscStrcmp(lname, "vtk", &isVTK));
3204     if (isDepth || isDim || isCelltype || isVTK) continue;
3205     PetscCall(DMCreateLabel(subdm, lname));
3206     PetscCall(DMGetLabel(dm, lname, &label));
3207     PetscCall(DMGetLabel(subdm, lname, &newlabel));
3208     PetscCall(DMLabelGetDefaultValue(label, &v));
3209     PetscCall(DMLabelSetDefaultValue(newlabel, v));
3210     PetscCall(DMLabelGetValueIS(label, &valueIS));
3211     PetscCall(ISGetLocalSize(valueIS, &Nv));
3212     PetscCall(ISGetIndices(valueIS, &values));
3213     for (v = 0; v < Nv; ++v) {
3214       IS              pointIS;
3215       const PetscInt *points;
3216       PetscInt        Np, p;
3217 
3218       PetscCall(DMLabelGetStratumIS(label, values[v], &pointIS));
3219       PetscCall(ISGetLocalSize(pointIS, &Np));
3220       PetscCall(ISGetIndices(pointIS, &points));
3221       for (p = 0; p < Np; ++p) {
3222         const PetscInt point = points[p];
3223         PetscInt       subp;
3224 
3225         PetscCall(DMPlexGetPointDepth(dm, point, &d));
3226         subp = DMPlexFilterPoint_Internal(point, firstSubPoint[d], numSubPoints[d], subpoints[d]);
3227         if (subp >= 0) PetscCall(DMLabelSetValue(newlabel, subp, values[v]));
3228       }
3229       PetscCall(ISRestoreIndices(pointIS, &points));
3230       PetscCall(ISDestroy(&pointIS));
3231     }
3232     PetscCall(ISRestoreIndices(valueIS, &values));
3233     PetscCall(ISDestroy(&valueIS));
3234   }
3235   PetscFunctionReturn(0);
3236 }
3237 
3238 static PetscErrorCode DMPlexCreateSubmeshGeneric_Interpolated(DM dm, DMLabel label, PetscInt value, PetscBool markedFaces, PetscBool isCohesive, PetscInt cellHeight, DM subdm)
3239 {
3240   MPI_Comm         comm;
3241   DMLabel          subpointMap;
3242   IS              *subpointIS;
3243   const PetscInt **subpoints;
3244   PetscInt        *numSubPoints, *firstSubPoint, *coneNew, *orntNew;
3245   PetscInt         totSubPoints = 0, maxConeSize, dim, p, d, v;
3246   PetscMPIInt      rank;
3247 
3248   PetscFunctionBegin;
3249   PetscCall(PetscObjectGetComm((PetscObject)dm,&comm));
3250   PetscCallMPI(MPI_Comm_rank(comm, &rank));
3251   /* Create subpointMap which marks the submesh */
3252   PetscCall(DMLabelCreate(PETSC_COMM_SELF, "subpoint_map", &subpointMap));
3253   PetscCall(DMPlexSetSubpointMap(subdm, subpointMap));
3254   if (cellHeight) {
3255     if (isCohesive) PetscCall(DMPlexMarkCohesiveSubmesh_Interpolated(dm, label, value, subpointMap, subdm));
3256     else            PetscCall(DMPlexMarkSubmesh_Interpolated(dm, label, value, markedFaces, subpointMap, subdm));
3257   } else {
3258     DMLabel         depth;
3259     IS              pointIS;
3260     const PetscInt *points;
3261     PetscInt        numPoints=0;
3262 
3263     PetscCall(DMPlexGetDepthLabel(dm, &depth));
3264     PetscCall(DMLabelGetStratumIS(label, value, &pointIS));
3265     if (pointIS) {
3266       PetscCall(ISGetIndices(pointIS, &points));
3267       PetscCall(ISGetLocalSize(pointIS, &numPoints));
3268     }
3269     for (p = 0; p < numPoints; ++p) {
3270       PetscInt *closure = NULL;
3271       PetscInt  closureSize, c, pdim;
3272 
3273       PetscCall(DMPlexGetTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closure));
3274       for (c = 0; c < closureSize*2; c += 2) {
3275         PetscCall(DMLabelGetValue(depth, closure[c], &pdim));
3276         PetscCall(DMLabelSetValue(subpointMap, closure[c], pdim));
3277       }
3278       PetscCall(DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closure));
3279     }
3280     if (pointIS) PetscCall(ISRestoreIndices(pointIS, &points));
3281     PetscCall(ISDestroy(&pointIS));
3282   }
3283   /* Setup chart */
3284   PetscCall(DMGetDimension(dm, &dim));
3285   PetscCall(PetscMalloc4(dim+1,&numSubPoints,dim+1,&firstSubPoint,dim+1,&subpointIS,dim+1,&subpoints));
3286   for (d = 0; d <= dim; ++d) {
3287     PetscCall(DMLabelGetStratumSize(subpointMap, d, &numSubPoints[d]));
3288     totSubPoints += numSubPoints[d];
3289   }
3290   PetscCall(DMPlexSetChart(subdm, 0, totSubPoints));
3291   PetscCall(DMPlexSetVTKCellHeight(subdm, cellHeight));
3292   /* Set cone sizes */
3293   firstSubPoint[dim] = 0;
3294   firstSubPoint[0]   = firstSubPoint[dim] + numSubPoints[dim];
3295   if (dim > 1) {firstSubPoint[dim-1] = firstSubPoint[0]     + numSubPoints[0];}
3296   if (dim > 2) {firstSubPoint[dim-2] = firstSubPoint[dim-1] + numSubPoints[dim-1];}
3297   for (d = 0; d <= dim; ++d) {
3298     PetscCall(DMLabelGetStratumIS(subpointMap, d, &subpointIS[d]));
3299     if (subpointIS[d]) PetscCall(ISGetIndices(subpointIS[d], &subpoints[d]));
3300   }
3301   /* We do not want this label automatically computed, instead we compute it here */
3302   PetscCall(DMCreateLabel(subdm, "celltype"));
3303   for (d = 0; d <= dim; ++d) {
3304     for (p = 0; p < numSubPoints[d]; ++p) {
3305       const PetscInt  point    = subpoints[d][p];
3306       const PetscInt  subpoint = firstSubPoint[d] + p;
3307       const PetscInt *cone;
3308       PetscInt        coneSize;
3309 
3310       PetscCall(DMPlexGetConeSize(dm, point, &coneSize));
3311       if (cellHeight && (d == dim)) {
3312         PetscInt coneSizeNew, c, val;
3313 
3314         PetscCall(DMPlexGetCone(dm, point, &cone));
3315         for (c = 0, coneSizeNew = 0; c < coneSize; ++c) {
3316           PetscCall(DMLabelGetValue(subpointMap, cone[c], &val));
3317           if (val >= 0) coneSizeNew++;
3318         }
3319         PetscCall(DMPlexSetConeSize(subdm, subpoint, coneSizeNew));
3320         PetscCall(DMPlexSetCellType(subdm, subpoint, DM_POLYTOPE_FV_GHOST));
3321       } else {
3322         DMPolytopeType  ct;
3323 
3324         PetscCall(DMPlexSetConeSize(subdm, subpoint, coneSize));
3325         PetscCall(DMPlexGetCellType(dm, point, &ct));
3326         PetscCall(DMPlexSetCellType(subdm, subpoint, ct));
3327       }
3328     }
3329   }
3330   PetscCall(DMLabelDestroy(&subpointMap));
3331   PetscCall(DMSetUp(subdm));
3332   /* Set cones */
3333   PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, NULL));
3334   PetscCall(PetscMalloc2(maxConeSize,&coneNew,maxConeSize,&orntNew));
3335   for (d = 0; d <= dim; ++d) {
3336     for (p = 0; p < numSubPoints[d]; ++p) {
3337       const PetscInt  point    = subpoints[d][p];
3338       const PetscInt  subpoint = firstSubPoint[d] + p;
3339       const PetscInt *cone, *ornt;
3340       PetscInt        coneSize, subconeSize, coneSizeNew, c, subc, fornt = 0;
3341 
3342       if (d == dim-1) {
3343         const PetscInt *support, *cone, *ornt;
3344         PetscInt        supportSize, coneSize, s, subc;
3345 
3346         PetscCall(DMPlexGetSupport(dm, point, &support));
3347         PetscCall(DMPlexGetSupportSize(dm, point, &supportSize));
3348         for (s = 0; s < supportSize; ++s) {
3349           PetscBool isHybrid;
3350 
3351           PetscCall(DMPlexCellIsHybrid_Internal(dm, support[s], &isHybrid));
3352           if (!isHybrid) continue;
3353           PetscCall(PetscFindInt(support[s], numSubPoints[d+1], subpoints[d+1], &subc));
3354           if (subc >= 0) {
3355             const PetscInt ccell = subpoints[d+1][subc];
3356 
3357             PetscCall(DMPlexGetCone(dm, ccell, &cone));
3358             PetscCall(DMPlexGetConeSize(dm, ccell, &coneSize));
3359             PetscCall(DMPlexGetConeOrientation(dm, ccell, &ornt));
3360             for (c = 0; c < coneSize; ++c) {
3361               if (cone[c] == point) {
3362                 fornt = ornt[c];
3363                 break;
3364               }
3365             }
3366             break;
3367           }
3368         }
3369       }
3370       PetscCall(DMPlexGetConeSize(dm, point, &coneSize));
3371       PetscCall(DMPlexGetConeSize(subdm, subpoint, &subconeSize));
3372       PetscCall(DMPlexGetCone(dm, point, &cone));
3373       PetscCall(DMPlexGetConeOrientation(dm, point, &ornt));
3374       for (c = 0, coneSizeNew = 0; c < coneSize; ++c) {
3375         PetscCall(PetscFindInt(cone[c], numSubPoints[d-1], subpoints[d-1], &subc));
3376         if (subc >= 0) {
3377           coneNew[coneSizeNew] = firstSubPoint[d-1] + subc;
3378           orntNew[coneSizeNew] = ornt[c];
3379           ++coneSizeNew;
3380         }
3381       }
3382       PetscCheck(coneSizeNew == subconeSize,comm, PETSC_ERR_PLIB, "Number of cone points located %" PetscInt_FMT " does not match subcone size %" PetscInt_FMT, coneSizeNew, subconeSize);
3383       PetscCall(DMPlexSetCone(subdm, subpoint, coneNew));
3384       PetscCall(DMPlexSetConeOrientation(subdm, subpoint, orntNew));
3385       if (fornt < 0) PetscCall(DMPlexOrientPoint(subdm, subpoint, fornt));
3386     }
3387   }
3388   PetscCall(PetscFree2(coneNew,orntNew));
3389   PetscCall(DMPlexSymmetrize(subdm));
3390   PetscCall(DMPlexStratify(subdm));
3391   /* Build coordinates */
3392   {
3393     PetscSection coordSection, subCoordSection;
3394     Vec          coordinates, subCoordinates;
3395     PetscScalar *coords, *subCoords;
3396     PetscInt     cdim, numComp, coordSize;
3397     const char  *name;
3398 
3399     PetscCall(DMGetCoordinateDim(dm, &cdim));
3400     PetscCall(DMGetCoordinateSection(dm, &coordSection));
3401     PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
3402     PetscCall(DMGetCoordinateSection(subdm, &subCoordSection));
3403     PetscCall(PetscSectionSetNumFields(subCoordSection, 1));
3404     PetscCall(PetscSectionGetFieldComponents(coordSection, 0, &numComp));
3405     PetscCall(PetscSectionSetFieldComponents(subCoordSection, 0, numComp));
3406     PetscCall(PetscSectionSetChart(subCoordSection, firstSubPoint[0], firstSubPoint[0]+numSubPoints[0]));
3407     for (v = 0; v < numSubPoints[0]; ++v) {
3408       const PetscInt vertex    = subpoints[0][v];
3409       const PetscInt subvertex = firstSubPoint[0]+v;
3410       PetscInt       dof;
3411 
3412       PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3413       PetscCall(PetscSectionSetDof(subCoordSection, subvertex, dof));
3414       PetscCall(PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof));
3415     }
3416     PetscCall(PetscSectionSetUp(subCoordSection));
3417     PetscCall(PetscSectionGetStorageSize(subCoordSection, &coordSize));
3418     PetscCall(VecCreate(PETSC_COMM_SELF, &subCoordinates));
3419     PetscCall(PetscObjectGetName((PetscObject)coordinates,&name));
3420     PetscCall(PetscObjectSetName((PetscObject)subCoordinates,name));
3421     PetscCall(VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE));
3422     PetscCall(VecSetBlockSize(subCoordinates, cdim));
3423     PetscCall(VecSetType(subCoordinates,VECSTANDARD));
3424     PetscCall(VecGetArray(coordinates,    &coords));
3425     PetscCall(VecGetArray(subCoordinates, &subCoords));
3426     for (v = 0; v < numSubPoints[0]; ++v) {
3427       const PetscInt vertex    = subpoints[0][v];
3428       const PetscInt subvertex = firstSubPoint[0]+v;
3429       PetscInt dof, off, sdof, soff, d;
3430 
3431       PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3432       PetscCall(PetscSectionGetOffset(coordSection, vertex, &off));
3433       PetscCall(PetscSectionGetDof(subCoordSection, subvertex, &sdof));
3434       PetscCall(PetscSectionGetOffset(subCoordSection, subvertex, &soff));
3435       PetscCheck(dof == sdof,comm, PETSC_ERR_PLIB, "Coordinate dimension %" PetscInt_FMT " on subvertex %" PetscInt_FMT ", vertex %" PetscInt_FMT " should be %" PetscInt_FMT, sdof, subvertex, vertex, dof);
3436       for (d = 0; d < dof; ++d) subCoords[soff+d] = coords[off+d];
3437     }
3438     PetscCall(VecRestoreArray(coordinates,    &coords));
3439     PetscCall(VecRestoreArray(subCoordinates, &subCoords));
3440     PetscCall(DMSetCoordinatesLocal(subdm, subCoordinates));
3441     PetscCall(VecDestroy(&subCoordinates));
3442   }
3443   /* Build SF: We need this complexity because subpoints might not be selected on the owning process */
3444   {
3445     PetscSF            sfPoint, sfPointSub;
3446     IS                 subpIS;
3447     const PetscSFNode *remotePoints;
3448     PetscSFNode       *sremotePoints, *newLocalPoints, *newOwners;
3449     const PetscInt    *localPoints, *subpoints;
3450     PetscInt          *slocalPoints;
3451     PetscInt           numRoots, numLeaves, numSubpoints = 0, numSubroots, numSubleaves = 0, l, sl, ll, pStart, pEnd, p;
3452     PetscMPIInt        rank;
3453 
3454     PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank));
3455     PetscCall(DMGetPointSF(dm, &sfPoint));
3456     PetscCall(DMGetPointSF(subdm, &sfPointSub));
3457     PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
3458     PetscCall(DMPlexGetChart(subdm, NULL, &numSubroots));
3459     PetscCall(DMPlexGetSubpointIS(subdm, &subpIS));
3460     if (subpIS) {
3461       PetscCall(ISGetIndices(subpIS, &subpoints));
3462       PetscCall(ISGetLocalSize(subpIS, &numSubpoints));
3463     }
3464     PetscCall(PetscSFGetGraph(sfPoint, &numRoots, &numLeaves, &localPoints, &remotePoints));
3465     if (numRoots >= 0) {
3466       PetscCall(PetscMalloc2(pEnd-pStart,&newLocalPoints,numRoots,&newOwners));
3467       for (p = 0; p < pEnd-pStart; ++p) {
3468         newLocalPoints[p].rank  = -2;
3469         newLocalPoints[p].index = -2;
3470       }
3471       /* Set subleaves */
3472       for (l = 0; l < numLeaves; ++l) {
3473         const PetscInt point    = localPoints[l];
3474         const PetscInt subpoint = DMPlexFilterPoint_Internal(point, 0, numSubpoints, subpoints);
3475 
3476         if (subpoint < 0) continue;
3477         newLocalPoints[point-pStart].rank  = rank;
3478         newLocalPoints[point-pStart].index = subpoint;
3479         ++numSubleaves;
3480       }
3481       /* Must put in owned subpoints */
3482       for (p = pStart; p < pEnd; ++p) {
3483         const PetscInt subpoint = DMPlexFilterPoint_Internal(p, 0, numSubpoints, subpoints);
3484 
3485         if (subpoint < 0) {
3486           newOwners[p-pStart].rank  = -3;
3487           newOwners[p-pStart].index = -3;
3488         } else {
3489           newOwners[p-pStart].rank  = rank;
3490           newOwners[p-pStart].index = subpoint;
3491         }
3492       }
3493       PetscCall(PetscSFReduceBegin(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC));
3494       PetscCall(PetscSFReduceEnd(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC));
3495       PetscCall(PetscSFBcastBegin(sfPoint, MPIU_2INT, newOwners, newLocalPoints,MPI_REPLACE));
3496       PetscCall(PetscSFBcastEnd(sfPoint, MPIU_2INT, newOwners, newLocalPoints,MPI_REPLACE));
3497       PetscCall(PetscMalloc1(numSubleaves, &slocalPoints));
3498       PetscCall(PetscMalloc1(numSubleaves, &sremotePoints));
3499       for (l = 0, sl = 0, ll = 0; l < numLeaves; ++l) {
3500         const PetscInt point    = localPoints[l];
3501         const PetscInt subpoint = DMPlexFilterPoint_Internal(point, 0, numSubpoints, subpoints);
3502 
3503         if (subpoint < 0) continue;
3504         if (newLocalPoints[point].rank == rank) {++ll; continue;}
3505         slocalPoints[sl]        = subpoint;
3506         sremotePoints[sl].rank  = newLocalPoints[point].rank;
3507         sremotePoints[sl].index = newLocalPoints[point].index;
3508         PetscCheck(sremotePoints[sl].rank  >= 0,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote rank for local point %" PetscInt_FMT, point);
3509         PetscCheck(sremotePoints[sl].index >= 0,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote subpoint for local point %" PetscInt_FMT, point);
3510         ++sl;
3511       }
3512       PetscCheck(sl + ll == numSubleaves,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatch in number of subleaves %" PetscInt_FMT " + %" PetscInt_FMT " != %" PetscInt_FMT, sl, ll, numSubleaves);
3513       PetscCall(PetscFree2(newLocalPoints,newOwners));
3514       PetscCall(PetscSFSetGraph(sfPointSub, numSubroots, sl, slocalPoints, PETSC_OWN_POINTER, sremotePoints, PETSC_OWN_POINTER));
3515     }
3516     if (subpIS) {
3517       PetscCall(ISRestoreIndices(subpIS, &subpoints));
3518     }
3519   }
3520   /* Filter labels */
3521   PetscCall(DMPlexFilterLabels_Internal(dm, numSubPoints, subpoints, firstSubPoint, subdm));
3522   /* Cleanup */
3523   for (d = 0; d <= dim; ++d) {
3524     if (subpointIS[d]) PetscCall(ISRestoreIndices(subpointIS[d], &subpoints[d]));
3525     PetscCall(ISDestroy(&subpointIS[d]));
3526   }
3527   PetscCall(PetscFree4(numSubPoints,firstSubPoint,subpointIS,subpoints));
3528   PetscFunctionReturn(0);
3529 }
3530 
3531 static PetscErrorCode DMPlexCreateSubmesh_Interpolated(DM dm, DMLabel vertexLabel, PetscInt value, PetscBool markedFaces, DM subdm)
3532 {
3533   PetscFunctionBegin;
3534   PetscCall(DMPlexCreateSubmeshGeneric_Interpolated(dm, vertexLabel, value, markedFaces, PETSC_FALSE, 1, subdm));
3535   PetscFunctionReturn(0);
3536 }
3537 
3538 /*@
3539   DMPlexCreateSubmesh - Extract a hypersurface from the mesh using vertices defined by a label
3540 
3541   Input Parameters:
3542 + dm           - The original mesh
3543 . vertexLabel  - The DMLabel marking points contained in the surface
3544 . value        - The label value to use
3545 - markedFaces  - PETSC_TRUE if surface faces are marked in addition to vertices, PETSC_FALSE if only vertices are marked
3546 
3547   Output Parameter:
3548 . subdm - The surface mesh
3549 
3550   Note: This function produces a DMLabel mapping original points in the submesh to their depth. This can be obtained using DMPlexGetSubpointMap().
3551 
3552   Level: developer
3553 
3554 .seealso: `DMPlexGetSubpointMap()`, `DMGetLabel()`, `DMLabelSetValue()`
3555 @*/
3556 PetscErrorCode DMPlexCreateSubmesh(DM dm, DMLabel vertexLabel, PetscInt value, PetscBool markedFaces, DM *subdm)
3557 {
3558   DMPlexInterpolatedFlag interpolated;
3559   PetscInt       dim, cdim;
3560 
3561   PetscFunctionBegin;
3562   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3563   PetscValidPointer(subdm, 5);
3564   PetscCall(DMGetDimension(dm, &dim));
3565   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), subdm));
3566   PetscCall(DMSetType(*subdm, DMPLEX));
3567   PetscCall(DMSetDimension(*subdm, dim-1));
3568   PetscCall(DMGetCoordinateDim(dm, &cdim));
3569   PetscCall(DMSetCoordinateDim(*subdm, cdim));
3570   PetscCall(DMPlexIsInterpolated(dm, &interpolated));
3571   PetscCheck(interpolated != DMPLEX_INTERPOLATED_PARTIAL,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Not for partially interpolated meshes");
3572   if (interpolated) {
3573     PetscCall(DMPlexCreateSubmesh_Interpolated(dm, vertexLabel, value, markedFaces, *subdm));
3574   } else {
3575     PetscCall(DMPlexCreateSubmesh_Uninterpolated(dm, vertexLabel, value, *subdm));
3576   }
3577   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_TRUE, *subdm));
3578   PetscFunctionReturn(0);
3579 }
3580 
3581 static PetscErrorCode DMPlexCreateCohesiveSubmesh_Uninterpolated(DM dm, PetscBool hasLagrange, const char label[], PetscInt value, DM subdm)
3582 {
3583   MPI_Comm        comm;
3584   DMLabel         subpointMap;
3585   IS              subvertexIS;
3586   const PetscInt *subVertices;
3587   PetscInt        numSubVertices, firstSubVertex, numSubCells, *subCells = NULL;
3588   PetscInt       *subface, maxConeSize, numSubFaces, firstSubFace, newFacePoint, nFV;
3589   PetscInt        c, f;
3590 
3591   PetscFunctionBegin;
3592   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
3593   /* Create subpointMap which marks the submesh */
3594   PetscCall(DMLabelCreate(PETSC_COMM_SELF, "subpoint_map", &subpointMap));
3595   PetscCall(DMPlexSetSubpointMap(subdm, subpointMap));
3596   PetscCall(DMLabelDestroy(&subpointMap));
3597   PetscCall(DMPlexMarkCohesiveSubmesh_Uninterpolated(dm, hasLagrange, label, value, subpointMap, &numSubFaces, &nFV, &subCells, subdm));
3598   /* Setup chart */
3599   PetscCall(DMLabelGetStratumSize(subpointMap, 0, &numSubVertices));
3600   PetscCall(DMLabelGetStratumSize(subpointMap, 2, &numSubCells));
3601   PetscCall(DMPlexSetChart(subdm, 0, numSubCells+numSubFaces+numSubVertices));
3602   PetscCall(DMPlexSetVTKCellHeight(subdm, 1));
3603   /* Set cone sizes */
3604   firstSubVertex = numSubCells;
3605   firstSubFace   = numSubCells+numSubVertices;
3606   newFacePoint   = firstSubFace;
3607   PetscCall(DMLabelGetStratumIS(subpointMap, 0, &subvertexIS));
3608   if (subvertexIS) PetscCall(ISGetIndices(subvertexIS, &subVertices));
3609   for (c = 0; c < numSubCells; ++c) {
3610     PetscCall(DMPlexSetConeSize(subdm, c, 1));
3611   }
3612   for (f = firstSubFace; f < firstSubFace+numSubFaces; ++f) {
3613     PetscCall(DMPlexSetConeSize(subdm, f, nFV));
3614   }
3615   PetscCall(DMSetUp(subdm));
3616   /* Create face cones */
3617   PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, NULL));
3618   PetscCall(DMGetWorkArray(subdm, maxConeSize, MPIU_INT, (void**) &subface));
3619   for (c = 0; c < numSubCells; ++c) {
3620     const PetscInt  cell    = subCells[c];
3621     const PetscInt  subcell = c;
3622     const PetscInt *cone, *cells;
3623     PetscBool       isHybrid;
3624     PetscInt        numCells, subVertex, p, v;
3625 
3626     PetscCall(DMPlexCellIsHybrid_Internal(dm, cell, &isHybrid));
3627     if (!isHybrid) continue;
3628     PetscCall(DMPlexGetCone(dm, cell, &cone));
3629     for (v = 0; v < nFV; ++v) {
3630       PetscCall(PetscFindInt(cone[v], numSubVertices, subVertices, &subVertex));
3631       subface[v] = firstSubVertex+subVertex;
3632     }
3633     PetscCall(DMPlexSetCone(subdm, newFacePoint, subface));
3634     PetscCall(DMPlexSetCone(subdm, subcell, &newFacePoint));
3635     PetscCall(DMPlexGetJoin(dm, nFV, cone, &numCells, &cells));
3636     /* Not true in parallel
3637     PetscCheck(numCells == 2,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive cells should separate two cells"); */
3638     for (p = 0; p < numCells; ++p) {
3639       PetscInt  negsubcell;
3640       PetscBool isHybrid;
3641 
3642       PetscCall(DMPlexCellIsHybrid_Internal(dm, cells[p], &isHybrid));
3643       if (isHybrid) continue;
3644       /* I know this is a crap search */
3645       for (negsubcell = 0; negsubcell < numSubCells; ++negsubcell) {
3646         if (subCells[negsubcell] == cells[p]) break;
3647       }
3648       PetscCheck(negsubcell != numSubCells,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not find negative face neighbor for cohesive cell %" PetscInt_FMT, cell);
3649       PetscCall(DMPlexSetCone(subdm, negsubcell, &newFacePoint));
3650     }
3651     PetscCall(DMPlexRestoreJoin(dm, nFV, cone, &numCells, &cells));
3652     ++newFacePoint;
3653   }
3654   PetscCall(DMRestoreWorkArray(subdm, maxConeSize, MPIU_INT, (void**) &subface));
3655   PetscCall(DMPlexSymmetrize(subdm));
3656   PetscCall(DMPlexStratify(subdm));
3657   /* Build coordinates */
3658   {
3659     PetscSection coordSection, subCoordSection;
3660     Vec          coordinates, subCoordinates;
3661     PetscScalar *coords, *subCoords;
3662     PetscInt     cdim, numComp, coordSize, v;
3663     const char  *name;
3664 
3665     PetscCall(DMGetCoordinateDim(dm, &cdim));
3666     PetscCall(DMGetCoordinateSection(dm, &coordSection));
3667     PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
3668     PetscCall(DMGetCoordinateSection(subdm, &subCoordSection));
3669     PetscCall(PetscSectionSetNumFields(subCoordSection, 1));
3670     PetscCall(PetscSectionGetFieldComponents(coordSection, 0, &numComp));
3671     PetscCall(PetscSectionSetFieldComponents(subCoordSection, 0, numComp));
3672     PetscCall(PetscSectionSetChart(subCoordSection, firstSubVertex, firstSubVertex+numSubVertices));
3673     for (v = 0; v < numSubVertices; ++v) {
3674       const PetscInt vertex    = subVertices[v];
3675       const PetscInt subvertex = firstSubVertex+v;
3676       PetscInt       dof;
3677 
3678       PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3679       PetscCall(PetscSectionSetDof(subCoordSection, subvertex, dof));
3680       PetscCall(PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof));
3681     }
3682     PetscCall(PetscSectionSetUp(subCoordSection));
3683     PetscCall(PetscSectionGetStorageSize(subCoordSection, &coordSize));
3684     PetscCall(VecCreate(PETSC_COMM_SELF, &subCoordinates));
3685     PetscCall(PetscObjectGetName((PetscObject)coordinates,&name));
3686     PetscCall(PetscObjectSetName((PetscObject)subCoordinates,name));
3687     PetscCall(VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE));
3688     PetscCall(VecSetBlockSize(subCoordinates, cdim));
3689     PetscCall(VecSetType(subCoordinates,VECSTANDARD));
3690     PetscCall(VecGetArray(coordinates,    &coords));
3691     PetscCall(VecGetArray(subCoordinates, &subCoords));
3692     for (v = 0; v < numSubVertices; ++v) {
3693       const PetscInt vertex    = subVertices[v];
3694       const PetscInt subvertex = firstSubVertex+v;
3695       PetscInt       dof, off, sdof, soff, d;
3696 
3697       PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3698       PetscCall(PetscSectionGetOffset(coordSection, vertex, &off));
3699       PetscCall(PetscSectionGetDof(subCoordSection, subvertex, &sdof));
3700       PetscCall(PetscSectionGetOffset(subCoordSection, subvertex, &soff));
3701       PetscCheck(dof == sdof,comm, PETSC_ERR_PLIB, "Coordinate dimension %" PetscInt_FMT " on subvertex %" PetscInt_FMT ", vertex %" PetscInt_FMT " should be %" PetscInt_FMT, sdof, subvertex, vertex, dof);
3702       for (d = 0; d < dof; ++d) subCoords[soff+d] = coords[off+d];
3703     }
3704     PetscCall(VecRestoreArray(coordinates,    &coords));
3705     PetscCall(VecRestoreArray(subCoordinates, &subCoords));
3706     PetscCall(DMSetCoordinatesLocal(subdm, subCoordinates));
3707     PetscCall(VecDestroy(&subCoordinates));
3708   }
3709   /* Build SF */
3710   CHKMEMQ;
3711   {
3712     PetscSF            sfPoint, sfPointSub;
3713     const PetscSFNode *remotePoints;
3714     PetscSFNode       *sremotePoints, *newLocalPoints, *newOwners;
3715     const PetscInt    *localPoints;
3716     PetscInt          *slocalPoints;
3717     PetscInt           numRoots, numLeaves, numSubRoots = numSubCells+numSubFaces+numSubVertices, numSubLeaves = 0, l, sl, ll, pStart, pEnd, p, vStart, vEnd;
3718     PetscMPIInt        rank;
3719 
3720     PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank));
3721     PetscCall(DMGetPointSF(dm, &sfPoint));
3722     PetscCall(DMGetPointSF(subdm, &sfPointSub));
3723     PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
3724     PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
3725     PetscCall(PetscSFGetGraph(sfPoint, &numRoots, &numLeaves, &localPoints, &remotePoints));
3726     if (numRoots >= 0) {
3727       /* Only vertices should be shared */
3728       PetscCall(PetscMalloc2(pEnd-pStart,&newLocalPoints,numRoots,&newOwners));
3729       for (p = 0; p < pEnd-pStart; ++p) {
3730         newLocalPoints[p].rank  = -2;
3731         newLocalPoints[p].index = -2;
3732       }
3733       /* Set subleaves */
3734       for (l = 0; l < numLeaves; ++l) {
3735         const PetscInt point    = localPoints[l];
3736         const PetscInt subPoint = DMPlexFilterPoint_Internal(point, firstSubVertex, numSubVertices, subVertices);
3737 
3738         PetscCheck(!(point < vStart) || !(point >= vEnd),PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Should not be mapping anything but vertices, %" PetscInt_FMT, point);
3739         if (subPoint < 0) continue;
3740         newLocalPoints[point-pStart].rank  = rank;
3741         newLocalPoints[point-pStart].index = subPoint;
3742         ++numSubLeaves;
3743       }
3744       /* Must put in owned subpoints */
3745       for (p = pStart; p < pEnd; ++p) {
3746         const PetscInt subPoint = DMPlexFilterPoint_Internal(p, firstSubVertex, numSubVertices, subVertices);
3747 
3748         if (subPoint < 0) {
3749           newOwners[p-pStart].rank  = -3;
3750           newOwners[p-pStart].index = -3;
3751         } else {
3752           newOwners[p-pStart].rank  = rank;
3753           newOwners[p-pStart].index = subPoint;
3754         }
3755       }
3756       PetscCall(PetscSFReduceBegin(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC));
3757       PetscCall(PetscSFReduceEnd(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC));
3758       PetscCall(PetscSFBcastBegin(sfPoint, MPIU_2INT, newOwners, newLocalPoints,MPI_REPLACE));
3759       PetscCall(PetscSFBcastEnd(sfPoint, MPIU_2INT, newOwners, newLocalPoints,MPI_REPLACE));
3760       PetscCall(PetscMalloc1(numSubLeaves,    &slocalPoints));
3761       PetscCall(PetscMalloc1(numSubLeaves, &sremotePoints));
3762       for (l = 0, sl = 0, ll = 0; l < numLeaves; ++l) {
3763         const PetscInt point    = localPoints[l];
3764         const PetscInt subPoint = DMPlexFilterPoint_Internal(point, firstSubVertex, numSubVertices, subVertices);
3765 
3766         if (subPoint < 0) continue;
3767         if (newLocalPoints[point].rank == rank) {++ll; continue;}
3768         slocalPoints[sl]        = subPoint;
3769         sremotePoints[sl].rank  = newLocalPoints[point].rank;
3770         sremotePoints[sl].index = newLocalPoints[point].index;
3771         PetscCheck(sremotePoints[sl].rank  >= 0,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote rank for local point %" PetscInt_FMT, point);
3772         PetscCheck(sremotePoints[sl].index >= 0,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote subpoint for local point %" PetscInt_FMT, point);
3773         ++sl;
3774       }
3775       PetscCall(PetscFree2(newLocalPoints,newOwners));
3776       PetscCheck(sl + ll == numSubLeaves,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatch in number of subleaves %" PetscInt_FMT " + %" PetscInt_FMT " != %" PetscInt_FMT, sl, ll, numSubLeaves);
3777       PetscCall(PetscSFSetGraph(sfPointSub, numSubRoots, sl, slocalPoints, PETSC_OWN_POINTER, sremotePoints, PETSC_OWN_POINTER));
3778     }
3779   }
3780   CHKMEMQ;
3781   /* Cleanup */
3782   if (subvertexIS) PetscCall(ISRestoreIndices(subvertexIS, &subVertices));
3783   PetscCall(ISDestroy(&subvertexIS));
3784   PetscCall(PetscFree(subCells));
3785   PetscFunctionReturn(0);
3786 }
3787 
3788 static PetscErrorCode DMPlexCreateCohesiveSubmesh_Interpolated(DM dm, const char labelname[], PetscInt value, DM subdm)
3789 {
3790   DMLabel        label = NULL;
3791 
3792   PetscFunctionBegin;
3793   if (labelname) PetscCall(DMGetLabel(dm, labelname, &label));
3794   PetscCall(DMPlexCreateSubmeshGeneric_Interpolated(dm, label, value, PETSC_FALSE, PETSC_TRUE, 1, subdm));
3795   PetscFunctionReturn(0);
3796 }
3797 
3798 /*@C
3799   DMPlexCreateCohesiveSubmesh - Extract from a mesh with cohesive cells the hypersurface defined by one face of the cells. Optionally, a Label can be given to restrict the cells.
3800 
3801   Input Parameters:
3802 + dm          - The original mesh
3803 . hasLagrange - The mesh has Lagrange unknowns in the cohesive cells
3804 . label       - A label name, or NULL
3805 - value  - A label value
3806 
3807   Output Parameter:
3808 . subdm - The surface mesh
3809 
3810   Note: This function produces a DMLabel mapping original points in the submesh to their depth. This can be obtained using DMPlexGetSubpointMap().
3811 
3812   Level: developer
3813 
3814 .seealso: `DMPlexGetSubpointMap()`, `DMPlexCreateSubmesh()`
3815 @*/
3816 PetscErrorCode DMPlexCreateCohesiveSubmesh(DM dm, PetscBool hasLagrange, const char label[], PetscInt value, DM *subdm)
3817 {
3818   PetscInt       dim, cdim, depth;
3819 
3820   PetscFunctionBegin;
3821   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3822   PetscValidPointer(subdm, 5);
3823   PetscCall(DMGetDimension(dm, &dim));
3824   PetscCall(DMPlexGetDepth(dm, &depth));
3825   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), subdm));
3826   PetscCall(DMSetType(*subdm, DMPLEX));
3827   PetscCall(DMSetDimension(*subdm, dim-1));
3828   PetscCall(DMGetCoordinateDim(dm, &cdim));
3829   PetscCall(DMSetCoordinateDim(*subdm, cdim));
3830   if (depth == dim) {
3831     PetscCall(DMPlexCreateCohesiveSubmesh_Interpolated(dm, label, value, *subdm));
3832   } else {
3833     PetscCall(DMPlexCreateCohesiveSubmesh_Uninterpolated(dm, hasLagrange, label, value, *subdm));
3834   }
3835   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_TRUE, *subdm));
3836   PetscFunctionReturn(0);
3837 }
3838 
3839 /*@
3840   DMPlexFilter - Extract a subset of mesh cells defined by a label as a separate mesh
3841 
3842   Input Parameters:
3843 + dm        - The original mesh
3844 . cellLabel - The DMLabel marking cells contained in the new mesh
3845 - value     - The label value to use
3846 
3847   Output Parameter:
3848 . subdm - The new mesh
3849 
3850   Note: This function produces a DMLabel mapping original points in the submesh to their depth. This can be obtained using DMPlexGetSubpointMap().
3851 
3852   Level: developer
3853 
3854 .seealso: `DMPlexGetSubpointMap()`, `DMGetLabel()`, `DMLabelSetValue()`
3855 @*/
3856 PetscErrorCode DMPlexFilter(DM dm, DMLabel cellLabel, PetscInt value, DM *subdm)
3857 {
3858   PetscInt       dim;
3859 
3860   PetscFunctionBegin;
3861   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3862   PetscValidPointer(subdm, 4);
3863   PetscCall(DMGetDimension(dm, &dim));
3864   PetscCall(DMCreate(PetscObjectComm((PetscObject) dm), subdm));
3865   PetscCall(DMSetType(*subdm, DMPLEX));
3866   PetscCall(DMSetDimension(*subdm, dim));
3867   /* Extract submesh in place, could be empty on some procs, could have inconsistency if procs do not both extract a shared cell */
3868   PetscCall(DMPlexCreateSubmeshGeneric_Interpolated(dm, cellLabel, value, PETSC_FALSE, PETSC_FALSE, 0, *subdm));
3869   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_TRUE, *subdm));
3870   PetscFunctionReturn(0);
3871 }
3872 
3873 /*@
3874   DMPlexGetSubpointMap - Returns a DMLabel with point dimension as values
3875 
3876   Input Parameter:
3877 . dm - The submesh DM
3878 
3879   Output Parameter:
3880 . subpointMap - The DMLabel of all the points from the original mesh in this submesh, or NULL if this is not a submesh
3881 
3882   Level: developer
3883 
3884 .seealso: `DMPlexCreateSubmesh()`, `DMPlexGetSubpointIS()`
3885 @*/
3886 PetscErrorCode DMPlexGetSubpointMap(DM dm, DMLabel *subpointMap)
3887 {
3888   PetscFunctionBegin;
3889   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3890   PetscValidPointer(subpointMap, 2);
3891   *subpointMap = ((DM_Plex*) dm->data)->subpointMap;
3892   PetscFunctionReturn(0);
3893 }
3894 
3895 /*@
3896   DMPlexSetSubpointMap - Sets the DMLabel with point dimension as values
3897 
3898   Input Parameters:
3899 + dm - The submesh DM
3900 - subpointMap - The DMLabel of all the points from the original mesh in this submesh
3901 
3902   Note: Should normally not be called by the user, since it is set in DMPlexCreateSubmesh()
3903 
3904   Level: developer
3905 
3906 .seealso: `DMPlexCreateSubmesh()`, `DMPlexGetSubpointIS()`
3907 @*/
3908 PetscErrorCode DMPlexSetSubpointMap(DM dm, DMLabel subpointMap)
3909 {
3910   DM_Plex       *mesh = (DM_Plex *) dm->data;
3911   DMLabel        tmp;
3912 
3913   PetscFunctionBegin;
3914   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3915   tmp  = mesh->subpointMap;
3916   mesh->subpointMap = subpointMap;
3917   PetscCall(PetscObjectReference((PetscObject) mesh->subpointMap));
3918   PetscCall(DMLabelDestroy(&tmp));
3919   PetscFunctionReturn(0);
3920 }
3921 
3922 static PetscErrorCode DMPlexCreateSubpointIS_Internal(DM dm, IS *subpointIS)
3923 {
3924   DMLabel        spmap;
3925   PetscInt       depth, d;
3926 
3927   PetscFunctionBegin;
3928   PetscCall(DMPlexGetSubpointMap(dm, &spmap));
3929   PetscCall(DMPlexGetDepth(dm, &depth));
3930   if (spmap && depth >= 0) {
3931     DM_Plex  *mesh = (DM_Plex *) dm->data;
3932     PetscInt *points, *depths;
3933     PetscInt  pStart, pEnd, p, off;
3934 
3935     PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
3936     PetscCheck(!pStart,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Submeshes must start the point numbering at 0, not %" PetscInt_FMT, pStart);
3937     PetscCall(PetscMalloc1(pEnd, &points));
3938     PetscCall(DMGetWorkArray(dm, depth+1, MPIU_INT, &depths));
3939     depths[0] = depth;
3940     depths[1] = 0;
3941     for (d = 2; d <= depth; ++d) {depths[d] = depth+1 - d;}
3942     for (d = 0, off = 0; d <= depth; ++d) {
3943       const PetscInt dep = depths[d];
3944       PetscInt       depStart, depEnd, n;
3945 
3946       PetscCall(DMPlexGetDepthStratum(dm, dep, &depStart, &depEnd));
3947       PetscCall(DMLabelGetStratumSize(spmap, dep, &n));
3948       if (((d < 2) && (depth > 1)) || (d == 1)) { /* Only check vertices and cells for now since the map is broken for others */
3949         PetscCheck(n == depEnd-depStart,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "The number of mapped submesh points %" PetscInt_FMT " at depth %" PetscInt_FMT " should be %" PetscInt_FMT, n, dep, depEnd-depStart);
3950       } else {
3951         if (!n) {
3952           if (d == 0) {
3953             /* Missing cells */
3954             for (p = 0; p < depEnd-depStart; ++p, ++off) points[off] = -1;
3955           } else {
3956             /* Missing faces */
3957             for (p = 0; p < depEnd-depStart; ++p, ++off) points[off] = PETSC_MAX_INT;
3958           }
3959         }
3960       }
3961       if (n) {
3962         IS              is;
3963         const PetscInt *opoints;
3964 
3965         PetscCall(DMLabelGetStratumIS(spmap, dep, &is));
3966         PetscCall(ISGetIndices(is, &opoints));
3967         for (p = 0; p < n; ++p, ++off) points[off] = opoints[p];
3968         PetscCall(ISRestoreIndices(is, &opoints));
3969         PetscCall(ISDestroy(&is));
3970       }
3971     }
3972     PetscCall(DMRestoreWorkArray(dm, depth+1, MPIU_INT, &depths));
3973     PetscCheck(off == pEnd,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "The number of mapped submesh points %" PetscInt_FMT " should be %" PetscInt_FMT, off, pEnd);
3974     PetscCall(ISCreateGeneral(PETSC_COMM_SELF, pEnd, points, PETSC_OWN_POINTER, subpointIS));
3975     PetscCall(PetscObjectStateGet((PetscObject) spmap, &mesh->subpointState));
3976   }
3977   PetscFunctionReturn(0);
3978 }
3979 
3980 /*@
3981   DMPlexGetSubpointIS - Returns an IS covering the entire subdm chart with the original points as data
3982 
3983   Input Parameter:
3984 . dm - The submesh DM
3985 
3986   Output Parameter:
3987 . subpointIS - The IS of all the points from the original mesh in this submesh, or NULL if this is not a submesh
3988 
3989   Note: This IS is guaranteed to be sorted by the construction of the submesh
3990 
3991   Level: developer
3992 
3993 .seealso: `DMPlexCreateSubmesh()`, `DMPlexGetSubpointMap()`
3994 @*/
3995 PetscErrorCode DMPlexGetSubpointIS(DM dm, IS *subpointIS)
3996 {
3997   DM_Plex         *mesh = (DM_Plex *) dm->data;
3998   DMLabel          spmap;
3999   PetscObjectState state;
4000 
4001   PetscFunctionBegin;
4002   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4003   PetscValidPointer(subpointIS, 2);
4004   PetscCall(DMPlexGetSubpointMap(dm, &spmap));
4005   PetscCall(PetscObjectStateGet((PetscObject) spmap, &state));
4006   if (state != mesh->subpointState || !mesh->subpointIS) PetscCall(DMPlexCreateSubpointIS_Internal(dm, &mesh->subpointIS));
4007   *subpointIS = mesh->subpointIS;
4008   PetscFunctionReturn(0);
4009 }
4010 
4011 /*@
4012   DMGetEnclosureRelation - Get the relationship between dmA and dmB
4013 
4014   Input Parameters:
4015 + dmA - The first DM
4016 - dmB - The second DM
4017 
4018   Output Parameter:
4019 . rel - The relation of dmA to dmB
4020 
4021   Level: intermediate
4022 
4023 .seealso: `DMGetEnclosurePoint()`
4024 @*/
4025 PetscErrorCode DMGetEnclosureRelation(DM dmA, DM dmB, DMEnclosureType *rel)
4026 {
4027   DM             plexA, plexB, sdm;
4028   DMLabel        spmap;
4029   PetscInt       pStartA, pEndA, pStartB, pEndB, NpA, NpB;
4030 
4031   PetscFunctionBegin;
4032   PetscValidPointer(rel, 3);
4033   *rel = DM_ENC_NONE;
4034   if (!dmA || !dmB) PetscFunctionReturn(0);
4035   PetscValidHeaderSpecific(dmA, DM_CLASSID, 1);
4036   PetscValidHeaderSpecific(dmB, DM_CLASSID, 2);
4037   if (dmA == dmB) {*rel = DM_ENC_EQUALITY; PetscFunctionReturn(0);}
4038   PetscCall(DMConvert(dmA, DMPLEX, &plexA));
4039   PetscCall(DMConvert(dmB, DMPLEX, &plexB));
4040   PetscCall(DMPlexGetChart(plexA, &pStartA, &pEndA));
4041   PetscCall(DMPlexGetChart(plexB, &pStartB, &pEndB));
4042   /* Assumption 1: subDMs have smaller charts than the DMs that they originate from
4043     - The degenerate case of a subdomain which includes all of the domain on some process can be treated as equality */
4044   if ((pStartA == pStartB) && (pEndA == pEndB)) {
4045     *rel = DM_ENC_EQUALITY;
4046     goto end;
4047   }
4048   NpA = pEndA - pStartA;
4049   NpB = pEndB - pStartB;
4050   if (NpA == NpB) goto end;
4051   sdm = NpA > NpB ? plexB : plexA; /* The other is the original, enclosing dm */
4052   PetscCall(DMPlexGetSubpointMap(sdm, &spmap));
4053   if (!spmap) goto end;
4054   /* TODO Check the space mapped to by subpointMap is same size as dm */
4055   if (NpA > NpB) {
4056     *rel = DM_ENC_SUPERMESH;
4057   } else {
4058     *rel = DM_ENC_SUBMESH;
4059   }
4060   end:
4061   PetscCall(DMDestroy(&plexA));
4062   PetscCall(DMDestroy(&plexB));
4063   PetscFunctionReturn(0);
4064 }
4065 
4066 /*@
4067   DMGetEnclosurePoint - Get the point pA in dmA which corresponds to the point pB in dmB
4068 
4069   Input Parameters:
4070 + dmA   - The first DM
4071 . dmB   - The second DM
4072 . etype - The type of enclosure relation that dmA has to dmB
4073 - pB    - A point of dmB
4074 
4075   Output Parameter:
4076 . pA    - The corresponding point of dmA
4077 
4078   Level: intermediate
4079 
4080 .seealso: `DMGetEnclosureRelation()`
4081 @*/
4082 PetscErrorCode DMGetEnclosurePoint(DM dmA, DM dmB, DMEnclosureType etype, PetscInt pB, PetscInt *pA)
4083 {
4084   DM              sdm;
4085   IS              subpointIS;
4086   const PetscInt *subpoints;
4087   PetscInt        numSubpoints;
4088 
4089   PetscFunctionBegin;
4090   /* TODO Cache the IS, making it look like an index */
4091   switch (etype) {
4092     case DM_ENC_SUPERMESH:
4093     sdm  = dmB;
4094     PetscCall(DMPlexGetSubpointIS(sdm, &subpointIS));
4095     PetscCall(ISGetIndices(subpointIS, &subpoints));
4096     *pA  = subpoints[pB];
4097     PetscCall(ISRestoreIndices(subpointIS, &subpoints));
4098     break;
4099     case DM_ENC_SUBMESH:
4100     sdm  = dmA;
4101     PetscCall(DMPlexGetSubpointIS(sdm, &subpointIS));
4102     PetscCall(ISGetLocalSize(subpointIS, &numSubpoints));
4103     PetscCall(ISGetIndices(subpointIS, &subpoints));
4104     PetscCall(PetscFindInt(pB, numSubpoints, subpoints, pA));
4105     if (*pA < 0) {
4106       PetscCall(DMViewFromOptions(dmA, NULL, "-dm_enc_A_view"));
4107       PetscCall(DMViewFromOptions(dmB, NULL, "-dm_enc_B_view"));
4108       SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Point %" PetscInt_FMT " not found in submesh", pB);
4109     }
4110     PetscCall(ISRestoreIndices(subpointIS, &subpoints));
4111     break;
4112     case DM_ENC_EQUALITY:
4113     case DM_ENC_NONE:
4114     *pA = pB;break;
4115     case DM_ENC_UNKNOWN:
4116     {
4117       DMEnclosureType enc;
4118 
4119       PetscCall(DMGetEnclosureRelation(dmA, dmB, &enc));
4120       PetscCall(DMGetEnclosurePoint(dmA, dmB, enc, pB, pA));
4121     }
4122     break;
4123     default: SETERRQ(PetscObjectComm((PetscObject) dmA), PETSC_ERR_ARG_OUTOFRANGE, "Invalid enclosure type %d", (int) etype);
4124   }
4125   PetscFunctionReturn(0);
4126 }
4127