xref: /petsc/src/dm/impls/plex/plexsubmesh.c (revision c3e4dd79e17d3f0a51a524340fae4661630fd09e)
1 #include <petsc/private/dmpleximpl.h>    /*I      "petscdmplex.h"    I*/
2 #include <petsc/private/dmlabelimpl.h>   /*I      "petscdmlabel.h"   I*/
3 #include <petscsf.h>
4 
5 static PetscErrorCode DMPlexCellIsHybrid_Internal(DM dm, PetscInt p, PetscBool *isHybrid)
6 {
7   DMPolytopeType ct;
8 
9   PetscFunctionBegin;
10   PetscCall(DMPlexGetCellType(dm, p, &ct));
11   switch (ct) {
12     case DM_POLYTOPE_POINT_PRISM_TENSOR:
13     case DM_POLYTOPE_SEG_PRISM_TENSOR:
14     case DM_POLYTOPE_TRI_PRISM_TENSOR:
15     case DM_POLYTOPE_QUAD_PRISM_TENSOR:
16       *isHybrid = PETSC_TRUE;
17     default: *isHybrid = PETSC_FALSE;
18   }
19   PetscFunctionReturn(0);
20 }
21 
22 static PetscErrorCode DMPlexGetTensorPrismBounds_Internal(DM dm, PetscInt dim, PetscInt *cStart, PetscInt *cEnd)
23 {
24   DMLabel        ctLabel;
25 
26   PetscFunctionBegin;
27   if (cStart) *cStart = -1;
28   if (cEnd)   *cEnd   = -1;
29   PetscCall(DMPlexGetCellTypeLabel(dm, &ctLabel));
30   switch (dim) {
31     case 1: PetscCall(DMLabelGetStratumBounds(ctLabel, DM_POLYTOPE_POINT_PRISM_TENSOR, cStart, cEnd));break;
32     case 2: PetscCall(DMLabelGetStratumBounds(ctLabel, DM_POLYTOPE_SEG_PRISM_TENSOR, cStart, cEnd));break;
33     case 3:
34       PetscCall(DMLabelGetStratumBounds(ctLabel, DM_POLYTOPE_TRI_PRISM_TENSOR, cStart, cEnd));
35       if (*cStart < 0) PetscCall(DMLabelGetStratumBounds(ctLabel, DM_POLYTOPE_QUAD_PRISM_TENSOR, cStart, cEnd));
36       break;
37     default: PetscFunctionReturn(0);
38   }
39   PetscFunctionReturn(0);
40 }
41 
42 static PetscErrorCode DMPlexMarkBoundaryFaces_Internal(DM dm, PetscInt val, PetscInt cellHeight, DMLabel label)
43 {
44   PetscInt       fStart, fEnd, f;
45 
46   PetscFunctionBegin;
47   PetscCall(DMPlexGetHeightStratum(dm, cellHeight+1, &fStart, &fEnd));
48   for (f = fStart; f < fEnd; ++f) {
49     PetscInt supportSize;
50 
51     PetscCall(DMPlexGetSupportSize(dm, f, &supportSize));
52     if (supportSize == 1) {
53       if (val < 0) {
54         PetscInt *closure = NULL;
55         PetscInt  clSize, cl, cval;
56 
57         PetscCall(DMPlexGetTransitiveClosure(dm, f, PETSC_TRUE, &clSize, &closure));
58         for (cl = 0; cl < clSize*2; cl += 2) {
59           PetscCall(DMLabelGetValue(label, closure[cl], &cval));
60           if (cval < 0) continue;
61           PetscCall(DMLabelSetValue(label, f, cval));
62           break;
63         }
64         if (cl == clSize*2) PetscCall(DMLabelSetValue(label, f, 1));
65         PetscCall(DMPlexRestoreTransitiveClosure(dm, f, PETSC_TRUE, &clSize, &closure));
66       } else {
67         PetscCall(DMLabelSetValue(label, f, val));
68       }
69     }
70   }
71   PetscFunctionReturn(0);
72 }
73 
74 /*@
75   DMPlexMarkBoundaryFaces - Mark all faces on the boundary
76 
77   Not Collective
78 
79   Input Parameters:
80 + dm - The original DM
81 - val - The marker value, or PETSC_DETERMINE to use some value in the closure (or 1 if none are found)
82 
83   Output Parameter:
84 . label - The DMLabel marking boundary faces with the given value
85 
86   Level: developer
87 
88 .seealso: `DMLabelCreate()`, `DMCreateLabel()`
89 @*/
90 PetscErrorCode DMPlexMarkBoundaryFaces(DM dm, PetscInt val, DMLabel label)
91 {
92   DMPlexInterpolatedFlag  flg;
93 
94   PetscFunctionBegin;
95   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
96   PetscCall(DMPlexIsInterpolated(dm, &flg));
97   PetscCheck(flg == DMPLEX_INTERPOLATED_FULL,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM is not fully interpolated on this rank");
98   PetscCall(DMPlexMarkBoundaryFaces_Internal(dm, val, 0, label));
99   PetscFunctionReturn(0);
100 }
101 
102 static PetscErrorCode DMPlexLabelComplete_Internal(DM dm, DMLabel label, PetscBool completeCells)
103 {
104   IS              valueIS;
105   PetscSF         sfPoint;
106   const PetscInt *values;
107   PetscInt        numValues, v, cStart, cEnd, nroots;
108 
109   PetscFunctionBegin;
110   PetscCall(DMLabelGetNumValues(label, &numValues));
111   PetscCall(DMLabelGetValueIS(label, &valueIS));
112   PetscCall(DMPlexGetHeightStratum(dm,0,&cStart,&cEnd));
113   PetscCall(ISGetIndices(valueIS, &values));
114   for (v = 0; v < numValues; ++v) {
115     IS              pointIS;
116     const PetscInt *points;
117     PetscInt        numPoints, p;
118 
119     PetscCall(DMLabelGetStratumSize(label, values[v], &numPoints));
120     PetscCall(DMLabelGetStratumIS(label, values[v], &pointIS));
121     PetscCall(ISGetIndices(pointIS, &points));
122     for (p = 0; p < numPoints; ++p) {
123       PetscInt  q = points[p];
124       PetscInt *closure = NULL;
125       PetscInt  closureSize, c;
126 
127       if (cStart <= q && q < cEnd && !completeCells) { /* skip cells */
128         continue;
129       }
130       PetscCall(DMPlexGetTransitiveClosure(dm, q, PETSC_TRUE, &closureSize, &closure));
131       for (c = 0; c < closureSize*2; c += 2) {
132         PetscCall(DMLabelSetValue(label, closure[c], values[v]));
133       }
134       PetscCall(DMPlexRestoreTransitiveClosure(dm, q, PETSC_TRUE, &closureSize, &closure));
135     }
136     PetscCall(ISRestoreIndices(pointIS, &points));
137     PetscCall(ISDestroy(&pointIS));
138   }
139   PetscCall(ISRestoreIndices(valueIS, &values));
140   PetscCall(ISDestroy(&valueIS));
141   PetscCall(DMGetPointSF(dm, &sfPoint));
142   PetscCall(PetscSFGetGraph(sfPoint, &nroots, NULL, NULL, NULL));
143   if (nroots >= 0) {
144     DMLabel         lblRoots, lblLeaves;
145     IS              valueIS, pointIS;
146     const PetscInt *values;
147     PetscInt        numValues, v;
148 
149     /* Pull point contributions from remote leaves into local roots */
150     PetscCall(DMLabelGather(label, sfPoint, &lblLeaves));
151     PetscCall(DMLabelGetValueIS(lblLeaves, &valueIS));
152     PetscCall(ISGetLocalSize(valueIS, &numValues));
153     PetscCall(ISGetIndices(valueIS, &values));
154     for (v = 0; v < numValues; ++v) {
155       const PetscInt value = values[v];
156 
157       PetscCall(DMLabelGetStratumIS(lblLeaves, value, &pointIS));
158       PetscCall(DMLabelInsertIS(label, pointIS, value));
159       PetscCall(ISDestroy(&pointIS));
160     }
161     PetscCall(ISRestoreIndices(valueIS, &values));
162     PetscCall(ISDestroy(&valueIS));
163     PetscCall(DMLabelDestroy(&lblLeaves));
164     /* Push point contributions from roots into remote leaves */
165     PetscCall(DMLabelDistribute(label, sfPoint, &lblRoots));
166     PetscCall(DMLabelGetValueIS(lblRoots, &valueIS));
167     PetscCall(ISGetLocalSize(valueIS, &numValues));
168     PetscCall(ISGetIndices(valueIS, &values));
169     for (v = 0; v < numValues; ++v) {
170       const PetscInt value = values[v];
171 
172       PetscCall(DMLabelGetStratumIS(lblRoots, value, &pointIS));
173       PetscCall(DMLabelInsertIS(label, pointIS, value));
174       PetscCall(ISDestroy(&pointIS));
175     }
176     PetscCall(ISRestoreIndices(valueIS, &values));
177     PetscCall(ISDestroy(&valueIS));
178     PetscCall(DMLabelDestroy(&lblRoots));
179   }
180   PetscFunctionReturn(0);
181 }
182 
183 /*@
184   DMPlexLabelComplete - Starting with a label marking points on a surface, we add the transitive closure to the surface
185 
186   Input Parameters:
187 + dm - The DM
188 - label - A DMLabel marking the surface points
189 
190   Output Parameter:
191 . label - A DMLabel marking all surface points in the transitive closure
192 
193   Level: developer
194 
195 .seealso: `DMPlexLabelCohesiveComplete()`
196 @*/
197 PetscErrorCode DMPlexLabelComplete(DM dm, DMLabel label)
198 {
199   PetscFunctionBegin;
200   PetscCall(DMPlexLabelComplete_Internal(dm, label, PETSC_TRUE));
201   PetscFunctionReturn(0);
202 }
203 
204 /*@
205   DMPlexLabelAddCells - Starting with a label marking points on a surface, we add a cell for each point
206 
207   Input Parameters:
208 + dm - The DM
209 - label - A DMLabel marking the surface points
210 
211   Output Parameter:
212 . label - A DMLabel incorporating cells
213 
214   Level: developer
215 
216   Note: The cells allow FEM boundary conditions to be applied using the cell geometry
217 
218 .seealso: `DMPlexLabelAddFaceCells()`, `DMPlexLabelComplete()`, `DMPlexLabelCohesiveComplete()`
219 @*/
220 PetscErrorCode DMPlexLabelAddCells(DM dm, DMLabel label)
221 {
222   IS              valueIS;
223   const PetscInt *values;
224   PetscInt        numValues, v, cStart, cEnd;
225 
226   PetscFunctionBegin;
227   PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd));
228   PetscCall(DMLabelGetNumValues(label, &numValues));
229   PetscCall(DMLabelGetValueIS(label, &valueIS));
230   PetscCall(ISGetIndices(valueIS, &values));
231   for (v = 0; v < numValues; ++v) {
232     IS              pointIS;
233     const PetscInt *points;
234     PetscInt        numPoints, p;
235 
236     PetscCall(DMLabelGetStratumSize(label, values[v], &numPoints));
237     PetscCall(DMLabelGetStratumIS(label, values[v], &pointIS));
238     PetscCall(ISGetIndices(pointIS, &points));
239     for (p = 0; p < numPoints; ++p) {
240       PetscInt *closure = NULL;
241       PetscInt  closureSize, cl;
242 
243       PetscCall(DMPlexGetTransitiveClosure(dm, points[p], PETSC_FALSE, &closureSize, &closure));
244       for (cl = closureSize-1; cl > 0; --cl) {
245         const PetscInt cell = closure[cl*2];
246         if ((cell >= cStart) && (cell < cEnd)) {PetscCall(DMLabelSetValue(label, cell, values[v])); break;}
247       }
248       PetscCall(DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_FALSE, &closureSize, &closure));
249     }
250     PetscCall(ISRestoreIndices(pointIS, &points));
251     PetscCall(ISDestroy(&pointIS));
252   }
253   PetscCall(ISRestoreIndices(valueIS, &values));
254   PetscCall(ISDestroy(&valueIS));
255   PetscFunctionReturn(0);
256 }
257 
258 /*@
259   DMPlexLabelAddFaceCells - Starting with a label marking faces on a surface, we add a cell for each face
260 
261   Input Parameters:
262 + dm - The DM
263 - label - A DMLabel marking the surface points
264 
265   Output Parameter:
266 . label - A DMLabel incorporating cells
267 
268   Level: developer
269 
270   Note: The cells allow FEM boundary conditions to be applied using the cell geometry
271 
272 .seealso: `DMPlexLabelAddCells()`, `DMPlexLabelComplete()`, `DMPlexLabelCohesiveComplete()`
273 @*/
274 PetscErrorCode DMPlexLabelAddFaceCells(DM dm, DMLabel label)
275 {
276   IS              valueIS;
277   const PetscInt *values;
278   PetscInt        numValues, v, cStart, cEnd, fStart, fEnd;
279 
280   PetscFunctionBegin;
281   PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd));
282   PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
283   PetscCall(DMLabelGetNumValues(label, &numValues));
284   PetscCall(DMLabelGetValueIS(label, &valueIS));
285   PetscCall(ISGetIndices(valueIS, &values));
286   for (v = 0; v < numValues; ++v) {
287     IS              pointIS;
288     const PetscInt *points;
289     PetscInt        numPoints, p;
290 
291     PetscCall(DMLabelGetStratumSize(label, values[v], &numPoints));
292     PetscCall(DMLabelGetStratumIS(label, values[v], &pointIS));
293     PetscCall(ISGetIndices(pointIS, &points));
294     for (p = 0; p < numPoints; ++p) {
295       const PetscInt face = points[p];
296       PetscInt      *closure = NULL;
297       PetscInt       closureSize, cl;
298 
299       if ((face < fStart) || (face >= fEnd)) continue;
300       PetscCall(DMPlexGetTransitiveClosure(dm, face, PETSC_FALSE, &closureSize, &closure));
301       for (cl = closureSize-1; cl > 0; --cl) {
302         const PetscInt cell = closure[cl*2];
303         if ((cell >= cStart) && (cell < cEnd)) {PetscCall(DMLabelSetValue(label, cell, values[v])); break;}
304       }
305       PetscCall(DMPlexRestoreTransitiveClosure(dm, face, PETSC_FALSE, &closureSize, &closure));
306     }
307     PetscCall(ISRestoreIndices(pointIS, &points));
308     PetscCall(ISDestroy(&pointIS));
309   }
310   PetscCall(ISRestoreIndices(valueIS, &values));
311   PetscCall(ISDestroy(&valueIS));
312   PetscFunctionReturn(0);
313 }
314 
315 /*@
316   DMPlexLabelClearCells - Remove cells from a label
317 
318   Input Parameters:
319 + dm - The DM
320 - label - A DMLabel marking surface points and their adjacent cells
321 
322   Output Parameter:
323 . label - A DMLabel without cells
324 
325   Level: developer
326 
327   Note: This undoes DMPlexLabelAddCells() or DMPlexLabelAddFaceCells()
328 
329 .seealso: `DMPlexLabelComplete()`, `DMPlexLabelCohesiveComplete()`, `DMPlexLabelAddCells()`
330 @*/
331 PetscErrorCode DMPlexLabelClearCells(DM dm, DMLabel label)
332 {
333   IS              valueIS;
334   const PetscInt *values;
335   PetscInt        numValues, v, cStart, cEnd;
336 
337   PetscFunctionBegin;
338   PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd));
339   PetscCall(DMLabelGetNumValues(label, &numValues));
340   PetscCall(DMLabelGetValueIS(label, &valueIS));
341   PetscCall(ISGetIndices(valueIS, &values));
342   for (v = 0; v < numValues; ++v) {
343     IS              pointIS;
344     const PetscInt *points;
345     PetscInt        numPoints, p;
346 
347     PetscCall(DMLabelGetStratumSize(label, values[v], &numPoints));
348     PetscCall(DMLabelGetStratumIS(label, values[v], &pointIS));
349     PetscCall(ISGetIndices(pointIS, &points));
350     for (p = 0; p < numPoints; ++p) {
351       PetscInt point = points[p];
352 
353       if (point >= cStart && point < cEnd) {
354         PetscCall(DMLabelClearValue(label,point,values[v]));
355       }
356     }
357     PetscCall(ISRestoreIndices(pointIS, &points));
358     PetscCall(ISDestroy(&pointIS));
359   }
360   PetscCall(ISRestoreIndices(valueIS, &values));
361   PetscCall(ISDestroy(&valueIS));
362   PetscFunctionReturn(0);
363 }
364 
365 /* take (oldEnd, added) pairs, ordered by height and convert them to (oldstart, newstart) pairs, ordered by ascending
366  * index (skipping first, which is (0,0)) */
367 static inline PetscErrorCode DMPlexShiftPointSetUp_Internal(PetscInt depth, PetscInt depthShift[])
368 {
369   PetscInt d, off = 0;
370 
371   PetscFunctionBegin;
372   /* sort by (oldend): yes this is an O(n^2) sort, we expect depth <= 3 */
373   for (d = 0; d < depth; d++) {
374     PetscInt firstd = d;
375     PetscInt firstStart = depthShift[2*d];
376     PetscInt e;
377 
378     for (e = d+1; e <= depth; e++) {
379       if (depthShift[2*e] < firstStart) {
380         firstd = e;
381         firstStart = depthShift[2*d];
382       }
383     }
384     if (firstd != d) {
385       PetscInt swap[2];
386 
387       e = firstd;
388       swap[0] = depthShift[2*d];
389       swap[1] = depthShift[2*d+1];
390       depthShift[2*d]   = depthShift[2*e];
391       depthShift[2*d+1] = depthShift[2*e+1];
392       depthShift[2*e]   = swap[0];
393       depthShift[2*e+1] = swap[1];
394     }
395   }
396   /* convert (oldstart, added) to (oldstart, newstart) */
397   for (d = 0; d <= depth; d++) {
398     off += depthShift[2*d+1];
399     depthShift[2*d+1] = depthShift[2*d] + off;
400   }
401   PetscFunctionReturn(0);
402 }
403 
404 /* depthShift is a list of (old, new) pairs */
405 static inline PetscInt DMPlexShiftPoint_Internal(PetscInt p, PetscInt depth, PetscInt depthShift[])
406 {
407   PetscInt d;
408   PetscInt newOff = 0;
409 
410   for (d = 0; d <= depth; d++) {
411     if (p < depthShift[2*d]) return p + newOff;
412     else newOff = depthShift[2*d+1] - depthShift[2*d];
413   }
414   return p + newOff;
415 }
416 
417 /* depthShift is a list of (old, new) pairs */
418 static inline PetscInt DMPlexShiftPointInverse_Internal(PetscInt p, PetscInt depth, PetscInt depthShift[])
419 {
420   PetscInt d;
421   PetscInt newOff = 0;
422 
423   for (d = 0; d <= depth; d++) {
424     if (p < depthShift[2*d+1]) return p + newOff;
425     else newOff = depthShift[2*d] - depthShift[2*d+1];
426   }
427   return p + newOff;
428 }
429 
430 static PetscErrorCode DMPlexShiftSizes_Internal(DM dm, PetscInt depthShift[], DM dmNew)
431 {
432   PetscInt       depth = 0, d, pStart, pEnd, p;
433   DMLabel        depthLabel;
434 
435   PetscFunctionBegin;
436   PetscCall(DMPlexGetDepth(dm, &depth));
437   if (depth < 0) PetscFunctionReturn(0);
438   /* Step 1: Expand chart */
439   PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
440   pEnd = DMPlexShiftPoint_Internal(pEnd,depth,depthShift);
441   PetscCall(DMPlexSetChart(dmNew, pStart, pEnd));
442   PetscCall(DMCreateLabel(dmNew,"depth"));
443   PetscCall(DMPlexGetDepthLabel(dmNew,&depthLabel));
444   PetscCall(DMCreateLabel(dmNew, "celltype"));
445   /* Step 2: Set cone and support sizes */
446   for (d = 0; d <= depth; ++d) {
447     PetscInt pStartNew, pEndNew;
448     IS pIS;
449 
450     PetscCall(DMPlexGetDepthStratum(dm, d, &pStart, &pEnd));
451     pStartNew = DMPlexShiftPoint_Internal(pStart, depth, depthShift);
452     pEndNew = DMPlexShiftPoint_Internal(pEnd, depth, depthShift);
453     PetscCall(ISCreateStride(PETSC_COMM_SELF, pEndNew - pStartNew, pStartNew, 1, &pIS));
454     PetscCall(DMLabelSetStratumIS(depthLabel, d, pIS));
455     PetscCall(ISDestroy(&pIS));
456     for (p = pStart; p < pEnd; ++p) {
457       PetscInt       newp = DMPlexShiftPoint_Internal(p, depth, depthShift);
458       PetscInt       size;
459       DMPolytopeType ct;
460 
461       PetscCall(DMPlexGetConeSize(dm, p, &size));
462       PetscCall(DMPlexSetConeSize(dmNew, newp, size));
463       PetscCall(DMPlexGetSupportSize(dm, p, &size));
464       PetscCall(DMPlexSetSupportSize(dmNew, newp, size));
465       PetscCall(DMPlexGetCellType(dm, p, &ct));
466       PetscCall(DMPlexSetCellType(dmNew, newp, ct));
467     }
468   }
469   PetscFunctionReturn(0);
470 }
471 
472 static PetscErrorCode DMPlexShiftPoints_Internal(DM dm, PetscInt depthShift[], DM dmNew)
473 {
474   PetscInt      *newpoints;
475   PetscInt       depth = 0, maxConeSize, maxSupportSize, maxConeSizeNew, maxSupportSizeNew, pStart, pEnd, p;
476 
477   PetscFunctionBegin;
478   PetscCall(DMPlexGetDepth(dm, &depth));
479   if (depth < 0) PetscFunctionReturn(0);
480   PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize));
481   PetscCall(DMPlexGetMaxSizes(dmNew, &maxConeSizeNew, &maxSupportSizeNew));
482   PetscCall(PetscMalloc1(PetscMax(PetscMax(maxConeSize, maxSupportSize), PetscMax(maxConeSizeNew, maxSupportSizeNew)),&newpoints));
483   /* Step 5: Set cones and supports */
484   PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
485   for (p = pStart; p < pEnd; ++p) {
486     const PetscInt *points = NULL, *orientations = NULL;
487     PetscInt        size,sizeNew, i, newp = DMPlexShiftPoint_Internal(p, depth, depthShift);
488 
489     PetscCall(DMPlexGetConeSize(dm, p, &size));
490     PetscCall(DMPlexGetCone(dm, p, &points));
491     PetscCall(DMPlexGetConeOrientation(dm, p, &orientations));
492     for (i = 0; i < size; ++i) {
493       newpoints[i] = DMPlexShiftPoint_Internal(points[i], depth, depthShift);
494     }
495     PetscCall(DMPlexSetCone(dmNew, newp, newpoints));
496     PetscCall(DMPlexSetConeOrientation(dmNew, newp, orientations));
497     PetscCall(DMPlexGetSupportSize(dm, p, &size));
498     PetscCall(DMPlexGetSupportSize(dmNew, newp, &sizeNew));
499     PetscCall(DMPlexGetSupport(dm, p, &points));
500     for (i = 0; i < size; ++i) {
501       newpoints[i] = DMPlexShiftPoint_Internal(points[i], depth, depthShift);
502     }
503     for (i = size; i < sizeNew; ++i) newpoints[i] = 0;
504     PetscCall(DMPlexSetSupport(dmNew, newp, newpoints));
505   }
506   PetscCall(PetscFree(newpoints));
507   PetscFunctionReturn(0);
508 }
509 
510 static PetscErrorCode DMPlexShiftCoordinates_Internal(DM dm, PetscInt depthShift[], DM dmNew)
511 {
512   PetscSection   coordSection, newCoordSection;
513   Vec            coordinates, newCoordinates;
514   PetscScalar   *coords, *newCoords;
515   PetscInt       coordSize, sStart, sEnd;
516   PetscInt       dim, depth = 0, cStart, cEnd, cStartNew, cEndNew, c, vStart, vEnd, vStartNew, vEndNew, v;
517   PetscBool      hasCells;
518 
519   PetscFunctionBegin;
520   PetscCall(DMGetCoordinateDim(dm, &dim));
521   PetscCall(DMSetCoordinateDim(dmNew, dim));
522   PetscCall(DMPlexGetDepth(dm, &depth));
523   /* Step 8: Convert coordinates */
524   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
525   PetscCall(DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd));
526   PetscCall(DMPlexGetDepthStratum(dmNew, 0, &vStartNew, &vEndNew));
527   PetscCall(DMPlexGetHeightStratum(dmNew, 0, &cStartNew, &cEndNew));
528   PetscCall(DMGetCoordinateSection(dm, &coordSection));
529   PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dm), &newCoordSection));
530   PetscCall(PetscSectionSetNumFields(newCoordSection, 1));
531   PetscCall(PetscSectionSetFieldComponents(newCoordSection, 0, dim));
532   PetscCall(PetscSectionGetChart(coordSection, &sStart, &sEnd));
533   hasCells = sStart == cStart ? PETSC_TRUE : PETSC_FALSE;
534   PetscCall(PetscSectionSetChart(newCoordSection, hasCells ? cStartNew : vStartNew, vEndNew));
535   if (hasCells) {
536     for (c = cStart; c < cEnd; ++c) {
537       PetscInt cNew = DMPlexShiftPoint_Internal(c, depth, depthShift), dof;
538 
539       PetscCall(PetscSectionGetDof(coordSection, c, &dof));
540       PetscCall(PetscSectionSetDof(newCoordSection, cNew, dof));
541       PetscCall(PetscSectionSetFieldDof(newCoordSection, cNew, 0, dof));
542     }
543   }
544   for (v = vStartNew; v < vEndNew; ++v) {
545     PetscCall(PetscSectionSetDof(newCoordSection, v, dim));
546     PetscCall(PetscSectionSetFieldDof(newCoordSection, v, 0, dim));
547   }
548   PetscCall(PetscSectionSetUp(newCoordSection));
549   PetscCall(DMSetCoordinateSection(dmNew, PETSC_DETERMINE, newCoordSection));
550   PetscCall(PetscSectionGetStorageSize(newCoordSection, &coordSize));
551   PetscCall(VecCreate(PETSC_COMM_SELF, &newCoordinates));
552   PetscCall(PetscObjectSetName((PetscObject) newCoordinates, "coordinates"));
553   PetscCall(VecSetSizes(newCoordinates, coordSize, PETSC_DETERMINE));
554   PetscCall(VecSetBlockSize(newCoordinates, dim));
555   PetscCall(VecSetType(newCoordinates,VECSTANDARD));
556   PetscCall(DMSetCoordinatesLocal(dmNew, newCoordinates));
557   PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
558   PetscCall(VecGetArray(coordinates, &coords));
559   PetscCall(VecGetArray(newCoordinates, &newCoords));
560   if (hasCells) {
561     for (c = cStart; c < cEnd; ++c) {
562       PetscInt cNew = DMPlexShiftPoint_Internal(c, depth, depthShift), dof, off, noff, d;
563 
564       PetscCall(PetscSectionGetDof(coordSection, c, &dof));
565       PetscCall(PetscSectionGetOffset(coordSection, c, &off));
566       PetscCall(PetscSectionGetOffset(newCoordSection, cNew, &noff));
567       for (d = 0; d < dof; ++d) newCoords[noff+d] = coords[off+d];
568     }
569   }
570   for (v = vStart; v < vEnd; ++v) {
571     PetscInt dof, off, noff, d;
572 
573     PetscCall(PetscSectionGetDof(coordSection, v, &dof));
574     PetscCall(PetscSectionGetOffset(coordSection, v, &off));
575     PetscCall(PetscSectionGetOffset(newCoordSection, DMPlexShiftPoint_Internal(v, depth, depthShift), &noff));
576     for (d = 0; d < dof; ++d) newCoords[noff+d] = coords[off+d];
577   }
578   PetscCall(VecRestoreArray(coordinates, &coords));
579   PetscCall(VecRestoreArray(newCoordinates, &newCoords));
580   PetscCall(VecDestroy(&newCoordinates));
581   PetscCall(PetscSectionDestroy(&newCoordSection));
582   PetscFunctionReturn(0);
583 }
584 
585 static PetscErrorCode DMPlexShiftSF_Single(DM dm, PetscInt depthShift[], PetscSF sf, PetscSF sfNew)
586 {
587   const PetscSFNode *remotePoints;
588   PetscSFNode       *gremotePoints;
589   const PetscInt    *localPoints;
590   PetscInt          *glocalPoints, *newLocation, *newRemoteLocation;
591   PetscInt           numRoots, numLeaves, l, pStart, pEnd, depth = 0, totShift = 0;
592 
593   PetscFunctionBegin;
594   PetscCall(DMPlexGetDepth(dm, &depth));
595   PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
596   PetscCall(PetscSFGetGraph(sf, &numRoots, &numLeaves, &localPoints, &remotePoints));
597   totShift = DMPlexShiftPoint_Internal(pEnd, depth, depthShift) - pEnd;
598   if (numRoots >= 0) {
599     PetscCall(PetscMalloc2(numRoots, &newLocation, pEnd-pStart, &newRemoteLocation));
600     for (l = 0; l < numRoots; ++l) newLocation[l] = DMPlexShiftPoint_Internal(l, depth, depthShift);
601     PetscCall(PetscSFBcastBegin(sf, MPIU_INT, newLocation, newRemoteLocation, MPI_REPLACE));
602     PetscCall(PetscSFBcastEnd(sf, MPIU_INT, newLocation, newRemoteLocation, MPI_REPLACE));
603     PetscCall(PetscMalloc1(numLeaves, &glocalPoints));
604     PetscCall(PetscMalloc1(numLeaves, &gremotePoints));
605     for (l = 0; l < numLeaves; ++l) {
606       glocalPoints[l]        = DMPlexShiftPoint_Internal(localPoints[l], depth, depthShift);
607       gremotePoints[l].rank  = remotePoints[l].rank;
608       gremotePoints[l].index = newRemoteLocation[localPoints[l]];
609     }
610     PetscCall(PetscFree2(newLocation, newRemoteLocation));
611     PetscCall(PetscSFSetGraph(sfNew, numRoots + totShift, numLeaves, glocalPoints, PETSC_OWN_POINTER, gremotePoints, PETSC_OWN_POINTER));
612   }
613   PetscFunctionReturn(0);
614 }
615 
616 static PetscErrorCode DMPlexShiftSF_Internal(DM dm, PetscInt depthShift[], DM dmNew)
617 {
618   PetscSF        sfPoint, sfPointNew;
619   PetscBool      useNatural;
620 
621   PetscFunctionBegin;
622   /* Step 9: Convert pointSF */
623   PetscCall(DMGetPointSF(dm, &sfPoint));
624   PetscCall(DMGetPointSF(dmNew, &sfPointNew));
625   PetscCall(DMPlexShiftSF_Single(dm, depthShift, sfPoint, sfPointNew));
626   /* Step 9b: Convert naturalSF */
627   PetscCall(DMGetUseNatural(dm, &useNatural));
628   if (useNatural) {
629     PetscSF sfNat, sfNatNew;
630 
631     PetscCall(DMSetUseNatural(dmNew, useNatural));
632     PetscCall(DMGetNaturalSF(dm, &sfNat));
633     PetscCall(DMGetNaturalSF(dmNew, &sfNatNew));
634     PetscCall(DMPlexShiftSF_Single(dm, depthShift, sfNat, sfNatNew));
635   }
636   PetscFunctionReturn(0);
637 }
638 
639 static PetscErrorCode DMPlexShiftLabels_Internal(DM dm, PetscInt depthShift[], DM dmNew)
640 {
641   PetscInt           depth = 0, numLabels, l;
642 
643   PetscFunctionBegin;
644   PetscCall(DMPlexGetDepth(dm, &depth));
645   /* Step 10: Convert labels */
646   PetscCall(DMGetNumLabels(dm, &numLabels));
647   for (l = 0; l < numLabels; ++l) {
648     DMLabel         label, newlabel;
649     const char     *lname;
650     PetscBool       isDepth, isDim;
651     IS              valueIS;
652     const PetscInt *values;
653     PetscInt        numValues, val;
654 
655     PetscCall(DMGetLabelName(dm, l, &lname));
656     PetscCall(PetscStrcmp(lname, "depth", &isDepth));
657     if (isDepth) continue;
658     PetscCall(PetscStrcmp(lname, "dim", &isDim));
659     if (isDim) continue;
660     PetscCall(DMCreateLabel(dmNew, lname));
661     PetscCall(DMGetLabel(dm, lname, &label));
662     PetscCall(DMGetLabel(dmNew, lname, &newlabel));
663     PetscCall(DMLabelGetDefaultValue(label,&val));
664     PetscCall(DMLabelSetDefaultValue(newlabel,val));
665     PetscCall(DMLabelGetValueIS(label, &valueIS));
666     PetscCall(ISGetLocalSize(valueIS, &numValues));
667     PetscCall(ISGetIndices(valueIS, &values));
668     for (val = 0; val < numValues; ++val) {
669       IS              pointIS;
670       const PetscInt *points;
671       PetscInt        numPoints, p;
672 
673       PetscCall(DMLabelGetStratumIS(label, values[val], &pointIS));
674       PetscCall(ISGetLocalSize(pointIS, &numPoints));
675       PetscCall(ISGetIndices(pointIS, &points));
676       for (p = 0; p < numPoints; ++p) {
677         const PetscInt newpoint = DMPlexShiftPoint_Internal(points[p], depth, depthShift);
678 
679         PetscCall(DMLabelSetValue(newlabel, newpoint, values[val]));
680       }
681       PetscCall(ISRestoreIndices(pointIS, &points));
682       PetscCall(ISDestroy(&pointIS));
683     }
684     PetscCall(ISRestoreIndices(valueIS, &values));
685     PetscCall(ISDestroy(&valueIS));
686   }
687   PetscFunctionReturn(0);
688 }
689 
690 static PetscErrorCode DMPlexCreateVTKLabel_Internal(DM dm, PetscBool createGhostLabel, DM dmNew)
691 {
692   PetscSF            sfPoint;
693   DMLabel            vtkLabel, ghostLabel = NULL;
694   const PetscSFNode *leafRemote;
695   const PetscInt    *leafLocal;
696   PetscInt           cellHeight, cStart, cEnd, c, fStart, fEnd, f, numLeaves, l;
697   PetscMPIInt        rank;
698 
699   PetscFunctionBegin;
700    /* Step 11: Make label for output (vtk) and to mark ghost points (ghost) */
701   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
702   PetscCall(DMGetPointSF(dm, &sfPoint));
703   PetscCall(DMPlexGetVTKCellHeight(dmNew, &cellHeight));
704   PetscCall(DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd));
705   PetscCall(PetscSFGetGraph(sfPoint, NULL, &numLeaves, &leafLocal, &leafRemote));
706   PetscCall(DMCreateLabel(dmNew, "vtk"));
707   PetscCall(DMGetLabel(dmNew, "vtk", &vtkLabel));
708   if (createGhostLabel) {
709     PetscCall(DMCreateLabel(dmNew, "ghost"));
710     PetscCall(DMGetLabel(dmNew, "ghost", &ghostLabel));
711   }
712   for (l = 0, c = cStart; l < numLeaves && c < cEnd; ++l, ++c) {
713     for (; c < leafLocal[l] && c < cEnd; ++c) {
714       PetscCall(DMLabelSetValue(vtkLabel, c, 1));
715     }
716     if (leafLocal[l] >= cEnd) break;
717     if (leafRemote[l].rank == rank) {
718       PetscCall(DMLabelSetValue(vtkLabel, c, 1));
719     } else if (ghostLabel) PetscCall(DMLabelSetValue(ghostLabel, c, 2));
720   }
721   for (; c < cEnd; ++c) {
722     PetscCall(DMLabelSetValue(vtkLabel, c, 1));
723   }
724   if (ghostLabel) {
725     PetscCall(DMPlexGetHeightStratum(dmNew, 1, &fStart, &fEnd));
726     for (f = fStart; f < fEnd; ++f) {
727       PetscInt numCells;
728 
729       PetscCall(DMPlexGetSupportSize(dmNew, f, &numCells));
730       if (numCells < 2) {
731         PetscCall(DMLabelSetValue(ghostLabel, f, 1));
732       } else {
733         const PetscInt *cells = NULL;
734         PetscInt        vA, vB;
735 
736         PetscCall(DMPlexGetSupport(dmNew, f, &cells));
737         PetscCall(DMLabelGetValue(vtkLabel, cells[0], &vA));
738         PetscCall(DMLabelGetValue(vtkLabel, cells[1], &vB));
739         if (vA != 1 && vB != 1) PetscCall(DMLabelSetValue(ghostLabel, f, 1));
740       }
741     }
742   }
743   PetscFunctionReturn(0);
744 }
745 
746 static PetscErrorCode DMPlexShiftTree_Internal(DM dm, PetscInt depthShift[], DM dmNew)
747 {
748   DM             refTree;
749   PetscSection   pSec;
750   PetscInt       *parents, *childIDs;
751 
752   PetscFunctionBegin;
753   PetscCall(DMPlexGetReferenceTree(dm,&refTree));
754   PetscCall(DMPlexSetReferenceTree(dmNew,refTree));
755   PetscCall(DMPlexGetTree(dm,&pSec,&parents,&childIDs,NULL,NULL));
756   if (pSec) {
757     PetscInt p, pStart, pEnd, *parentsShifted, pStartShifted, pEndShifted, depth;
758     PetscInt *childIDsShifted;
759     PetscSection pSecShifted;
760 
761     PetscCall(PetscSectionGetChart(pSec,&pStart,&pEnd));
762     PetscCall(DMPlexGetDepth(dm,&depth));
763     pStartShifted = DMPlexShiftPoint_Internal(pStart,depth,depthShift);
764     pEndShifted   = DMPlexShiftPoint_Internal(pEnd,depth,depthShift);
765     PetscCall(PetscMalloc2(pEndShifted - pStartShifted,&parentsShifted,pEndShifted-pStartShifted,&childIDsShifted));
766     PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dmNew),&pSecShifted));
767     PetscCall(PetscSectionSetChart(pSecShifted,pStartShifted,pEndShifted));
768     for (p = pStartShifted; p < pEndShifted; p++) {
769       /* start off assuming no children */
770       PetscCall(PetscSectionSetDof(pSecShifted,p,0));
771     }
772     for (p = pStart; p < pEnd; p++) {
773       PetscInt dof;
774       PetscInt pNew = DMPlexShiftPoint_Internal(p,depth,depthShift);
775 
776       PetscCall(PetscSectionGetDof(pSec,p,&dof));
777       PetscCall(PetscSectionSetDof(pSecShifted,pNew,dof));
778     }
779     PetscCall(PetscSectionSetUp(pSecShifted));
780     for (p = pStart; p < pEnd; p++) {
781       PetscInt dof;
782       PetscInt pNew = DMPlexShiftPoint_Internal(p,depth,depthShift);
783 
784       PetscCall(PetscSectionGetDof(pSec,p,&dof));
785       if (dof) {
786         PetscInt off, offNew;
787 
788         PetscCall(PetscSectionGetOffset(pSec,p,&off));
789         PetscCall(PetscSectionGetOffset(pSecShifted,pNew,&offNew));
790         parentsShifted[offNew] = DMPlexShiftPoint_Internal(parents[off],depth,depthShift);
791         childIDsShifted[offNew] = childIDs[off];
792       }
793     }
794     PetscCall(DMPlexSetTree(dmNew,pSecShifted,parentsShifted,childIDsShifted));
795     PetscCall(PetscFree2(parentsShifted,childIDsShifted));
796     PetscCall(PetscSectionDestroy(&pSecShifted));
797   }
798   PetscFunctionReturn(0);
799 }
800 
801 static PetscErrorCode DMPlexConstructGhostCells_Internal(DM dm, DMLabel label, PetscInt *numGhostCells, DM gdm)
802 {
803   PetscSF               sf;
804   IS                    valueIS;
805   const PetscInt       *values, *leaves;
806   PetscInt             *depthShift;
807   PetscInt              d, depth = 0, nleaves, loc, Ng, numFS, fs, fStart, fEnd, ghostCell, cEnd, c;
808   PetscBool             isper;
809   const PetscReal      *maxCell, *L;
810   const DMBoundaryType *bd;
811 
812   PetscFunctionBegin;
813   PetscCall(DMGetPointSF(dm, &sf));
814   PetscCall(PetscSFGetGraph(sf, NULL, &nleaves, &leaves, NULL));
815   nleaves = PetscMax(0, nleaves);
816   PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
817   /* Count ghost cells */
818   PetscCall(DMLabelGetValueIS(label, &valueIS));
819   PetscCall(ISGetLocalSize(valueIS, &numFS));
820   PetscCall(ISGetIndices(valueIS, &values));
821   Ng   = 0;
822   for (fs = 0; fs < numFS; ++fs) {
823     IS              faceIS;
824     const PetscInt *faces;
825     PetscInt        numFaces, f, numBdFaces = 0;
826 
827     PetscCall(DMLabelGetStratumIS(label, values[fs], &faceIS));
828     PetscCall(ISGetLocalSize(faceIS, &numFaces));
829     PetscCall(ISGetIndices(faceIS, &faces));
830     for (f = 0; f < numFaces; ++f) {
831       PetscInt numChildren;
832 
833       PetscCall(PetscFindInt(faces[f], nleaves, leaves, &loc));
834       PetscCall(DMPlexGetTreeChildren(dm,faces[f],&numChildren,NULL));
835       /* non-local and ancestors points don't get to register ghosts */
836       if (loc >= 0 || numChildren) continue;
837       if ((faces[f] >= fStart) && (faces[f] < fEnd)) ++numBdFaces;
838     }
839     Ng += numBdFaces;
840     PetscCall(ISRestoreIndices(faceIS, &faces));
841     PetscCall(ISDestroy(&faceIS));
842   }
843   PetscCall(DMPlexGetDepth(dm, &depth));
844   PetscCall(PetscMalloc1(2*(depth+1), &depthShift));
845   for (d = 0; d <= depth; d++) {
846     PetscInt dEnd;
847 
848     PetscCall(DMPlexGetDepthStratum(dm,d,NULL,&dEnd));
849     depthShift[2*d]   = dEnd;
850     depthShift[2*d+1] = 0;
851   }
852   if (depth >= 0) depthShift[2*depth+1] = Ng;
853   PetscCall(DMPlexShiftPointSetUp_Internal(depth,depthShift));
854   PetscCall(DMPlexShiftSizes_Internal(dm, depthShift, gdm));
855   /* Step 3: Set cone/support sizes for new points */
856   PetscCall(DMPlexGetHeightStratum(dm, 0, NULL, &cEnd));
857   for (c = cEnd; c < cEnd + Ng; ++c) {
858     PetscCall(DMPlexSetConeSize(gdm, c, 1));
859   }
860   for (fs = 0; fs < numFS; ++fs) {
861     IS              faceIS;
862     const PetscInt *faces;
863     PetscInt        numFaces, f;
864 
865     PetscCall(DMLabelGetStratumIS(label, values[fs], &faceIS));
866     PetscCall(ISGetLocalSize(faceIS, &numFaces));
867     PetscCall(ISGetIndices(faceIS, &faces));
868     for (f = 0; f < numFaces; ++f) {
869       PetscInt size, numChildren;
870 
871       PetscCall(PetscFindInt(faces[f], nleaves, leaves, &loc));
872       PetscCall(DMPlexGetTreeChildren(dm,faces[f],&numChildren,NULL));
873       if (loc >= 0 || numChildren) continue;
874       if ((faces[f] < fStart) || (faces[f] >= fEnd)) continue;
875       PetscCall(DMPlexGetSupportSize(dm, faces[f], &size));
876       PetscCheck(size == 1,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM has boundary face %" PetscInt_FMT " with %" PetscInt_FMT " support cells", faces[f], size);
877       PetscCall(DMPlexSetSupportSize(gdm, faces[f] + Ng, 2));
878     }
879     PetscCall(ISRestoreIndices(faceIS, &faces));
880     PetscCall(ISDestroy(&faceIS));
881   }
882   /* Step 4: Setup ghosted DM */
883   PetscCall(DMSetUp(gdm));
884   PetscCall(DMPlexShiftPoints_Internal(dm, depthShift, gdm));
885   /* Step 6: Set cones and supports for new points */
886   ghostCell = cEnd;
887   for (fs = 0; fs < numFS; ++fs) {
888     IS              faceIS;
889     const PetscInt *faces;
890     PetscInt        numFaces, f;
891 
892     PetscCall(DMLabelGetStratumIS(label, values[fs], &faceIS));
893     PetscCall(ISGetLocalSize(faceIS, &numFaces));
894     PetscCall(ISGetIndices(faceIS, &faces));
895     for (f = 0; f < numFaces; ++f) {
896       PetscInt newFace = faces[f] + Ng, numChildren;
897 
898       PetscCall(PetscFindInt(faces[f], nleaves, leaves, &loc));
899       PetscCall(DMPlexGetTreeChildren(dm,faces[f],&numChildren,NULL));
900       if (loc >= 0 || numChildren) continue;
901       if ((faces[f] < fStart) || (faces[f] >= fEnd)) continue;
902       PetscCall(DMPlexSetCone(gdm, ghostCell, &newFace));
903       PetscCall(DMPlexInsertSupport(gdm, newFace, 1, ghostCell));
904       ++ghostCell;
905     }
906     PetscCall(ISRestoreIndices(faceIS, &faces));
907     PetscCall(ISDestroy(&faceIS));
908   }
909   PetscCall(ISRestoreIndices(valueIS, &values));
910   PetscCall(ISDestroy(&valueIS));
911   PetscCall(DMPlexShiftCoordinates_Internal(dm, depthShift, gdm));
912   PetscCall(DMPlexShiftSF_Internal(dm, depthShift, gdm));
913   PetscCall(DMPlexShiftLabels_Internal(dm, depthShift, gdm));
914   PetscCall(DMPlexCreateVTKLabel_Internal(dm, PETSC_TRUE, gdm));
915   PetscCall(DMPlexShiftTree_Internal(dm, depthShift, gdm));
916   PetscCall(PetscFree(depthShift));
917   for (c = cEnd; c < cEnd + Ng; ++c) {
918     PetscCall(DMPlexSetCellType(gdm, c, DM_POLYTOPE_FV_GHOST));
919   }
920   /* Step 7: Periodicity */
921   PetscCall(DMGetPeriodicity(dm, &isper, &maxCell, &L, &bd));
922   PetscCall(DMSetPeriodicity(gdm, isper, maxCell,  L,  bd));
923   if (numGhostCells) *numGhostCells = Ng;
924   PetscFunctionReturn(0);
925 }
926 
927 /*@C
928   DMPlexConstructGhostCells - Construct ghost cells which connect to every boundary face
929 
930   Collective on dm
931 
932   Input Parameters:
933 + dm - The original DM
934 - labelName - The label specifying the boundary faces, or "Face Sets" if this is NULL
935 
936   Output Parameters:
937 + numGhostCells - The number of ghost cells added to the DM
938 - dmGhosted - The new DM
939 
940   Note: If no label exists of that name, one will be created marking all boundary faces
941 
942   Level: developer
943 
944 .seealso: `DMCreate()`
945 @*/
946 PetscErrorCode DMPlexConstructGhostCells(DM dm, const char labelName[], PetscInt *numGhostCells, DM *dmGhosted)
947 {
948   DM             gdm;
949   DMLabel        label;
950   const char    *name = labelName ? labelName : "Face Sets";
951   PetscInt       dim, Ng = 0;
952   PetscBool      useCone, useClosure;
953 
954   PetscFunctionBegin;
955   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
956   if (numGhostCells) PetscValidIntPointer(numGhostCells, 3);
957   PetscValidPointer(dmGhosted, 4);
958   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &gdm));
959   PetscCall(DMSetType(gdm, DMPLEX));
960   PetscCall(DMGetDimension(dm, &dim));
961   PetscCall(DMSetDimension(gdm, dim));
962   PetscCall(DMGetBasicAdjacency(dm, &useCone, &useClosure));
963   PetscCall(DMSetBasicAdjacency(gdm, useCone,  useClosure));
964   PetscCall(DMGetLabel(dm, name, &label));
965   if (!label) {
966     /* Get label for boundary faces */
967     PetscCall(DMCreateLabel(dm, name));
968     PetscCall(DMGetLabel(dm, name, &label));
969     PetscCall(DMPlexMarkBoundaryFaces(dm, 1, label));
970   }
971   PetscCall(DMPlexConstructGhostCells_Internal(dm, label, &Ng, gdm));
972   PetscCall(DMCopyDisc(dm, gdm));
973   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_TRUE, gdm));
974   gdm->setfromoptionscalled = dm->setfromoptionscalled;
975   if (numGhostCells) *numGhostCells = Ng;
976   *dmGhosted = gdm;
977   PetscFunctionReturn(0);
978 }
979 
980 static PetscErrorCode DivideCells_Private(DM dm, DMLabel label, DMPlexPointQueue queue)
981 {
982   PetscInt dim, d, shift = 100, *pStart, *pEnd;
983 
984   PetscFunctionBegin;
985   PetscCall(DMGetDimension(dm, &dim));
986   PetscCall(PetscMalloc2(dim, &pStart, dim, &pEnd));
987   for (d = 0; d < dim; ++d) {
988     PetscCall(DMPlexGetDepthStratum(dm, d, &pStart[d], &pEnd[d]));
989   }
990   while (!DMPlexPointQueueEmpty(queue)) {
991     PetscInt  cell    = -1;
992     PetscInt *closure = NULL;
993     PetscInt  closureSize, cl, cval;
994 
995     PetscCall(DMPlexPointQueueDequeue(queue, &cell));
996     PetscCall(DMLabelGetValue(label, cell, &cval));
997     PetscCall(DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
998     /* Mark points in the cell closure that touch the fault */
999     for (d = 0; d < dim; ++d) {
1000       for (cl = 0; cl < closureSize*2; cl += 2) {
1001         const PetscInt clp = closure[cl];
1002         PetscInt       clval;
1003 
1004         if ((clp < pStart[d]) || (clp >= pEnd[d])) continue;
1005         PetscCall(DMLabelGetValue(label, clp, &clval));
1006         if (clval == -1) {
1007           const PetscInt *cone;
1008           PetscInt        coneSize, c;
1009 
1010           /* If a cone point touches the fault, then this point touches the fault */
1011           PetscCall(DMPlexGetCone(dm, clp, &cone));
1012           PetscCall(DMPlexGetConeSize(dm, clp, &coneSize));
1013           for (c = 0; c < coneSize; ++c) {
1014             PetscInt cpval;
1015 
1016             PetscCall(DMLabelGetValue(label, cone[c], &cpval));
1017             if (cpval != -1) {
1018               PetscInt dep;
1019 
1020               PetscCall(DMPlexGetPointDepth(dm, clp, &dep));
1021               clval = cval < 0 ? -(shift+dep) : shift+dep;
1022               PetscCall(DMLabelSetValue(label, clp, clval));
1023               break;
1024             }
1025           }
1026         }
1027         /* Mark neighbor cells through marked faces (these cells must also touch the fault) */
1028         if (d == dim-1 && clval != -1) {
1029           const PetscInt *support;
1030           PetscInt        supportSize, s, nval;
1031 
1032           PetscCall(DMPlexGetSupport(dm, clp, &support));
1033           PetscCall(DMPlexGetSupportSize(dm, clp, &supportSize));
1034           for (s = 0; s < supportSize; ++s) {
1035             PetscCall(DMLabelGetValue(label, support[s], &nval));
1036             if (nval == -1) {
1037               PetscCall(DMLabelSetValue(label, support[s], clval < 0 ? clval-1 : clval+1));
1038               PetscCall(DMPlexPointQueueEnqueue(queue, support[s]));
1039             }
1040           }
1041         }
1042       }
1043     }
1044     PetscCall(DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
1045   }
1046   PetscCall(PetscFree2(pStart, pEnd));
1047   PetscFunctionReturn(0);
1048 }
1049 
1050 typedef struct {
1051   DM               dm;
1052   DMPlexPointQueue queue;
1053 } PointDivision;
1054 
1055 static PetscErrorCode divideCell(DMLabel label, PetscInt p, PetscInt val, void *ctx)
1056 {
1057   PointDivision  *div  = (PointDivision *) ctx;
1058   PetscInt        cval = val < 0 ? val-1 : val+1;
1059   const PetscInt *support;
1060   PetscInt        supportSize, s;
1061 
1062   PetscFunctionBegin;
1063   PetscCall(DMPlexGetSupport(div->dm, p, &support));
1064   PetscCall(DMPlexGetSupportSize(div->dm, p, &supportSize));
1065   for (s = 0; s < supportSize; ++s) {
1066     PetscCall(DMLabelSetValue(label, support[s], cval));
1067     PetscCall(DMPlexPointQueueEnqueue(div->queue, support[s]));
1068   }
1069   PetscFunctionReturn(0);
1070 }
1071 
1072 /* Mark cells by label propagation */
1073 static PetscErrorCode DMPlexLabelFaultHalo(DM dm, DMLabel faultLabel)
1074 {
1075   DMPlexPointQueue queue = NULL;
1076   PointDivision    div;
1077   PetscSF          pointSF;
1078   IS               pointIS;
1079   const PetscInt  *points;
1080   PetscBool        empty;
1081   PetscInt         dim, shift = 100, n, i;
1082 
1083   PetscFunctionBegin;
1084   PetscCall(DMGetDimension(dm, &dim));
1085   PetscCall(DMPlexPointQueueCreate(1024, &queue));
1086   div.dm    = dm;
1087   div.queue = queue;
1088   /* Enqueue cells on fault */
1089   PetscCall(DMLabelGetStratumIS(faultLabel, shift+dim, &pointIS));
1090   if (pointIS) {
1091     PetscCall(ISGetLocalSize(pointIS, &n));
1092     PetscCall(ISGetIndices(pointIS, &points));
1093     for (i = 0; i < n; ++i) {PetscCall(DMPlexPointQueueEnqueue(queue, points[i]));}
1094     PetscCall(ISRestoreIndices(pointIS, &points));
1095     PetscCall(ISDestroy(&pointIS));
1096   }
1097   PetscCall(DMLabelGetStratumIS(faultLabel, -(shift+dim), &pointIS));
1098   if (pointIS) {
1099     PetscCall(ISGetLocalSize(pointIS, &n));
1100     PetscCall(ISGetIndices(pointIS, &points));
1101     for (i = 0; i < n; ++i) {PetscCall(DMPlexPointQueueEnqueue(queue, points[i]));}
1102     PetscCall(ISRestoreIndices(pointIS, &points));
1103     PetscCall(ISDestroy(&pointIS));
1104   }
1105 
1106   PetscCall(DMGetPointSF(dm, &pointSF));
1107   PetscCall(DMLabelPropagateBegin(faultLabel, pointSF));
1108   /* While edge queue is not empty: */
1109   PetscCall(DMPlexPointQueueEmptyCollective((PetscObject) dm, queue, &empty));
1110   while (!empty) {
1111     PetscCall(DivideCells_Private(dm, faultLabel, queue));
1112     PetscCall(DMLabelPropagatePush(faultLabel, pointSF, divideCell, &div));
1113     PetscCall(DMPlexPointQueueEmptyCollective((PetscObject) dm, queue, &empty));
1114   }
1115   PetscCall(DMLabelPropagateEnd(faultLabel, pointSF));
1116   PetscCall(DMPlexPointQueueDestroy(&queue));
1117   PetscFunctionReturn(0);
1118 }
1119 
1120 /*
1121   We are adding three kinds of points here:
1122     Replicated:     Copies of points which exist in the mesh, such as vertices identified across a fault
1123     Non-replicated: Points which exist on the fault, but are not replicated
1124     Ghost:          These are shared fault faces which are not owned by this process. These do not produce hybrid cells and do not replicate
1125     Hybrid:         Entirely new points, such as cohesive cells
1126 
1127   When creating subsequent cohesive cells, we shift the old hybrid cells to the end of the numbering at
1128   each depth so that the new split/hybrid points can be inserted as a block.
1129 */
1130 static PetscErrorCode DMPlexConstructCohesiveCells_Internal(DM dm, DMLabel label, DMLabel splitLabel, DM sdm)
1131 {
1132   MPI_Comm         comm;
1133   IS               valueIS;
1134   PetscInt         numSP = 0;          /* The number of depths for which we have replicated points */
1135   const PetscInt  *values;             /* List of depths for which we have replicated points */
1136   IS              *splitIS;
1137   IS              *unsplitIS;
1138   IS               ghostIS;
1139   PetscInt        *numSplitPoints;     /* The number of replicated points at each depth */
1140   PetscInt        *numUnsplitPoints;   /* The number of non-replicated points at each depth which still give rise to hybrid points */
1141   PetscInt        *numHybridPoints;    /* The number of new hybrid points at each depth */
1142   PetscInt        *numHybridPointsOld; /* The number of existing hybrid points at each depth */
1143   PetscInt         numGhostPoints;     /* The number of unowned, shared fault faces */
1144   const PetscInt **splitPoints;        /* Replicated points for each depth */
1145   const PetscInt **unsplitPoints;      /* Non-replicated points for each depth */
1146   const PetscInt  *ghostPoints;        /* Ghost fault faces */
1147   PetscSection     coordSection;
1148   Vec              coordinates;
1149   PetscScalar     *coords;
1150   PetscInt        *depthMax;           /* The first hybrid point at each depth in the original mesh */
1151   PetscInt        *depthEnd;           /* The point limit at each depth in the original mesh */
1152   PetscInt        *depthShift;         /* Number of replicated+hybrid points at each depth */
1153   PetscInt        *pMaxNew;            /* The first replicated point at each depth in the new mesh, hybrids come after this */
1154   PetscInt        *coneNew, *coneONew, *supportNew;
1155   PetscInt         shift = 100, shift2 = 200, depth = 0, dep, dim, d, sp, maxConeSize, maxSupportSize, maxConeSizeNew, maxSupportSizeNew, numLabels, vStart, vEnd, pEnd, p, v;
1156 
1157   PetscFunctionBegin;
1158   PetscCall(PetscObjectGetComm((PetscObject)dm,&comm));
1159   PetscCall(DMGetDimension(dm, &dim));
1160   PetscCall(DMPlexGetDepth(dm, &depth));
1161   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
1162   /* We do not want this label automatically computed, instead we compute it here */
1163   PetscCall(DMCreateLabel(sdm, "celltype"));
1164   /* Count split points and add cohesive cells */
1165   PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize));
1166   PetscCall(PetscMalloc5(depth+1,&depthMax,depth+1,&depthEnd,2*(depth+1),&depthShift,depth+1,&pMaxNew,depth+1,&numHybridPointsOld));
1167   PetscCall(PetscMalloc7(depth+1,&splitIS,depth+1,&unsplitIS,depth+1,&numSplitPoints,depth+1,&numUnsplitPoints,depth+1,&numHybridPoints,depth+1,&splitPoints,depth+1,&unsplitPoints));
1168   for (d = 0; d <= depth; ++d) {
1169     PetscCall(DMPlexGetDepthStratum(dm, d, NULL, &pMaxNew[d]));
1170     PetscCall(DMPlexGetTensorPrismBounds_Internal(dm, d, &depthMax[d], NULL));
1171     depthEnd[d]           = pMaxNew[d];
1172     depthMax[d]           = depthMax[d] < 0 ? depthEnd[d] : depthMax[d];
1173     numSplitPoints[d]     = 0;
1174     numUnsplitPoints[d]   = 0;
1175     numHybridPoints[d]    = 0;
1176     numHybridPointsOld[d] = depthMax[d] < 0 ? 0 : depthEnd[d] - depthMax[d];
1177     splitPoints[d]        = NULL;
1178     unsplitPoints[d]      = NULL;
1179     splitIS[d]            = NULL;
1180     unsplitIS[d]          = NULL;
1181     /* we are shifting the existing hybrid points with the stratum behind them, so
1182      * the split comes at the end of the normal points, i.e., at depthMax[d] */
1183     depthShift[2*d]       = depthMax[d];
1184     depthShift[2*d+1]     = 0;
1185   }
1186   numGhostPoints = 0;
1187   ghostPoints    = NULL;
1188   if (label) {
1189     PetscCall(DMLabelGetValueIS(label, &valueIS));
1190     PetscCall(ISGetLocalSize(valueIS, &numSP));
1191     PetscCall(ISGetIndices(valueIS, &values));
1192   }
1193   for (sp = 0; sp < numSP; ++sp) {
1194     const PetscInt dep = values[sp];
1195 
1196     if ((dep < 0) || (dep > depth)) continue;
1197     PetscCall(DMLabelGetStratumIS(label, dep, &splitIS[dep]));
1198     if (splitIS[dep]) {
1199       PetscCall(ISGetLocalSize(splitIS[dep], &numSplitPoints[dep]));
1200       PetscCall(ISGetIndices(splitIS[dep], &splitPoints[dep]));
1201     }
1202     PetscCall(DMLabelGetStratumIS(label, shift2+dep, &unsplitIS[dep]));
1203     if (unsplitIS[dep]) {
1204       PetscCall(ISGetLocalSize(unsplitIS[dep], &numUnsplitPoints[dep]));
1205       PetscCall(ISGetIndices(unsplitIS[dep], &unsplitPoints[dep]));
1206     }
1207   }
1208   PetscCall(DMLabelGetStratumIS(label, shift2+dim-1, &ghostIS));
1209   if (ghostIS) {
1210     PetscCall(ISGetLocalSize(ghostIS, &numGhostPoints));
1211     PetscCall(ISGetIndices(ghostIS, &ghostPoints));
1212   }
1213   /* Calculate number of hybrid points */
1214   for (d = 1; d <= depth; ++d) numHybridPoints[d]     = numSplitPoints[d-1] + numUnsplitPoints[d-1]; /* There is a hybrid cell/face/edge for every split face/edge/vertex   */
1215   for (d = 0; d <= depth; ++d) depthShift[2*d+1]      = numSplitPoints[d] + numHybridPoints[d];
1216   PetscCall(DMPlexShiftPointSetUp_Internal(depth,depthShift));
1217   /* the end of the points in this stratum that come before the new points:
1218    * shifting pMaxNew[d] gets the new start of the next stratum, then count back the old hybrid points and the newly
1219    * added points */
1220   for (d = 0; d <= depth; ++d) pMaxNew[d]             = DMPlexShiftPoint_Internal(pMaxNew[d],depth,depthShift) - (numHybridPointsOld[d] + numSplitPoints[d] + numHybridPoints[d]);
1221   PetscCall(DMPlexShiftSizes_Internal(dm, depthShift, sdm));
1222   /* Step 3: Set cone/support sizes for new points */
1223   for (dep = 0; dep <= depth; ++dep) {
1224     for (p = 0; p < numSplitPoints[dep]; ++p) {
1225       const PetscInt  oldp   = splitPoints[dep][p];
1226       const PetscInt  newp   = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*oldp + depthOffset[dep]*/;
1227       const PetscInt  splitp = p    + pMaxNew[dep];
1228       const PetscInt *support;
1229       DMPolytopeType  ct;
1230       PetscInt        coneSize, supportSize, qf, qn, qp, e;
1231 
1232       PetscCall(DMPlexGetConeSize(dm, oldp, &coneSize));
1233       PetscCall(DMPlexSetConeSize(sdm, splitp, coneSize));
1234       PetscCall(DMPlexGetSupportSize(dm, oldp, &supportSize));
1235       PetscCall(DMPlexSetSupportSize(sdm, splitp, supportSize));
1236       PetscCall(DMPlexGetCellType(dm, oldp, &ct));
1237       PetscCall(DMPlexSetCellType(sdm, splitp, ct));
1238       if (dep == depth-1) {
1239         const PetscInt hybcell = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1240 
1241         /* Add cohesive cells, they are prisms */
1242         PetscCall(DMPlexSetConeSize(sdm, hybcell, 2 + coneSize));
1243         switch (coneSize) {
1244           case 2: PetscCall(DMPlexSetCellType(sdm, hybcell, DM_POLYTOPE_SEG_PRISM_TENSOR));break;
1245           case 3: PetscCall(DMPlexSetCellType(sdm, hybcell, DM_POLYTOPE_TRI_PRISM_TENSOR));break;
1246           case 4: PetscCall(DMPlexSetCellType(sdm, hybcell, DM_POLYTOPE_QUAD_PRISM_TENSOR));break;
1247         }
1248         /* Shared fault faces with only one support cell now have two with the cohesive cell */
1249         /*   TODO Check thaat oldp has rootdegree == 1 */
1250         if (supportSize == 1) {
1251           const PetscInt *support;
1252           PetscInt        val;
1253 
1254           PetscCall(DMPlexGetSupport(dm, oldp, &support));
1255           PetscCall(DMLabelGetValue(label, support[0], &val));
1256           if (val < 0) PetscCall(DMPlexSetSupportSize(sdm, splitp, 2));
1257           else         PetscCall(DMPlexSetSupportSize(sdm, newp,   2));
1258         }
1259       } else if (dep == 0) {
1260         const PetscInt hybedge = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1261 
1262         PetscCall(DMPlexGetSupport(dm, oldp, &support));
1263         for (e = 0, qn = 0, qp = 0, qf = 0; e < supportSize; ++e) {
1264           PetscInt val;
1265 
1266           PetscCall(DMLabelGetValue(label, support[e], &val));
1267           if (val == 1) ++qf;
1268           if ((val == 1) || (val ==  (shift + 1))) ++qn;
1269           if ((val == 1) || (val == -(shift + 1))) ++qp;
1270         }
1271         /* Split old vertex: Edges into original vertex and new cohesive edge */
1272         PetscCall(DMPlexSetSupportSize(sdm, newp, qn+1));
1273         /* Split new vertex: Edges into split vertex and new cohesive edge */
1274         PetscCall(DMPlexSetSupportSize(sdm, splitp, qp+1));
1275         /* Add hybrid edge */
1276         PetscCall(DMPlexSetConeSize(sdm, hybedge, 2));
1277         PetscCall(DMPlexSetSupportSize(sdm, hybedge, qf));
1278         PetscCall(DMPlexSetCellType(sdm, hybedge, DM_POLYTOPE_POINT_PRISM_TENSOR));
1279       } else if (dep == dim-2) {
1280         const PetscInt hybface = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1281 
1282         PetscCall(DMPlexGetSupport(dm, oldp, &support));
1283         for (e = 0, qn = 0, qp = 0, qf = 0; e < supportSize; ++e) {
1284           PetscInt val;
1285 
1286           PetscCall(DMLabelGetValue(label, support[e], &val));
1287           if (val == dim-1) ++qf;
1288           if ((val == dim-1) || (val ==  (shift + dim-1))) ++qn;
1289           if ((val == dim-1) || (val == -(shift + dim-1))) ++qp;
1290         }
1291         /* Split old edge: Faces into original edge and cohesive face (positive side?) */
1292         PetscCall(DMPlexSetSupportSize(sdm, newp, qn+1));
1293         /* Split new edge: Faces into split edge and cohesive face (negative side?) */
1294         PetscCall(DMPlexSetSupportSize(sdm, splitp, qp+1));
1295         /* Add hybrid face */
1296         PetscCall(DMPlexSetConeSize(sdm, hybface, 4));
1297         PetscCall(DMPlexSetSupportSize(sdm, hybface, qf));
1298         PetscCall(DMPlexSetCellType(sdm, hybface, DM_POLYTOPE_SEG_PRISM_TENSOR));
1299       }
1300     }
1301   }
1302   for (dep = 0; dep <= depth; ++dep) {
1303     for (p = 0; p < numUnsplitPoints[dep]; ++p) {
1304       const PetscInt  oldp   = unsplitPoints[dep][p];
1305       const PetscInt  newp   = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*oldp + depthOffset[dep]*/;
1306       const PetscInt *support;
1307       PetscInt        coneSize, supportSize, qf, e, s;
1308 
1309       PetscCall(DMPlexGetConeSize(dm, oldp, &coneSize));
1310       PetscCall(DMPlexGetSupportSize(dm, oldp, &supportSize));
1311       PetscCall(DMPlexGetSupport(dm, oldp, &support));
1312       if (dep == 0) {
1313         const PetscInt hybedge = p + pMaxNew[dep+1] + numSplitPoints[dep+1] + numSplitPoints[dep];
1314 
1315         /* Unsplit vertex: Edges into original vertex, split edges, and new cohesive edge twice */
1316         for (s = 0, qf = 0; s < supportSize; ++s, ++qf) {
1317           PetscCall(PetscFindInt(support[s], numSplitPoints[dep+1], splitPoints[dep+1], &e));
1318           if (e >= 0) ++qf;
1319         }
1320         PetscCall(DMPlexSetSupportSize(sdm, newp, qf+2));
1321         /* Add hybrid edge */
1322         PetscCall(DMPlexSetConeSize(sdm, hybedge, 2));
1323         for (e = 0, qf = 0; e < supportSize; ++e) {
1324           PetscInt val;
1325 
1326           PetscCall(DMLabelGetValue(label, support[e], &val));
1327           /* Split and unsplit edges produce hybrid faces */
1328           if (val == 1) ++qf;
1329           if (val == (shift2 + 1)) ++qf;
1330         }
1331         PetscCall(DMPlexSetSupportSize(sdm, hybedge, qf));
1332         PetscCall(DMPlexSetCellType(sdm, hybedge, DM_POLYTOPE_POINT_PRISM_TENSOR));
1333       } else if (dep == dim-2) {
1334         const PetscInt hybface = p + pMaxNew[dep+1] + numSplitPoints[dep+1] + numSplitPoints[dep];
1335         PetscInt       val;
1336 
1337         for (e = 0, qf = 0; e < supportSize; ++e) {
1338           PetscCall(DMLabelGetValue(label, support[e], &val));
1339           if (val == dim-1) qf += 2;
1340           else              ++qf;
1341         }
1342         /* Unsplit edge: Faces into original edge, split face, and cohesive face twice */
1343         PetscCall(DMPlexSetSupportSize(sdm, newp, qf+2));
1344         /* Add hybrid face */
1345         for (e = 0, qf = 0; e < supportSize; ++e) {
1346           PetscCall(DMLabelGetValue(label, support[e], &val));
1347           if (val == dim-1) ++qf;
1348         }
1349         PetscCall(DMPlexSetConeSize(sdm, hybface, 4));
1350         PetscCall(DMPlexSetSupportSize(sdm, hybface, qf));
1351         PetscCall(DMPlexSetCellType(sdm, hybface, DM_POLYTOPE_SEG_PRISM_TENSOR));
1352       }
1353     }
1354   }
1355   /* Step 4: Setup split DM */
1356   PetscCall(DMSetUp(sdm));
1357   PetscCall(DMPlexShiftPoints_Internal(dm, depthShift, sdm));
1358   PetscCall(DMPlexGetMaxSizes(sdm, &maxConeSizeNew, &maxSupportSizeNew));
1359   PetscCall(PetscMalloc3(PetscMax(maxConeSize, maxConeSizeNew)*3,&coneNew,PetscMax(maxConeSize, maxConeSizeNew)*3,&coneONew,PetscMax(maxSupportSize, maxSupportSizeNew),&supportNew));
1360   /* Step 6: Set cones and supports for new points */
1361   for (dep = 0; dep <= depth; ++dep) {
1362     for (p = 0; p < numSplitPoints[dep]; ++p) {
1363       const PetscInt  oldp   = splitPoints[dep][p];
1364       const PetscInt  newp   = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*oldp + depthOffset[dep]*/;
1365       const PetscInt  splitp = p    + pMaxNew[dep];
1366       const PetscInt *cone, *support, *ornt;
1367       DMPolytopeType  ct;
1368       PetscInt        coneSize, supportSize, q, qf, qn, qp, v, e, s;
1369 
1370       PetscCall(DMPlexGetCellType(dm, oldp, &ct));
1371       PetscCall(DMPlexGetConeSize(dm, oldp, &coneSize));
1372       PetscCall(DMPlexGetCone(dm, oldp, &cone));
1373       PetscCall(DMPlexGetConeOrientation(dm, oldp, &ornt));
1374       PetscCall(DMPlexGetSupportSize(dm, oldp, &supportSize));
1375       PetscCall(DMPlexGetSupport(dm, oldp, &support));
1376       if (dep == depth-1) {
1377         PetscBool       hasUnsplit = PETSC_FALSE;
1378         const PetscInt  hybcell    = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1379         const PetscInt *supportF;
1380 
1381         coneONew[0] = coneONew[1] = -1000;
1382         /* Split face:       copy in old face to new face to start */
1383         PetscCall(DMPlexGetSupport(sdm, newp,  &supportF));
1384         PetscCall(DMPlexSetSupport(sdm, splitp, supportF));
1385         /* Split old face:   old vertices/edges in cone so no change */
1386         /* Split new face:   new vertices/edges in cone */
1387         for (q = 0; q < coneSize; ++q) {
1388           PetscCall(PetscFindInt(cone[q], numSplitPoints[dep-1], splitPoints[dep-1], &v));
1389           if (v < 0) {
1390             PetscCall(PetscFindInt(cone[q], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &v));
1391             PetscCheck(v >= 0,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %" PetscInt_FMT " in split or unsplit points of depth %" PetscInt_FMT, cone[q], dep-1);
1392             coneNew[2+q] = DMPlexShiftPoint_Internal(cone[q], depth, depthShift) /*cone[q] + depthOffset[dep-1]*/;
1393             hasUnsplit   = PETSC_TRUE;
1394           } else {
1395             coneNew[2+q] = v + pMaxNew[dep-1];
1396             if (dep > 1) {
1397               const PetscInt *econe;
1398               PetscInt        econeSize, r, vs, vu;
1399 
1400               PetscCall(DMPlexGetConeSize(dm, cone[q], &econeSize));
1401               PetscCall(DMPlexGetCone(dm, cone[q], &econe));
1402               for (r = 0; r < econeSize; ++r) {
1403                 PetscCall(PetscFindInt(econe[r], numSplitPoints[dep-2],   splitPoints[dep-2],   &vs));
1404                 PetscCall(PetscFindInt(econe[r], numUnsplitPoints[dep-2], unsplitPoints[dep-2], &vu));
1405                 if (vs >= 0) continue;
1406                 PetscCheck(vu >= 0,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %" PetscInt_FMT " in split or unsplit points of depth %" PetscInt_FMT, econe[r], dep-2);
1407                 hasUnsplit   = PETSC_TRUE;
1408               }
1409             }
1410           }
1411         }
1412         PetscCall(DMPlexSetCone(sdm, splitp, &coneNew[2]));
1413         PetscCall(DMPlexSetConeOrientation(sdm, splitp, ornt));
1414         /* Face support */
1415         PetscInt vals[2];
1416 
1417         PetscCall(DMLabelGetValue(label, support[0], &vals[0]));
1418         if (supportSize > 1) PetscCall(DMLabelGetValue(label, support[1], &vals[1]));
1419         else                 vals[1] = -vals[0];
1420         PetscCheck(vals[0]*vals[1] < 0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid support labels %" PetscInt_FMT " %" PetscInt_FMT, vals[0], vals[1]);
1421 
1422         for (s = 0; s < 2; ++s) {
1423           if (s >= supportSize) {
1424             if (vals[s] < 0) {
1425               /* Ghost old face:   Replace negative side cell with cohesive cell */
1426               PetscCall(DMPlexInsertSupport(sdm, newp,   s, hybcell));
1427             } else {
1428               /* Ghost new face:   Replace positive side cell with cohesive cell */
1429               PetscCall(DMPlexInsertSupport(sdm, splitp, s, hybcell));
1430             }
1431           } else {
1432             if (vals[s] < 0) {
1433               /* Split old face:   Replace negative side cell with cohesive cell */
1434               PetscCall(DMPlexInsertSupport(sdm, newp,   s, hybcell));
1435             } else {
1436               /* Split new face:   Replace positive side cell with cohesive cell */
1437               PetscCall(DMPlexInsertSupport(sdm, splitp, s, hybcell));
1438             }
1439           }
1440         }
1441         /* Get orientation for cohesive face using the positive side cell */
1442         {
1443           const PetscInt *ncone, *nconeO;
1444           PetscInt        nconeSize, nc, ocell;
1445           PetscBool       flip = PETSC_FALSE;
1446 
1447           if (supportSize > 1) {ocell = vals[0] < 0 ? support[1] : support[0];}
1448           else                 {ocell = support[0]; flip = vals[0] < 0 ? PETSC_TRUE : PETSC_FALSE;}
1449           PetscCall(DMPlexGetConeSize(dm, ocell, &nconeSize));
1450           PetscCall(DMPlexGetCone(dm, ocell, &ncone));
1451           PetscCall(DMPlexGetConeOrientation(dm, ocell, &nconeO));
1452           for (nc = 0; nc < nconeSize; ++nc) {
1453             if (ncone[nc] == oldp) {
1454               coneONew[0] = flip ? -(nconeO[nc]+1) : nconeO[nc];
1455               break;
1456             }
1457           }
1458           PetscCheck(nc < nconeSize, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate face %" PetscInt_FMT " in neighboring cell %" PetscInt_FMT, oldp, ocell);
1459         }
1460         /* Cohesive cell:    Old and new split face, then new cohesive faces */
1461         {
1462           const PetscInt No = DMPolytopeTypeGetNumArrangments(ct)/2;
1463           PetscCheck((coneONew[0] >= -No) && (coneONew[0] < No), PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid %s orientation %" PetscInt_FMT, DMPolytopeTypes[ct], coneONew[0]);
1464         }
1465         const PetscInt *arr = DMPolytopeTypeGetArrangment(ct, coneONew[0]);
1466 
1467         coneNew[0]  = newp;   /* Extracted negative side orientation above */
1468         coneNew[1]  = splitp;
1469         coneONew[1] = coneONew[0];
1470         for (q = 0; q < coneSize; ++q) {
1471           /* Hybrid faces must follow order from oriented end face */
1472           const PetscInt qa = arr[q*2+0];
1473           const PetscInt qo = arr[q*2+1];
1474           DMPolytopeType ft = dep == 2 ? DM_POLYTOPE_SEGMENT : DM_POLYTOPE_POINT;
1475 
1476           PetscCall(PetscFindInt(cone[qa], numSplitPoints[dep-1], splitPoints[dep-1], &v));
1477           if (v < 0) {
1478             PetscCall(PetscFindInt(cone[qa], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &v));
1479             coneNew[2+q]  = v + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep-1];
1480           } else {
1481             coneNew[2+q]  = v + pMaxNew[dep] + numSplitPoints[dep];
1482           }
1483           coneONew[2+q] = DMPolytopeTypeComposeOrientation(ft, qo, ornt[qa]);
1484         }
1485         PetscCall(DMPlexSetCone(sdm, hybcell, coneNew));
1486         PetscCall(DMPlexSetConeOrientation(sdm, hybcell, coneONew));
1487         /* Label the hybrid cells on the boundary of the split */
1488         if (hasUnsplit) PetscCall(DMLabelSetValue(label, -hybcell, dim));
1489       } else if (dep == 0) {
1490         const PetscInt hybedge = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1491 
1492         /* Split old vertex: Edges in old split faces and new cohesive edge */
1493         for (e = 0, qn = 0; e < supportSize; ++e) {
1494           PetscInt val;
1495 
1496           PetscCall(DMLabelGetValue(label, support[e], &val));
1497           if ((val == 1) || (val == (shift + 1))) {
1498             supportNew[qn++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1499           }
1500         }
1501         supportNew[qn] = hybedge;
1502         PetscCall(DMPlexSetSupport(sdm, newp, supportNew));
1503         /* Split new vertex: Edges in new split faces and new cohesive edge */
1504         for (e = 0, qp = 0; e < supportSize; ++e) {
1505           PetscInt val, edge;
1506 
1507           PetscCall(DMLabelGetValue(label, support[e], &val));
1508           if (val == 1) {
1509             PetscCall(PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &edge));
1510             PetscCheck(edge >= 0,comm, PETSC_ERR_ARG_WRONG, "Edge %" PetscInt_FMT " is not a split edge", support[e]);
1511             supportNew[qp++] = edge + pMaxNew[dep+1];
1512           } else if (val == -(shift + 1)) {
1513             supportNew[qp++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1514           }
1515         }
1516         supportNew[qp] = hybedge;
1517         PetscCall(DMPlexSetSupport(sdm, splitp, supportNew));
1518         /* Hybrid edge:    Old and new split vertex */
1519         coneNew[0] = newp;
1520         coneNew[1] = splitp;
1521         PetscCall(DMPlexSetCone(sdm, hybedge, coneNew));
1522         for (e = 0, qf = 0; e < supportSize; ++e) {
1523           PetscInt val, edge;
1524 
1525           PetscCall(DMLabelGetValue(label, support[e], &val));
1526           if (val == 1) {
1527             PetscCall(PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &edge));
1528             PetscCheck(edge >= 0,comm, PETSC_ERR_ARG_WRONG, "Edge %" PetscInt_FMT " is not a split edge", support[e]);
1529             supportNew[qf++] = edge + pMaxNew[dep+2] + numSplitPoints[dep+2];
1530           }
1531         }
1532         PetscCall(DMPlexSetSupport(sdm, hybedge, supportNew));
1533       } else if (dep == dim-2) {
1534         const PetscInt hybface = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1535 
1536         /* Split old edge:   old vertices in cone so no change */
1537         /* Split new edge:   new vertices in cone */
1538         for (q = 0; q < coneSize; ++q) {
1539           PetscCall(PetscFindInt(cone[q], numSplitPoints[dep-1], splitPoints[dep-1], &v));
1540           if (v < 0) {
1541             PetscCall(PetscFindInt(cone[q], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &v));
1542             PetscCheck(v >= 0,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %" PetscInt_FMT " in split or unsplit points of depth %" PetscInt_FMT, cone[q], dep-1);
1543             coneNew[q] = DMPlexShiftPoint_Internal(cone[q], depth, depthShift) /*cone[q] + depthOffset[dep-1]*/;
1544           } else {
1545             coneNew[q] = v + pMaxNew[dep-1];
1546           }
1547         }
1548         PetscCall(DMPlexSetCone(sdm, splitp, coneNew));
1549         /* Split old edge: Faces in positive side cells and old split faces */
1550         for (e = 0, q = 0; e < supportSize; ++e) {
1551           PetscInt val;
1552 
1553           PetscCall(DMLabelGetValue(label, support[e], &val));
1554           if (val == dim-1) {
1555             supportNew[q++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1556           } else if (val == (shift + dim-1)) {
1557             supportNew[q++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1558           }
1559         }
1560         supportNew[q++] = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1561         PetscCall(DMPlexSetSupport(sdm, newp, supportNew));
1562         /* Split new edge: Faces in negative side cells and new split faces */
1563         for (e = 0, q = 0; e < supportSize; ++e) {
1564           PetscInt val, face;
1565 
1566           PetscCall(DMLabelGetValue(label, support[e], &val));
1567           if (val == dim-1) {
1568             PetscCall(PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &face));
1569             PetscCheck(face >= 0,comm, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " is not a split face", support[e]);
1570             supportNew[q++] = face + pMaxNew[dep+1];
1571           } else if (val == -(shift + dim-1)) {
1572             supportNew[q++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1573           }
1574         }
1575         supportNew[q++] = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1576         PetscCall(DMPlexSetSupport(sdm, splitp, supportNew));
1577         /* Hybrid face */
1578         coneNew[0] = newp;
1579         coneNew[1] = splitp;
1580         for (v = 0; v < coneSize; ++v) {
1581           PetscInt vertex;
1582           PetscCall(PetscFindInt(cone[v], numSplitPoints[dep-1], splitPoints[dep-1], &vertex));
1583           if (vertex < 0) {
1584             PetscCall(PetscFindInt(cone[v], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &vertex));
1585             PetscCheck(vertex >= 0,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %" PetscInt_FMT " in split or unsplit points of depth %" PetscInt_FMT, cone[v], dep-1);
1586             coneNew[2+v] = vertex + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep-1];
1587           } else {
1588             coneNew[2+v] = vertex + pMaxNew[dep] + numSplitPoints[dep];
1589           }
1590         }
1591         PetscCall(DMPlexSetCone(sdm, hybface, coneNew));
1592         for (e = 0, qf = 0; e < supportSize; ++e) {
1593           PetscInt val, face;
1594 
1595           PetscCall(DMLabelGetValue(label, support[e], &val));
1596           if (val == dim-1) {
1597             PetscCall(PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &face));
1598             PetscCheck(face >= 0,comm, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " is not a split face", support[e]);
1599             supportNew[qf++] = face + pMaxNew[dep+2] + numSplitPoints[dep+2];
1600           }
1601         }
1602         PetscCall(DMPlexSetSupport(sdm, hybface, supportNew));
1603       }
1604     }
1605   }
1606   for (dep = 0; dep <= depth; ++dep) {
1607     for (p = 0; p < numUnsplitPoints[dep]; ++p) {
1608       const PetscInt  oldp   = unsplitPoints[dep][p];
1609       const PetscInt  newp   = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*oldp + depthOffset[dep]*/;
1610       const PetscInt *cone, *support;
1611       PetscInt        coneSize, supportSize, supportSizeNew, q, qf, e, f, s;
1612 
1613       PetscCall(DMPlexGetConeSize(dm, oldp, &coneSize));
1614       PetscCall(DMPlexGetCone(dm, oldp, &cone));
1615       PetscCall(DMPlexGetSupportSize(dm, oldp, &supportSize));
1616       PetscCall(DMPlexGetSupport(dm, oldp, &support));
1617       if (dep == 0) {
1618         const PetscInt hybedge = p + pMaxNew[dep+1] + numSplitPoints[dep+1] + numSplitPoints[dep];
1619 
1620         /* Unsplit vertex */
1621         PetscCall(DMPlexGetSupportSize(sdm, newp, &supportSizeNew));
1622         for (s = 0, q = 0; s < supportSize; ++s) {
1623           supportNew[q++] = DMPlexShiftPoint_Internal(support[s], depth, depthShift) /*support[s] + depthOffset[dep+1]*/;
1624           PetscCall(PetscFindInt(support[s], numSplitPoints[dep+1], splitPoints[dep+1], &e));
1625           if (e >= 0) {
1626             supportNew[q++] = e + pMaxNew[dep+1];
1627           }
1628         }
1629         supportNew[q++] = hybedge;
1630         supportNew[q++] = hybedge;
1631         PetscCheck(q == supportSizeNew,comm, PETSC_ERR_ARG_WRONG, "Support size %" PetscInt_FMT " != %" PetscInt_FMT " for vertex %" PetscInt_FMT, q, supportSizeNew, newp);
1632         PetscCall(DMPlexSetSupport(sdm, newp, supportNew));
1633         /* Hybrid edge */
1634         coneNew[0] = newp;
1635         coneNew[1] = newp;
1636         PetscCall(DMPlexSetCone(sdm, hybedge, coneNew));
1637         for (e = 0, qf = 0; e < supportSize; ++e) {
1638           PetscInt val, edge;
1639 
1640           PetscCall(DMLabelGetValue(label, support[e], &val));
1641           if (val == 1) {
1642             PetscCall(PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &edge));
1643             PetscCheck(edge >= 0,comm, PETSC_ERR_ARG_WRONG, "Edge %" PetscInt_FMT " is not a split edge", support[e]);
1644             supportNew[qf++] = edge + pMaxNew[dep+2] + numSplitPoints[dep+2];
1645           } else if  (val ==  (shift2 + 1)) {
1646             PetscCall(PetscFindInt(support[e], numUnsplitPoints[dep+1], unsplitPoints[dep+1], &edge));
1647             PetscCheck(edge >= 0,comm, PETSC_ERR_ARG_WRONG, "Edge %" PetscInt_FMT " is not a unsplit edge", support[e]);
1648             supportNew[qf++] = edge + pMaxNew[dep+2] + numSplitPoints[dep+2] + numSplitPoints[dep+1];
1649           }
1650         }
1651         PetscCall(DMPlexSetSupport(sdm, hybedge, supportNew));
1652       } else if (dep == dim-2) {
1653         const PetscInt hybface = p + pMaxNew[dep+1] + numSplitPoints[dep+1] + numSplitPoints[dep];
1654 
1655         /* Unsplit edge: Faces into original edge, split face, and hybrid face twice */
1656         for (f = 0, qf = 0; f < supportSize; ++f) {
1657           PetscInt val, face;
1658 
1659           PetscCall(DMLabelGetValue(label, support[f], &val));
1660           if (val == dim-1) {
1661             PetscCall(PetscFindInt(support[f], numSplitPoints[dep+1], splitPoints[dep+1], &face));
1662             PetscCheck(face >= 0,comm, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " is not a split face", support[f]);
1663             supportNew[qf++] = DMPlexShiftPoint_Internal(support[f], depth, depthShift) /*support[f] + depthOffset[dep+1]*/;
1664             supportNew[qf++] = face + pMaxNew[dep+1];
1665           } else {
1666             supportNew[qf++] = DMPlexShiftPoint_Internal(support[f], depth, depthShift) /*support[f] + depthOffset[dep+1]*/;
1667           }
1668         }
1669         supportNew[qf++] = hybface;
1670         supportNew[qf++] = hybface;
1671         PetscCall(DMPlexGetSupportSize(sdm, newp, &supportSizeNew));
1672         PetscCheck(qf == supportSizeNew,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Support size for unsplit edge %" PetscInt_FMT " is %" PetscInt_FMT " != %" PetscInt_FMT, newp, qf, supportSizeNew);
1673         PetscCall(DMPlexSetSupport(sdm, newp, supportNew));
1674         /* Add hybrid face */
1675         coneNew[0] = newp;
1676         coneNew[1] = newp;
1677         PetscCall(PetscFindInt(cone[0], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &v));
1678         PetscCheck(v >= 0,comm, PETSC_ERR_ARG_WRONG, "Vertex %" PetscInt_FMT " is not an unsplit vertex", cone[0]);
1679         coneNew[2] = v + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep-1];
1680         PetscCall(PetscFindInt(cone[1], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &v));
1681         PetscCheck(v >= 0,comm, PETSC_ERR_ARG_WRONG, "Vertex %" PetscInt_FMT " is not an unsplit vertex", cone[1]);
1682         coneNew[3] = v + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep-1];
1683         PetscCall(DMPlexSetCone(sdm, hybface, coneNew));
1684         for (f = 0, qf = 0; f < supportSize; ++f) {
1685           PetscInt val, face;
1686 
1687           PetscCall(DMLabelGetValue(label, support[f], &val));
1688           if (val == dim-1) {
1689             PetscCall(PetscFindInt(support[f], numSplitPoints[dep+1], splitPoints[dep+1], &face));
1690             supportNew[qf++] = face + pMaxNew[dep+2] + numSplitPoints[dep+2];
1691           }
1692         }
1693         PetscCall(DMPlexGetSupportSize(sdm, hybface, &supportSizeNew));
1694         PetscCheck(qf == supportSizeNew,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Support size for hybrid face %" PetscInt_FMT " is %" PetscInt_FMT " != %" PetscInt_FMT, hybface, qf, supportSizeNew);
1695         PetscCall(DMPlexSetSupport(sdm, hybface, supportNew));
1696       }
1697     }
1698   }
1699   /* Step 6b: Replace split points in negative side cones */
1700   for (sp = 0; sp < numSP; ++sp) {
1701     PetscInt        dep = values[sp];
1702     IS              pIS;
1703     PetscInt        numPoints;
1704     const PetscInt *points;
1705 
1706     if (dep >= 0) continue;
1707     PetscCall(DMLabelGetStratumIS(label, dep, &pIS));
1708     if (!pIS) continue;
1709     dep  = -dep - shift;
1710     PetscCall(ISGetLocalSize(pIS, &numPoints));
1711     PetscCall(ISGetIndices(pIS, &points));
1712     for (p = 0; p < numPoints; ++p) {
1713       const PetscInt  oldp = points[p];
1714       const PetscInt  newp = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*depthOffset[dep] + oldp*/;
1715       const PetscInt *cone;
1716       PetscInt        coneSize, c;
1717       /* PetscBool       replaced = PETSC_FALSE; */
1718 
1719       /* Negative edge: replace split vertex */
1720       /* Negative cell: replace split face */
1721       PetscCall(DMPlexGetConeSize(sdm, newp, &coneSize));
1722       PetscCall(DMPlexGetCone(sdm, newp, &cone));
1723       for (c = 0; c < coneSize; ++c) {
1724         const PetscInt coldp = DMPlexShiftPointInverse_Internal(cone[c],depth,depthShift);
1725         PetscInt       csplitp, cp, val;
1726 
1727         PetscCall(DMLabelGetValue(label, coldp, &val));
1728         if (val == dep-1) {
1729           PetscCall(PetscFindInt(coldp, numSplitPoints[dep-1], splitPoints[dep-1], &cp));
1730           PetscCheck(cp >= 0,comm, PETSC_ERR_ARG_WRONG, "Point %" PetscInt_FMT " is not a split point of dimension %" PetscInt_FMT, oldp, dep-1);
1731           csplitp  = pMaxNew[dep-1] + cp;
1732           PetscCall(DMPlexInsertCone(sdm, newp, c, csplitp));
1733           /* replaced = PETSC_TRUE; */
1734         }
1735       }
1736       /* Cells with only a vertex or edge on the submesh have no replacement */
1737       /* PetscCheck(replaced,comm, PETSC_ERR_ARG_WRONG, "The cone of point %d does not contain split points", oldp); */
1738     }
1739     PetscCall(ISRestoreIndices(pIS, &points));
1740     PetscCall(ISDestroy(&pIS));
1741   }
1742   /* Step 7: Coordinates */
1743   PetscCall(DMPlexShiftCoordinates_Internal(dm, depthShift, sdm));
1744   PetscCall(DMGetCoordinateSection(sdm, &coordSection));
1745   PetscCall(DMGetCoordinatesLocal(sdm, &coordinates));
1746   PetscCall(VecGetArray(coordinates, &coords));
1747   for (v = 0; v < (numSplitPoints ? numSplitPoints[0] : 0); ++v) {
1748     const PetscInt newp   = DMPlexShiftPoint_Internal(splitPoints[0][v], depth, depthShift) /*depthOffset[0] + splitPoints[0][v]*/;
1749     const PetscInt splitp = pMaxNew[0] + v;
1750     PetscInt       dof, off, soff, d;
1751 
1752     PetscCall(PetscSectionGetDof(coordSection, newp, &dof));
1753     PetscCall(PetscSectionGetOffset(coordSection, newp, &off));
1754     PetscCall(PetscSectionGetOffset(coordSection, splitp, &soff));
1755     for (d = 0; d < dof; ++d) coords[soff+d] = coords[off+d];
1756   }
1757   PetscCall(VecRestoreArray(coordinates, &coords));
1758   /* Step 8: SF, if I can figure this out we can split the mesh in parallel */
1759   PetscCall(DMPlexShiftSF_Internal(dm, depthShift, sdm));
1760   /*   TODO We need to associate the ghost points with the correct replica */
1761   /* Step 9: Labels */
1762   PetscCall(DMPlexShiftLabels_Internal(dm, depthShift, sdm));
1763   PetscCall(DMPlexCreateVTKLabel_Internal(dm, PETSC_FALSE, sdm));
1764   PetscCall(DMGetNumLabels(sdm, &numLabels));
1765   for (dep = 0; dep <= depth; ++dep) {
1766     for (p = 0; p < numSplitPoints[dep]; ++p) {
1767       const PetscInt newp   = DMPlexShiftPoint_Internal(splitPoints[dep][p], depth, depthShift) /*depthOffset[dep] + splitPoints[dep][p]*/;
1768       const PetscInt splitp = pMaxNew[dep] + p;
1769       PetscInt       l;
1770 
1771       if (splitLabel) {
1772         const PetscInt val = 100 + dep;
1773 
1774         PetscCall(DMLabelSetValue(splitLabel, newp,    val));
1775         PetscCall(DMLabelSetValue(splitLabel, splitp, -val));
1776       }
1777       for (l = 0; l < numLabels; ++l) {
1778         DMLabel     mlabel;
1779         const char *lname;
1780         PetscInt    val;
1781         PetscBool   isDepth;
1782 
1783         PetscCall(DMGetLabelName(sdm, l, &lname));
1784         PetscCall(PetscStrcmp(lname, "depth", &isDepth));
1785         if (isDepth) continue;
1786         PetscCall(DMGetLabel(sdm, lname, &mlabel));
1787         PetscCall(DMLabelGetValue(mlabel, newp, &val));
1788         if (val >= 0) {
1789           PetscCall(DMLabelSetValue(mlabel, splitp, val));
1790         }
1791       }
1792     }
1793   }
1794   for (sp = 0; sp < numSP; ++sp) {
1795     const PetscInt dep = values[sp];
1796 
1797     if ((dep < 0) || (dep > depth)) continue;
1798     if (splitIS[dep]) PetscCall(ISRestoreIndices(splitIS[dep], &splitPoints[dep]));
1799     PetscCall(ISDestroy(&splitIS[dep]));
1800     if (unsplitIS[dep]) PetscCall(ISRestoreIndices(unsplitIS[dep], &unsplitPoints[dep]));
1801     PetscCall(ISDestroy(&unsplitIS[dep]));
1802   }
1803   if (ghostIS) PetscCall(ISRestoreIndices(ghostIS, &ghostPoints));
1804   PetscCall(ISDestroy(&ghostIS));
1805   if (label) {
1806     PetscCall(ISRestoreIndices(valueIS, &values));
1807     PetscCall(ISDestroy(&valueIS));
1808   }
1809   for (d = 0; d <= depth; ++d) {
1810     PetscCall(DMPlexGetDepthStratum(sdm, d, NULL, &pEnd));
1811     pMaxNew[d] = pEnd - numHybridPoints[d] - numHybridPointsOld[d];
1812   }
1813   PetscCall(PetscFree3(coneNew, coneONew, supportNew));
1814   PetscCall(PetscFree5(depthMax, depthEnd, depthShift, pMaxNew, numHybridPointsOld));
1815   PetscCall(PetscFree7(splitIS, unsplitIS, numSplitPoints, numUnsplitPoints, numHybridPoints, splitPoints, unsplitPoints));
1816   PetscFunctionReturn(0);
1817 }
1818 
1819 /*@C
1820   DMPlexConstructCohesiveCells - Construct cohesive cells which split the face along an internal interface
1821 
1822   Collective on dm
1823 
1824   Input Parameters:
1825 + dm - The original DM
1826 - label - The label specifying the boundary faces (this could be auto-generated)
1827 
1828   Output Parameters:
1829 + splitLabel - The label containing the split points, or NULL if no output is desired
1830 - dmSplit - The new DM
1831 
1832   Level: developer
1833 
1834 .seealso: `DMCreate()`, `DMPlexLabelCohesiveComplete()`
1835 @*/
1836 PetscErrorCode DMPlexConstructCohesiveCells(DM dm, DMLabel label, DMLabel splitLabel, DM *dmSplit)
1837 {
1838   DM             sdm;
1839   PetscInt       dim;
1840 
1841   PetscFunctionBegin;
1842   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1843   PetscValidPointer(dmSplit, 4);
1844   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &sdm));
1845   PetscCall(DMSetType(sdm, DMPLEX));
1846   PetscCall(DMGetDimension(dm, &dim));
1847   PetscCall(DMSetDimension(sdm, dim));
1848   switch (dim) {
1849   case 2:
1850   case 3:
1851     PetscCall(DMPlexConstructCohesiveCells_Internal(dm, label, splitLabel, sdm));
1852     break;
1853   default:
1854     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cannot construct cohesive cells for dimension %" PetscInt_FMT, dim);
1855   }
1856   *dmSplit = sdm;
1857   PetscFunctionReturn(0);
1858 }
1859 
1860 /* Returns the side of the surface for a given cell with a face on the surface */
1861 static PetscErrorCode GetSurfaceSide_Static(DM dm, DM subdm, PetscInt numSubpoints, const PetscInt *subpoints, PetscInt cell, PetscInt face, PetscBool *pos)
1862 {
1863   const PetscInt *cone, *ornt;
1864   PetscInt        dim, coneSize, c;
1865 
1866   PetscFunctionBegin;
1867   *pos = PETSC_TRUE;
1868   PetscCall(DMGetDimension(dm, &dim));
1869   PetscCall(DMPlexGetConeSize(dm, cell, &coneSize));
1870   PetscCall(DMPlexGetCone(dm, cell, &cone));
1871   PetscCall(DMPlexGetConeOrientation(dm, cell, &ornt));
1872   for (c = 0; c < coneSize; ++c) {
1873     if (cone[c] == face) {
1874       PetscInt o = ornt[c];
1875 
1876       if (subdm) {
1877         const PetscInt *subcone, *subornt;
1878         PetscInt        subpoint, subface, subconeSize, sc;
1879 
1880         PetscCall(PetscFindInt(cell, numSubpoints, subpoints, &subpoint));
1881         PetscCall(PetscFindInt(face, numSubpoints, subpoints, &subface));
1882         PetscCall(DMPlexGetConeSize(subdm, subpoint, &subconeSize));
1883         PetscCall(DMPlexGetCone(subdm, subpoint, &subcone));
1884         PetscCall(DMPlexGetConeOrientation(subdm, subpoint, &subornt));
1885         for (sc = 0; sc < subconeSize; ++sc) {
1886           if (subcone[sc] == subface) {
1887             o = subornt[0];
1888             break;
1889           }
1890         }
1891         PetscCheck(sc < subconeSize,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not find subpoint %" PetscInt_FMT " (%" PetscInt_FMT ") in cone for subpoint %" PetscInt_FMT " (%" PetscInt_FMT ")", subface, face, subpoint, cell);
1892       }
1893       if (o >= 0) *pos = PETSC_TRUE;
1894       else        *pos = PETSC_FALSE;
1895       break;
1896     }
1897   }
1898   PetscCheck(c != coneSize,PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Cell %" PetscInt_FMT " in split face %" PetscInt_FMT " support does not have it in the cone", cell, face);
1899   PetscFunctionReturn(0);
1900 }
1901 
1902 static PetscErrorCode CheckFaultEdge_Private(DM dm, DMLabel label)
1903 {
1904   IS              facePosIS, faceNegIS, dimIS;
1905   const PetscInt *points;
1906   PetscInt        dim, numPoints, p, shift = 100, shift2 = 200;
1907 
1908   PetscFunctionBegin;
1909   PetscCall(DMGetDimension(dm, &dim));
1910   /* If any faces touching the fault divide cells on either side, split them */
1911   PetscCall(DMLabelGetStratumIS(label,   shift+dim-1,  &facePosIS));
1912   PetscCall(DMLabelGetStratumIS(label, -(shift+dim-1), &faceNegIS));
1913   if (!facePosIS || !faceNegIS) {
1914     PetscCall(ISDestroy(&facePosIS));
1915     PetscCall(ISDestroy(&faceNegIS));
1916     PetscFunctionReturn(0);
1917   }
1918   PetscCall(ISExpand(facePosIS, faceNegIS, &dimIS));
1919   PetscCall(ISDestroy(&facePosIS));
1920   PetscCall(ISDestroy(&faceNegIS));
1921   PetscCall(ISGetLocalSize(dimIS, &numPoints));
1922   PetscCall(ISGetIndices(dimIS, &points));
1923   for (p = 0; p < numPoints; ++p) {
1924     const PetscInt  point = points[p];
1925     const PetscInt *support;
1926     PetscInt        supportSize, valA, valB;
1927 
1928     PetscCall(DMPlexGetSupportSize(dm, point, &supportSize));
1929     if (supportSize != 2) continue;
1930     PetscCall(DMPlexGetSupport(dm, point, &support));
1931     PetscCall(DMLabelGetValue(label, support[0], &valA));
1932     PetscCall(DMLabelGetValue(label, support[1], &valB));
1933     if ((valA == -1) || (valB == -1)) continue;
1934     if (valA*valB > 0) continue;
1935     /* Check that this face is not incident on only unsplit faces, meaning has at least one split face */
1936     {
1937       PetscInt *closure = NULL;
1938       PetscBool split   = PETSC_FALSE;
1939       PetscInt  closureSize, cl;
1940 
1941       PetscCall(DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
1942       for (cl = 0; cl < closureSize*2; cl += 2) {
1943         PetscCall(DMLabelGetValue(label, closure[cl], &valA));
1944         if ((valA >= 0) && (valA <= dim)) {split = PETSC_TRUE; break;}
1945       }
1946       PetscCall(DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
1947       if (!split) continue;
1948     }
1949     /* Split the face */
1950     PetscCall(DMLabelGetValue(label, point, &valA));
1951     PetscCall(DMLabelClearValue(label, point, valA));
1952     PetscCall(DMLabelSetValue(label, point, dim-1));
1953     /* Label its closure:
1954       unmarked: label as unsplit
1955       incident: relabel as split
1956       split:    do nothing
1957     */
1958     {
1959       PetscInt *closure = NULL;
1960       PetscInt  closureSize, cl, dep;
1961 
1962       PetscCall(DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
1963       for (cl = 0; cl < closureSize*2; cl += 2) {
1964         PetscCall(DMLabelGetValue(label, closure[cl], &valA));
1965         if (valA == -1) { /* Mark as unsplit */
1966           PetscCall(DMPlexGetPointDepth(dm, closure[cl], &dep));
1967           PetscCall(DMLabelSetValue(label, closure[cl], shift2+dep));
1968         } else if (((valA >= shift) && (valA < shift2)) || ((valA <= -shift) && (valA > -shift2))) {
1969           PetscCall(DMPlexGetPointDepth(dm, closure[cl], &dep));
1970           PetscCall(DMLabelClearValue(label, closure[cl], valA));
1971           PetscCall(DMLabelSetValue(label, closure[cl], dep));
1972         }
1973       }
1974       PetscCall(DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
1975     }
1976   }
1977   PetscCall(ISRestoreIndices(dimIS, &points));
1978   PetscCall(ISDestroy(&dimIS));
1979   PetscFunctionReturn(0);
1980 }
1981 
1982 /*@
1983   DMPlexLabelCohesiveComplete - Starting with a label marking points on an internal surface, we add all other mesh pieces
1984   to complete the surface
1985 
1986   Input Parameters:
1987 + dm     - The DM
1988 . label  - A DMLabel marking the surface
1989 . blabel - A DMLabel marking the vertices on the boundary which will not be duplicated, or NULL to find them automatically
1990 . bvalue - Value of DMLabel marking the vertices on the boundary
1991 . flip   - Flag to flip the submesh normal and replace points on the other side
1992 - subdm  - The subDM associated with the label, or NULL
1993 
1994   Output Parameter:
1995 . label - A DMLabel marking all surface points
1996 
1997   Note: The vertices in blabel are called "unsplit" in the terminology from hybrid cell creation.
1998 
1999   Level: developer
2000 
2001 .seealso: `DMPlexConstructCohesiveCells()`, `DMPlexLabelComplete()`
2002 @*/
2003 PetscErrorCode DMPlexLabelCohesiveComplete(DM dm, DMLabel label, DMLabel blabel, PetscInt bvalue, PetscBool flip, DM subdm)
2004 {
2005   DMLabel         depthLabel;
2006   IS              dimIS, subpointIS = NULL;
2007   const PetscInt *points, *subpoints;
2008   const PetscInt  rev   = flip ? -1 : 1;
2009   PetscInt        shift = 100, shift2 = 200, shift3 = 300, dim, depth, numPoints, numSubpoints, p, val;
2010 
2011   PetscFunctionBegin;
2012   PetscCall(DMPlexGetDepth(dm, &depth));
2013   PetscCall(DMGetDimension(dm, &dim));
2014   PetscCall(DMPlexGetDepthLabel(dm, &depthLabel));
2015   if (subdm) {
2016     PetscCall(DMPlexGetSubpointIS(subdm, &subpointIS));
2017     if (subpointIS) {
2018       PetscCall(ISGetLocalSize(subpointIS, &numSubpoints));
2019       PetscCall(ISGetIndices(subpointIS, &subpoints));
2020     }
2021   }
2022   /* Mark cell on the fault, and its faces which touch the fault: cell orientation for face gives the side of the fault */
2023   PetscCall(DMLabelGetStratumIS(label, dim-1, &dimIS));
2024   if (!dimIS) goto divide;
2025   PetscCall(ISGetLocalSize(dimIS, &numPoints));
2026   PetscCall(ISGetIndices(dimIS, &points));
2027   for (p = 0; p < numPoints; ++p) { /* Loop over fault faces */
2028     const PetscInt *support;
2029     PetscInt        supportSize, s;
2030 
2031     PetscCall(DMPlexGetSupportSize(dm, points[p], &supportSize));
2032 #if 0
2033     if (supportSize != 2) {
2034       const PetscInt *lp;
2035       PetscInt        Nlp, pind;
2036 
2037       /* Check that for a cell with a single support face, that face is in the SF */
2038       /*   THis check only works for the remote side. We would need root side information */
2039       PetscCall(PetscSFGetGraph(dm->sf, NULL, &Nlp, &lp, NULL));
2040       PetscCall(PetscFindInt(points[p], Nlp, lp, &pind));
2041       PetscCheck(pind >= 0,PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Split face %" PetscInt_FMT " has %" PetscInt_FMT " != 2 supports, and the face is not shared with another process", points[p], supportSize);
2042     }
2043 #endif
2044     PetscCall(DMPlexGetSupport(dm, points[p], &support));
2045     for (s = 0; s < supportSize; ++s) {
2046       const PetscInt *cone;
2047       PetscInt        coneSize, c;
2048       PetscBool       pos;
2049 
2050       PetscCall(GetSurfaceSide_Static(dm, subdm, numSubpoints, subpoints, support[s], points[p], &pos));
2051       if (pos) PetscCall(DMLabelSetValue(label, support[s],  rev*(shift+dim)));
2052       else     PetscCall(DMLabelSetValue(label, support[s], -rev*(shift+dim)));
2053       if (rev < 0) pos = !pos ? PETSC_TRUE : PETSC_FALSE;
2054       /* Put faces touching the fault in the label */
2055       PetscCall(DMPlexGetConeSize(dm, support[s], &coneSize));
2056       PetscCall(DMPlexGetCone(dm, support[s], &cone));
2057       for (c = 0; c < coneSize; ++c) {
2058         const PetscInt point = cone[c];
2059 
2060         PetscCall(DMLabelGetValue(label, point, &val));
2061         if (val == -1) {
2062           PetscInt *closure = NULL;
2063           PetscInt  closureSize, cl;
2064 
2065           PetscCall(DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
2066           for (cl = 0; cl < closureSize*2; cl += 2) {
2067             const PetscInt clp  = closure[cl];
2068             PetscInt       bval = -1;
2069 
2070             PetscCall(DMLabelGetValue(label, clp, &val));
2071             if (blabel) PetscCall(DMLabelGetValue(blabel, clp, &bval));
2072             if ((val >= 0) && (val < dim-1) && (bval < 0)) {
2073               PetscCall(DMLabelSetValue(label, point, pos == PETSC_TRUE ? shift+dim-1 : -(shift+dim-1)));
2074               break;
2075             }
2076           }
2077           PetscCall(DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
2078         }
2079       }
2080     }
2081   }
2082   PetscCall(ISRestoreIndices(dimIS, &points));
2083   PetscCall(ISDestroy(&dimIS));
2084   /* Mark boundary points as unsplit */
2085   if (blabel) {
2086     IS bdIS;
2087 
2088     PetscCall(DMLabelGetStratumIS(blabel, bvalue, &bdIS));
2089     PetscCall(ISGetLocalSize(bdIS, &numPoints));
2090     PetscCall(ISGetIndices(bdIS, &points));
2091     for (p = 0; p < numPoints; ++p) {
2092       const PetscInt point = points[p];
2093       PetscInt       val, bval;
2094 
2095       PetscCall(DMLabelGetValue(blabel, point, &bval));
2096       if (bval >= 0) {
2097         PetscCall(DMLabelGetValue(label, point, &val));
2098         if ((val < 0) || (val > dim)) {
2099           /* This could be a point added from splitting a vertex on an adjacent fault, otherwise its just wrong */
2100           PetscCall(DMLabelClearValue(blabel, point, bval));
2101         }
2102       }
2103     }
2104     for (p = 0; p < numPoints; ++p) {
2105       const PetscInt point = points[p];
2106       PetscInt       val, bval;
2107 
2108       PetscCall(DMLabelGetValue(blabel, point, &bval));
2109       if (bval >= 0) {
2110         const PetscInt *cone,    *support;
2111         PetscInt        coneSize, supportSize, s, valA, valB, valE;
2112 
2113         /* Mark as unsplit */
2114         PetscCall(DMLabelGetValue(label, point, &val));
2115         PetscCheck(val >= 0 && val <= dim, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %" PetscInt_FMT " has label value %" PetscInt_FMT ", should be part of the fault", point, val);
2116         PetscCall(DMLabelClearValue(label, point, val));
2117         PetscCall(DMLabelSetValue(label, point, shift2+val));
2118         /* Check for cross-edge
2119              A cross-edge has endpoints which are both on the boundary of the surface, but the edge itself is not. */
2120         if (val != 0) continue;
2121         PetscCall(DMPlexGetSupport(dm, point, &support));
2122         PetscCall(DMPlexGetSupportSize(dm, point, &supportSize));
2123         for (s = 0; s < supportSize; ++s) {
2124           PetscCall(DMPlexGetCone(dm, support[s], &cone));
2125           PetscCall(DMPlexGetConeSize(dm, support[s], &coneSize));
2126           PetscCheck(coneSize == 2, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Edge %" PetscInt_FMT " has %" PetscInt_FMT " vertices != 2", support[s], coneSize);
2127           PetscCall(DMLabelGetValue(blabel, cone[0], &valA));
2128           PetscCall(DMLabelGetValue(blabel, cone[1], &valB));
2129           PetscCall(DMLabelGetValue(blabel, support[s], &valE));
2130           if ((valE < 0) && (valA >= 0) && (valB >= 0) && (cone[0] != cone[1])) PetscCall(DMLabelSetValue(blabel, support[s], 2));
2131         }
2132       }
2133     }
2134     PetscCall(ISRestoreIndices(bdIS, &points));
2135     PetscCall(ISDestroy(&bdIS));
2136   }
2137   /* Mark ghost fault cells */
2138   {
2139     PetscSF         sf;
2140     const PetscInt *leaves;
2141     PetscInt         Nl, l;
2142 
2143     PetscCall(DMGetPointSF(dm, &sf));
2144     PetscCall(PetscSFGetGraph(sf, NULL, &Nl, &leaves, NULL));
2145     PetscCall(DMLabelGetStratumIS(label, dim-1, &dimIS));
2146     if (!dimIS) goto divide;
2147     PetscCall(ISGetLocalSize(dimIS, &numPoints));
2148     PetscCall(ISGetIndices(dimIS, &points));
2149     if (Nl > 0) {
2150       for (p = 0; p < numPoints; ++p) {
2151         const PetscInt point = points[p];
2152         PetscInt       val;
2153 
2154         PetscCall(PetscFindInt(point, Nl, leaves, &l));
2155         if (l >= 0) {
2156           PetscInt *closure = NULL;
2157           PetscInt  closureSize, cl;
2158 
2159           PetscCall(DMLabelGetValue(label, point, &val));
2160           PetscCheck((val == dim-1) || (val == shift2+dim-1), PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %" PetscInt_FMT " has label value %" PetscInt_FMT ", should be a fault face", point, val);
2161           PetscCall(DMLabelClearValue(label, point, val));
2162           PetscCall(DMLabelSetValue(label, point, shift3+val));
2163           PetscCall(DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
2164           for (cl = 2; cl < closureSize*2; cl += 2) {
2165             const PetscInt clp  = closure[cl];
2166 
2167             PetscCall(DMLabelGetValue(label, clp, &val));
2168             PetscCheck(val != -1, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %" PetscInt_FMT " is missing from label, but is in the closure of a fault face", point);
2169             PetscCall(DMLabelClearValue(label, clp, val));
2170             PetscCall(DMLabelSetValue(label, clp, shift3+val));
2171           }
2172           PetscCall(DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
2173         }
2174       }
2175     }
2176     PetscCall(ISRestoreIndices(dimIS, &points));
2177     PetscCall(ISDestroy(&dimIS));
2178   }
2179   divide:
2180   if (subpointIS) PetscCall(ISRestoreIndices(subpointIS, &subpoints));
2181   PetscCall(DMPlexLabelFaultHalo(dm, label));
2182   PetscCall(CheckFaultEdge_Private(dm, label));
2183   PetscFunctionReturn(0);
2184 }
2185 
2186 /* Check that no cell have all vertices on the fault */
2187 PetscErrorCode DMPlexCheckValidSubmesh_Private(DM dm, DMLabel label, DM subdm)
2188 {
2189   IS              subpointIS;
2190   const PetscInt *dmpoints;
2191   PetscInt        defaultValue, cStart, cEnd, c, vStart, vEnd;
2192 
2193   PetscFunctionBegin;
2194   if (!label) PetscFunctionReturn(0);
2195   PetscCall(DMLabelGetDefaultValue(label, &defaultValue));
2196   PetscCall(DMPlexGetSubpointIS(subdm, &subpointIS));
2197   if (!subpointIS) PetscFunctionReturn(0);
2198   PetscCall(DMPlexGetHeightStratum(subdm, 0, &cStart, &cEnd));
2199   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
2200   PetscCall(ISGetIndices(subpointIS, &dmpoints));
2201   for (c = cStart; c < cEnd; ++c) {
2202     PetscBool invalidCell = PETSC_TRUE;
2203     PetscInt *closure     = NULL;
2204     PetscInt  closureSize, cl;
2205 
2206     PetscCall(DMPlexGetTransitiveClosure(dm, dmpoints[c], PETSC_TRUE, &closureSize, &closure));
2207     for (cl = 0; cl < closureSize*2; cl += 2) {
2208       PetscInt value = 0;
2209 
2210       if ((closure[cl] < vStart) || (closure[cl] >= vEnd)) continue;
2211       PetscCall(DMLabelGetValue(label, closure[cl], &value));
2212       if (value == defaultValue) {invalidCell = PETSC_FALSE; break;}
2213     }
2214     PetscCall(DMPlexRestoreTransitiveClosure(dm, dmpoints[c], PETSC_TRUE, &closureSize, &closure));
2215     if (invalidCell) {
2216       PetscCall(ISRestoreIndices(subpointIS, &dmpoints));
2217       PetscCall(ISDestroy(&subpointIS));
2218       PetscCall(DMDestroy(&subdm));
2219       SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Ambiguous submesh. Cell %" PetscInt_FMT " has all of its vertices on the submesh.", dmpoints[c]);
2220     }
2221   }
2222   PetscCall(ISRestoreIndices(subpointIS, &dmpoints));
2223   PetscFunctionReturn(0);
2224 }
2225 
2226 /*@
2227   DMPlexCreateHybridMesh - Create a mesh with hybrid cells along an internal interface
2228 
2229   Collective on dm
2230 
2231   Input Parameters:
2232 + dm - The original DM
2233 . label - The label specifying the interface vertices
2234 . bdlabel - The optional label specifying the interface boundary vertices
2235 - bdvalue - Value of optional label specifying the interface boundary vertices
2236 
2237   Output Parameters:
2238 + hybridLabel - The label fully marking the interface, or NULL if no output is desired
2239 . splitLabel - The label containing the split points, or NULL if no output is desired
2240 . dmInterface - The new interface DM, or NULL
2241 - dmHybrid - The new DM with cohesive cells
2242 
2243   Note: The hybridLabel indicates what parts of the original mesh impinged on the on division surface. For points
2244   directly on the division surface, they are labeled with their dimension, so an edge 7 on the division surface would be
2245   7 (1) in hybridLabel. For points that impinge from the positive side, they are labeled with 100+dim, so an edge 6 with
2246   one vertex 3 on the surface would be 6 (101) and 3 (0) in hybridLabel. If an edge 9 from the negative side of the
2247   surface also hits vertex 3, it would be 9 (-101) in hybridLabel.
2248 
2249   The splitLabel indicates what points in the new hybrid mesh were the result of splitting points in the original
2250   mesh. The label value is +=100+dim for each point. For example, if two edges 10 and 14 in the hybrid resulting from
2251   splitting an edge in the original mesh, you would have 10 (101) and 14 (-101) in the splitLabel.
2252 
2253   The dmInterface is a DM built from the original division surface. It has a label which can be retrieved using
2254   DMPlexGetSubpointMap() which maps each point back to the point in the surface of the original mesh.
2255 
2256   Level: developer
2257 
2258 .seealso: `DMPlexConstructCohesiveCells()`, `DMPlexLabelCohesiveComplete()`, `DMPlexGetSubpointMap()`, `DMCreate()`
2259 @*/
2260 PetscErrorCode DMPlexCreateHybridMesh(DM dm, DMLabel label, DMLabel bdlabel, PetscInt bdvalue, DMLabel *hybridLabel, DMLabel *splitLabel, DM *dmInterface, DM *dmHybrid)
2261 {
2262   DM             idm;
2263   DMLabel        subpointMap, hlabel, slabel = NULL;
2264   PetscInt       dim;
2265 
2266   PetscFunctionBegin;
2267   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2268   if (label) PetscValidPointer(label, 2);
2269   if (bdlabel) PetscValidPointer(bdlabel, 3);
2270   if (hybridLabel) PetscValidPointer(hybridLabel, 5);
2271   if (splitLabel)  PetscValidPointer(splitLabel, 6);
2272   if (dmInterface) PetscValidPointer(dmInterface, 7);
2273   PetscValidPointer(dmHybrid, 8);
2274   PetscCall(DMGetDimension(dm, &dim));
2275   PetscCall(DMPlexCreateSubmesh(dm, label, 1, PETSC_FALSE, &idm));
2276   PetscCall(DMPlexCheckValidSubmesh_Private(dm, label, idm));
2277   PetscCall(DMPlexOrient(idm));
2278   PetscCall(DMPlexGetSubpointMap(idm, &subpointMap));
2279   PetscCall(DMLabelDuplicate(subpointMap, &hlabel));
2280   PetscCall(DMLabelClearStratum(hlabel, dim));
2281   if (splitLabel) {
2282     const char *name;
2283     char        sname[PETSC_MAX_PATH_LEN];
2284 
2285     PetscCall(PetscObjectGetName((PetscObject) hlabel, &name));
2286     PetscCall(PetscStrncpy(sname, name, PETSC_MAX_PATH_LEN));
2287     PetscCall(PetscStrcat(sname, " split"));
2288     PetscCall(DMLabelCreate(PETSC_COMM_SELF, sname, &slabel));
2289   }
2290   PetscCall(DMPlexLabelCohesiveComplete(dm, hlabel, bdlabel, bdvalue, PETSC_FALSE, idm));
2291   if (dmInterface) {*dmInterface = idm;}
2292   else             PetscCall(DMDestroy(&idm));
2293   PetscCall(DMPlexConstructCohesiveCells(dm, hlabel, slabel, dmHybrid));
2294   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_TRUE, *dmHybrid));
2295   if (hybridLabel) *hybridLabel = hlabel;
2296   else             PetscCall(DMLabelDestroy(&hlabel));
2297   if (splitLabel)  *splitLabel  = slabel;
2298   {
2299     DM      cdm;
2300     DMLabel ctLabel;
2301 
2302     /* We need to somehow share the celltype label with the coordinate dm */
2303     PetscCall(DMGetCoordinateDM(*dmHybrid, &cdm));
2304     PetscCall(DMPlexGetCellTypeLabel(*dmHybrid, &ctLabel));
2305     PetscCall(DMSetLabel(cdm, ctLabel));
2306   }
2307   PetscFunctionReturn(0);
2308 }
2309 
2310 /* Here we need the explicit assumption that:
2311 
2312      For any marked cell, the marked vertices constitute a single face
2313 */
2314 static PetscErrorCode DMPlexMarkSubmesh_Uninterpolated(DM dm, DMLabel vertexLabel, PetscInt value, DMLabel subpointMap, PetscInt *numFaces, PetscInt *nFV, DM subdm)
2315 {
2316   IS               subvertexIS = NULL;
2317   const PetscInt  *subvertices;
2318   PetscInt        *pStart, *pEnd, pSize;
2319   PetscInt         depth, dim, d, numSubVerticesInitial = 0, v;
2320 
2321   PetscFunctionBegin;
2322   *numFaces = 0;
2323   *nFV      = 0;
2324   PetscCall(DMPlexGetDepth(dm, &depth));
2325   PetscCall(DMGetDimension(dm, &dim));
2326   pSize = PetscMax(depth, dim) + 1;
2327   PetscCall(PetscMalloc2(pSize, &pStart, pSize, &pEnd));
2328   for (d = 0; d <= depth; ++d) {
2329     PetscCall(DMPlexGetSimplexOrBoxCells(dm, depth-d, &pStart[d], &pEnd[d]));
2330   }
2331   /* Loop over initial vertices and mark all faces in the collective star() */
2332   if (vertexLabel) PetscCall(DMLabelGetStratumIS(vertexLabel, value, &subvertexIS));
2333   if (subvertexIS) {
2334     PetscCall(ISGetSize(subvertexIS, &numSubVerticesInitial));
2335     PetscCall(ISGetIndices(subvertexIS, &subvertices));
2336   }
2337   for (v = 0; v < numSubVerticesInitial; ++v) {
2338     const PetscInt vertex = subvertices[v];
2339     PetscInt      *star   = NULL;
2340     PetscInt       starSize, s, numCells = 0, c;
2341 
2342     PetscCall(DMPlexGetTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star));
2343     for (s = 0; s < starSize*2; s += 2) {
2344       const PetscInt point = star[s];
2345       if ((point >= pStart[depth]) && (point < pEnd[depth])) star[numCells++] = point;
2346     }
2347     for (c = 0; c < numCells; ++c) {
2348       const PetscInt cell    = star[c];
2349       PetscInt      *closure = NULL;
2350       PetscInt       closureSize, cl;
2351       PetscInt       cellLoc, numCorners = 0, faceSize = 0;
2352 
2353       PetscCall(DMLabelGetValue(subpointMap, cell, &cellLoc));
2354       if (cellLoc == 2) continue;
2355       PetscCheck(cellLoc < 0,PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Cell %" PetscInt_FMT " has dimension %" PetscInt_FMT " in the surface label", cell, cellLoc);
2356       PetscCall(DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
2357       for (cl = 0; cl < closureSize*2; cl += 2) {
2358         const PetscInt point = closure[cl];
2359         PetscInt       vertexLoc;
2360 
2361         if ((point >= pStart[0]) && (point < pEnd[0])) {
2362           ++numCorners;
2363           PetscCall(DMLabelGetValue(vertexLabel, point, &vertexLoc));
2364           if (vertexLoc == value) closure[faceSize++] = point;
2365         }
2366       }
2367       if (!(*nFV)) PetscCall(DMPlexGetNumFaceVertices(dm, dim, numCorners, nFV));
2368       PetscCheck(faceSize <= *nFV,PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Invalid submesh: Too many vertices %" PetscInt_FMT " of an element on the surface", faceSize);
2369       if (faceSize == *nFV) {
2370         const PetscInt *cells = NULL;
2371         PetscInt        numCells, nc;
2372 
2373         ++(*numFaces);
2374         for (cl = 0; cl < faceSize; ++cl) {
2375           PetscCall(DMLabelSetValue(subpointMap, closure[cl], 0));
2376         }
2377         PetscCall(DMPlexGetJoin(dm, faceSize, closure, &numCells, &cells));
2378         for (nc = 0; nc < numCells; ++nc) {
2379           PetscCall(DMLabelSetValue(subpointMap, cells[nc], 2));
2380         }
2381         PetscCall(DMPlexRestoreJoin(dm, faceSize, closure, &numCells, &cells));
2382       }
2383       PetscCall(DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
2384     }
2385     PetscCall(DMPlexRestoreTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star));
2386   }
2387   if (subvertexIS) {
2388     PetscCall(ISRestoreIndices(subvertexIS, &subvertices));
2389   }
2390   PetscCall(ISDestroy(&subvertexIS));
2391   PetscCall(PetscFree2(pStart, pEnd));
2392   PetscFunctionReturn(0);
2393 }
2394 
2395 static PetscErrorCode DMPlexMarkSubmesh_Interpolated(DM dm, DMLabel vertexLabel, PetscInt value, PetscBool markedFaces, DMLabel subpointMap, DM subdm)
2396 {
2397   IS               subvertexIS = NULL;
2398   const PetscInt  *subvertices;
2399   PetscInt        *pStart, *pEnd;
2400   PetscInt         dim, d, numSubVerticesInitial = 0, v;
2401 
2402   PetscFunctionBegin;
2403   PetscCall(DMGetDimension(dm, &dim));
2404   PetscCall(PetscMalloc2(dim+1, &pStart, dim+1, &pEnd));
2405   for (d = 0; d <= dim; ++d) {
2406     PetscCall(DMPlexGetSimplexOrBoxCells(dm, dim-d, &pStart[d], &pEnd[d]));
2407   }
2408   /* Loop over initial vertices and mark all faces in the collective star() */
2409   if (vertexLabel) {
2410     PetscCall(DMLabelGetStratumIS(vertexLabel, value, &subvertexIS));
2411     if (subvertexIS) {
2412       PetscCall(ISGetSize(subvertexIS, &numSubVerticesInitial));
2413       PetscCall(ISGetIndices(subvertexIS, &subvertices));
2414     }
2415   }
2416   for (v = 0; v < numSubVerticesInitial; ++v) {
2417     const PetscInt vertex = subvertices[v];
2418     PetscInt      *star   = NULL;
2419     PetscInt       starSize, s, numFaces = 0, f;
2420 
2421     PetscCall(DMPlexGetTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star));
2422     for (s = 0; s < starSize*2; s += 2) {
2423       const PetscInt point = star[s];
2424       PetscInt       faceLoc;
2425 
2426       if ((point >= pStart[dim-1]) && (point < pEnd[dim-1])) {
2427         if (markedFaces) {
2428           PetscCall(DMLabelGetValue(vertexLabel, point, &faceLoc));
2429           if (faceLoc < 0) continue;
2430         }
2431         star[numFaces++] = point;
2432       }
2433     }
2434     for (f = 0; f < numFaces; ++f) {
2435       const PetscInt face    = star[f];
2436       PetscInt      *closure = NULL;
2437       PetscInt       closureSize, c;
2438       PetscInt       faceLoc;
2439 
2440       PetscCall(DMLabelGetValue(subpointMap, face, &faceLoc));
2441       if (faceLoc == dim-1) continue;
2442       PetscCheck(faceLoc < 0,PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Face %" PetscInt_FMT " has dimension %" PetscInt_FMT " in the surface label", face, faceLoc);
2443       PetscCall(DMPlexGetTransitiveClosure(dm, face, PETSC_TRUE, &closureSize, &closure));
2444       for (c = 0; c < closureSize*2; c += 2) {
2445         const PetscInt point = closure[c];
2446         PetscInt       vertexLoc;
2447 
2448         if ((point >= pStart[0]) && (point < pEnd[0])) {
2449           PetscCall(DMLabelGetValue(vertexLabel, point, &vertexLoc));
2450           if (vertexLoc != value) break;
2451         }
2452       }
2453       if (c == closureSize*2) {
2454         const PetscInt *support;
2455         PetscInt        supportSize, s;
2456 
2457         for (c = 0; c < closureSize*2; c += 2) {
2458           const PetscInt point = closure[c];
2459 
2460           for (d = 0; d < dim; ++d) {
2461             if ((point >= pStart[d]) && (point < pEnd[d])) {
2462               PetscCall(DMLabelSetValue(subpointMap, point, d));
2463               break;
2464             }
2465           }
2466         }
2467         PetscCall(DMPlexGetSupportSize(dm, face, &supportSize));
2468         PetscCall(DMPlexGetSupport(dm, face, &support));
2469         for (s = 0; s < supportSize; ++s) {
2470           PetscCall(DMLabelSetValue(subpointMap, support[s], dim));
2471         }
2472       }
2473       PetscCall(DMPlexRestoreTransitiveClosure(dm, face, PETSC_TRUE, &closureSize, &closure));
2474     }
2475     PetscCall(DMPlexRestoreTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star));
2476   }
2477   if (subvertexIS) PetscCall(ISRestoreIndices(subvertexIS, &subvertices));
2478   PetscCall(ISDestroy(&subvertexIS));
2479   PetscCall(PetscFree2(pStart, pEnd));
2480   PetscFunctionReturn(0);
2481 }
2482 
2483 static PetscErrorCode DMPlexMarkCohesiveSubmesh_Uninterpolated(DM dm, PetscBool hasLagrange, const char labelname[], PetscInt value, DMLabel subpointMap, PetscInt *numFaces, PetscInt *nFV, PetscInt *subCells[], DM subdm)
2484 {
2485   DMLabel         label = NULL;
2486   const PetscInt *cone;
2487   PetscInt        dim, cMax, cEnd, c, subc = 0, p, coneSize = -1;
2488 
2489   PetscFunctionBegin;
2490   *numFaces = 0;
2491   *nFV = 0;
2492   if (labelname) PetscCall(DMGetLabel(dm, labelname, &label));
2493   *subCells = NULL;
2494   PetscCall(DMGetDimension(dm, &dim));
2495   PetscCall(DMPlexGetTensorPrismBounds_Internal(dm, dim, &cMax, &cEnd));
2496   if (cMax < 0) PetscFunctionReturn(0);
2497   if (label) {
2498     for (c = cMax; c < cEnd; ++c) {
2499       PetscInt val;
2500 
2501       PetscCall(DMLabelGetValue(label, c, &val));
2502       if (val == value) {
2503         ++(*numFaces);
2504         PetscCall(DMPlexGetConeSize(dm, c, &coneSize));
2505       }
2506     }
2507   } else {
2508     *numFaces = cEnd - cMax;
2509     PetscCall(DMPlexGetConeSize(dm, cMax, &coneSize));
2510   }
2511   PetscCall(PetscMalloc1(*numFaces *2, subCells));
2512   if (!(*numFaces)) PetscFunctionReturn(0);
2513   *nFV = hasLagrange ? coneSize/3 : coneSize/2;
2514   for (c = cMax; c < cEnd; ++c) {
2515     const PetscInt *cells;
2516     PetscInt        numCells;
2517 
2518     if (label) {
2519       PetscInt val;
2520 
2521       PetscCall(DMLabelGetValue(label, c, &val));
2522       if (val != value) continue;
2523     }
2524     PetscCall(DMPlexGetCone(dm, c, &cone));
2525     for (p = 0; p < *nFV; ++p) {
2526       PetscCall(DMLabelSetValue(subpointMap, cone[p], 0));
2527     }
2528     /* Negative face */
2529     PetscCall(DMPlexGetJoin(dm, *nFV, cone, &numCells, &cells));
2530     /* Not true in parallel
2531     PetscCheck(numCells == 2,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive cells should separate two cells"); */
2532     for (p = 0; p < numCells; ++p) {
2533       PetscCall(DMLabelSetValue(subpointMap, cells[p], 2));
2534       (*subCells)[subc++] = cells[p];
2535     }
2536     PetscCall(DMPlexRestoreJoin(dm, *nFV, cone, &numCells, &cells));
2537     /* Positive face is not included */
2538   }
2539   PetscFunctionReturn(0);
2540 }
2541 
2542 static PetscErrorCode DMPlexMarkCohesiveSubmesh_Interpolated(DM dm, DMLabel label, PetscInt value, DMLabel subpointMap, DM subdm)
2543 {
2544   PetscInt      *pStart, *pEnd;
2545   PetscInt       dim, cMax, cEnd, c, d;
2546 
2547   PetscFunctionBegin;
2548   PetscCall(DMGetDimension(dm, &dim));
2549   PetscCall(DMPlexGetTensorPrismBounds_Internal(dm, dim, &cMax, &cEnd));
2550   if (cMax < 0) PetscFunctionReturn(0);
2551   PetscCall(PetscMalloc2(dim+1,&pStart,dim+1,&pEnd));
2552   for (d = 0; d <= dim; ++d) PetscCall(DMPlexGetDepthStratum(dm, d, &pStart[d], &pEnd[d]));
2553   for (c = cMax; c < cEnd; ++c) {
2554     const PetscInt *cone;
2555     PetscInt       *closure = NULL;
2556     PetscInt        fconeSize, coneSize, closureSize, cl, val;
2557 
2558     if (label) {
2559       PetscCall(DMLabelGetValue(label, c, &val));
2560       if (val != value) continue;
2561     }
2562     PetscCall(DMPlexGetConeSize(dm, c, &coneSize));
2563     PetscCall(DMPlexGetCone(dm, c, &cone));
2564     PetscCall(DMPlexGetConeSize(dm, cone[0], &fconeSize));
2565     PetscCheck(coneSize == (fconeSize ? fconeSize : 1) + 2,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive cells should separate two cells");
2566     /* Negative face */
2567     PetscCall(DMPlexGetTransitiveClosure(dm, cone[0], PETSC_TRUE, &closureSize, &closure));
2568     for (cl = 0; cl < closureSize*2; cl += 2) {
2569       const PetscInt point = closure[cl];
2570 
2571       for (d = 0; d <= dim; ++d) {
2572         if ((point >= pStart[d]) && (point < pEnd[d])) {
2573           PetscCall(DMLabelSetValue(subpointMap, point, d));
2574           break;
2575         }
2576       }
2577     }
2578     PetscCall(DMPlexRestoreTransitiveClosure(dm, cone[0], PETSC_TRUE, &closureSize, &closure));
2579     /* Cells -- positive face is not included */
2580     for (cl = 0; cl < 1; ++cl) {
2581       const PetscInt *support;
2582       PetscInt        supportSize, s;
2583 
2584       PetscCall(DMPlexGetSupportSize(dm, cone[cl], &supportSize));
2585       /* PetscCheck(supportSize == 2,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive faces should separate two cells"); */
2586       PetscCall(DMPlexGetSupport(dm, cone[cl], &support));
2587       for (s = 0; s < supportSize; ++s) {
2588         PetscCall(DMLabelSetValue(subpointMap, support[s], dim));
2589       }
2590     }
2591   }
2592   PetscCall(PetscFree2(pStart, pEnd));
2593   PetscFunctionReturn(0);
2594 }
2595 
2596 static PetscErrorCode DMPlexGetFaceOrientation(DM dm, PetscInt cell, PetscInt numCorners, PetscInt indices[], PetscInt oppositeVertex, PetscInt origVertices[], PetscInt faceVertices[], PetscBool *posOriented)
2597 {
2598   MPI_Comm       comm;
2599   PetscBool      posOrient = PETSC_FALSE;
2600   const PetscInt debug     = 0;
2601   PetscInt       cellDim, faceSize, f;
2602 
2603   PetscFunctionBegin;
2604   PetscCall(PetscObjectGetComm((PetscObject)dm,&comm));
2605   PetscCall(DMGetDimension(dm, &cellDim));
2606   if (debug) PetscCall(PetscPrintf(comm, "cellDim: %" PetscInt_FMT " numCorners: %" PetscInt_FMT "\n", cellDim, numCorners));
2607 
2608   if (cellDim == 1 && numCorners == 2) {
2609     /* Triangle */
2610     faceSize  = numCorners-1;
2611     posOrient = !(oppositeVertex%2) ? PETSC_TRUE : PETSC_FALSE;
2612   } else if (cellDim == 2 && numCorners == 3) {
2613     /* Triangle */
2614     faceSize  = numCorners-1;
2615     posOrient = !(oppositeVertex%2) ? PETSC_TRUE : PETSC_FALSE;
2616   } else if (cellDim == 3 && numCorners == 4) {
2617     /* Tetrahedron */
2618     faceSize  = numCorners-1;
2619     posOrient = (oppositeVertex%2) ? PETSC_TRUE : PETSC_FALSE;
2620   } else if (cellDim == 1 && numCorners == 3) {
2621     /* Quadratic line */
2622     faceSize  = 1;
2623     posOrient = PETSC_TRUE;
2624   } else if (cellDim == 2 && numCorners == 4) {
2625     /* Quads */
2626     faceSize = 2;
2627     if ((indices[1] > indices[0]) && (indices[1] - indices[0] == 1)) {
2628       posOrient = PETSC_TRUE;
2629     } else if ((indices[0] == 3) && (indices[1] == 0)) {
2630       posOrient = PETSC_TRUE;
2631     } else {
2632       if (((indices[0] > indices[1]) && (indices[0] - indices[1] == 1)) || ((indices[0] == 0) && (indices[1] == 3))) {
2633         posOrient = PETSC_FALSE;
2634       } else SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid quad crossedge");
2635     }
2636   } else if (cellDim == 2 && numCorners == 6) {
2637     /* Quadratic triangle (I hate this) */
2638     /* Edges are determined by the first 2 vertices (corners of edges) */
2639     const PetscInt faceSizeTri = 3;
2640     PetscInt       sortedIndices[3], i, iFace;
2641     PetscBool      found                    = PETSC_FALSE;
2642     PetscInt       faceVerticesTriSorted[9] = {
2643       0, 3,  4, /* bottom */
2644       1, 4,  5, /* right */
2645       2, 3,  5, /* left */
2646     };
2647     PetscInt       faceVerticesTri[9] = {
2648       0, 3,  4, /* bottom */
2649       1, 4,  5, /* right */
2650       2, 5,  3, /* left */
2651     };
2652 
2653     for (i = 0; i < faceSizeTri; ++i) sortedIndices[i] = indices[i];
2654     PetscCall(PetscSortInt(faceSizeTri, sortedIndices));
2655     for (iFace = 0; iFace < 3; ++iFace) {
2656       const PetscInt ii = iFace*faceSizeTri;
2657       PetscInt       fVertex, cVertex;
2658 
2659       if ((sortedIndices[0] == faceVerticesTriSorted[ii+0]) &&
2660           (sortedIndices[1] == faceVerticesTriSorted[ii+1])) {
2661         for (fVertex = 0; fVertex < faceSizeTri; ++fVertex) {
2662           for (cVertex = 0; cVertex < faceSizeTri; ++cVertex) {
2663             if (indices[cVertex] == faceVerticesTri[ii+fVertex]) {
2664               faceVertices[fVertex] = origVertices[cVertex];
2665               break;
2666             }
2667           }
2668         }
2669         found = PETSC_TRUE;
2670         break;
2671       }
2672     }
2673     PetscCheck(found,comm, PETSC_ERR_ARG_WRONG, "Invalid tri crossface");
2674     if (posOriented) *posOriented = PETSC_TRUE;
2675     PetscFunctionReturn(0);
2676   } else if (cellDim == 2 && numCorners == 9) {
2677     /* Quadratic quad (I hate this) */
2678     /* Edges are determined by the first 2 vertices (corners of edges) */
2679     const PetscInt faceSizeQuad = 3;
2680     PetscInt       sortedIndices[3], i, iFace;
2681     PetscBool      found                      = PETSC_FALSE;
2682     PetscInt       faceVerticesQuadSorted[12] = {
2683       0, 1,  4, /* bottom */
2684       1, 2,  5, /* right */
2685       2, 3,  6, /* top */
2686       0, 3,  7, /* left */
2687     };
2688     PetscInt       faceVerticesQuad[12] = {
2689       0, 1,  4, /* bottom */
2690       1, 2,  5, /* right */
2691       2, 3,  6, /* top */
2692       3, 0,  7, /* left */
2693     };
2694 
2695     for (i = 0; i < faceSizeQuad; ++i) sortedIndices[i] = indices[i];
2696     PetscCall(PetscSortInt(faceSizeQuad, sortedIndices));
2697     for (iFace = 0; iFace < 4; ++iFace) {
2698       const PetscInt ii = iFace*faceSizeQuad;
2699       PetscInt       fVertex, cVertex;
2700 
2701       if ((sortedIndices[0] == faceVerticesQuadSorted[ii+0]) &&
2702           (sortedIndices[1] == faceVerticesQuadSorted[ii+1])) {
2703         for (fVertex = 0; fVertex < faceSizeQuad; ++fVertex) {
2704           for (cVertex = 0; cVertex < faceSizeQuad; ++cVertex) {
2705             if (indices[cVertex] == faceVerticesQuad[ii+fVertex]) {
2706               faceVertices[fVertex] = origVertices[cVertex];
2707               break;
2708             }
2709           }
2710         }
2711         found = PETSC_TRUE;
2712         break;
2713       }
2714     }
2715     PetscCheck(found,comm, PETSC_ERR_ARG_WRONG, "Invalid quad crossface");
2716     if (posOriented) *posOriented = PETSC_TRUE;
2717     PetscFunctionReturn(0);
2718   } else if (cellDim == 3 && numCorners == 8) {
2719     /* Hexes
2720        A hex is two oriented quads with the normal of the first
2721        pointing up at the second.
2722 
2723           7---6
2724          /|  /|
2725         4---5 |
2726         | 1-|-2
2727         |/  |/
2728         0---3
2729 
2730         Faces are determined by the first 4 vertices (corners of faces) */
2731     const PetscInt faceSizeHex = 4;
2732     PetscInt       sortedIndices[4], i, iFace;
2733     PetscBool      found                     = PETSC_FALSE;
2734     PetscInt       faceVerticesHexSorted[24] = {
2735       0, 1, 2, 3,  /* bottom */
2736       4, 5, 6, 7,  /* top */
2737       0, 3, 4, 5,  /* front */
2738       2, 3, 5, 6,  /* right */
2739       1, 2, 6, 7,  /* back */
2740       0, 1, 4, 7,  /* left */
2741     };
2742     PetscInt       faceVerticesHex[24] = {
2743       1, 2, 3, 0,  /* bottom */
2744       4, 5, 6, 7,  /* top */
2745       0, 3, 5, 4,  /* front */
2746       3, 2, 6, 5,  /* right */
2747       2, 1, 7, 6,  /* back */
2748       1, 0, 4, 7,  /* left */
2749     };
2750 
2751     for (i = 0; i < faceSizeHex; ++i) sortedIndices[i] = indices[i];
2752     PetscCall(PetscSortInt(faceSizeHex, sortedIndices));
2753     for (iFace = 0; iFace < 6; ++iFace) {
2754       const PetscInt ii = iFace*faceSizeHex;
2755       PetscInt       fVertex, cVertex;
2756 
2757       if ((sortedIndices[0] == faceVerticesHexSorted[ii+0]) &&
2758           (sortedIndices[1] == faceVerticesHexSorted[ii+1]) &&
2759           (sortedIndices[2] == faceVerticesHexSorted[ii+2]) &&
2760           (sortedIndices[3] == faceVerticesHexSorted[ii+3])) {
2761         for (fVertex = 0; fVertex < faceSizeHex; ++fVertex) {
2762           for (cVertex = 0; cVertex < faceSizeHex; ++cVertex) {
2763             if (indices[cVertex] == faceVerticesHex[ii+fVertex]) {
2764               faceVertices[fVertex] = origVertices[cVertex];
2765               break;
2766             }
2767           }
2768         }
2769         found = PETSC_TRUE;
2770         break;
2771       }
2772     }
2773     PetscCheck(found,comm, PETSC_ERR_ARG_WRONG, "Invalid hex crossface");
2774     if (posOriented) *posOriented = PETSC_TRUE;
2775     PetscFunctionReturn(0);
2776   } else if (cellDim == 3 && numCorners == 10) {
2777     /* Quadratic tet */
2778     /* Faces are determined by the first 3 vertices (corners of faces) */
2779     const PetscInt faceSizeTet = 6;
2780     PetscInt       sortedIndices[6], i, iFace;
2781     PetscBool      found                     = PETSC_FALSE;
2782     PetscInt       faceVerticesTetSorted[24] = {
2783       0, 1, 2,  6, 7, 8, /* bottom */
2784       0, 3, 4,  6, 7, 9,  /* front */
2785       1, 4, 5,  7, 8, 9,  /* right */
2786       2, 3, 5,  6, 8, 9,  /* left */
2787     };
2788     PetscInt       faceVerticesTet[24] = {
2789       0, 1, 2,  6, 7, 8, /* bottom */
2790       0, 4, 3,  6, 7, 9,  /* front */
2791       1, 5, 4,  7, 8, 9,  /* right */
2792       2, 3, 5,  8, 6, 9,  /* left */
2793     };
2794 
2795     for (i = 0; i < faceSizeTet; ++i) sortedIndices[i] = indices[i];
2796     PetscCall(PetscSortInt(faceSizeTet, sortedIndices));
2797     for (iFace=0; iFace < 4; ++iFace) {
2798       const PetscInt ii = iFace*faceSizeTet;
2799       PetscInt       fVertex, cVertex;
2800 
2801       if ((sortedIndices[0] == faceVerticesTetSorted[ii+0]) &&
2802           (sortedIndices[1] == faceVerticesTetSorted[ii+1]) &&
2803           (sortedIndices[2] == faceVerticesTetSorted[ii+2]) &&
2804           (sortedIndices[3] == faceVerticesTetSorted[ii+3])) {
2805         for (fVertex = 0; fVertex < faceSizeTet; ++fVertex) {
2806           for (cVertex = 0; cVertex < faceSizeTet; ++cVertex) {
2807             if (indices[cVertex] == faceVerticesTet[ii+fVertex]) {
2808               faceVertices[fVertex] = origVertices[cVertex];
2809               break;
2810             }
2811           }
2812         }
2813         found = PETSC_TRUE;
2814         break;
2815       }
2816     }
2817     PetscCheck(found,comm, PETSC_ERR_ARG_WRONG, "Invalid tet crossface");
2818     if (posOriented) *posOriented = PETSC_TRUE;
2819     PetscFunctionReturn(0);
2820   } else if (cellDim == 3 && numCorners == 27) {
2821     /* Quadratic hexes (I hate this)
2822        A hex is two oriented quads with the normal of the first
2823        pointing up at the second.
2824 
2825          7---6
2826         /|  /|
2827        4---5 |
2828        | 3-|-2
2829        |/  |/
2830        0---1
2831 
2832        Faces are determined by the first 4 vertices (corners of faces) */
2833     const PetscInt faceSizeQuadHex = 9;
2834     PetscInt       sortedIndices[9], i, iFace;
2835     PetscBool      found                         = PETSC_FALSE;
2836     PetscInt       faceVerticesQuadHexSorted[54] = {
2837       0, 1, 2, 3,  8, 9, 10, 11,  24, /* bottom */
2838       4, 5, 6, 7,  12, 13, 14, 15,  25, /* top */
2839       0, 1, 4, 5,  8, 12, 16, 17,  22, /* front */
2840       1, 2, 5, 6,  9, 13, 17, 18,  21, /* right */
2841       2, 3, 6, 7,  10, 14, 18, 19,  23, /* back */
2842       0, 3, 4, 7,  11, 15, 16, 19,  20, /* left */
2843     };
2844     PetscInt       faceVerticesQuadHex[54] = {
2845       3, 2, 1, 0,  10, 9, 8, 11,  24, /* bottom */
2846       4, 5, 6, 7,  12, 13, 14, 15,  25, /* top */
2847       0, 1, 5, 4,  8, 17, 12, 16,  22, /* front */
2848       1, 2, 6, 5,  9, 18, 13, 17,  21, /* right */
2849       2, 3, 7, 6,  10, 19, 14, 18,  23, /* back */
2850       3, 0, 4, 7,  11, 16, 15, 19,  20 /* left */
2851     };
2852 
2853     for (i = 0; i < faceSizeQuadHex; ++i) sortedIndices[i] = indices[i];
2854     PetscCall(PetscSortInt(faceSizeQuadHex, sortedIndices));
2855     for (iFace = 0; iFace < 6; ++iFace) {
2856       const PetscInt ii = iFace*faceSizeQuadHex;
2857       PetscInt       fVertex, cVertex;
2858 
2859       if ((sortedIndices[0] == faceVerticesQuadHexSorted[ii+0]) &&
2860           (sortedIndices[1] == faceVerticesQuadHexSorted[ii+1]) &&
2861           (sortedIndices[2] == faceVerticesQuadHexSorted[ii+2]) &&
2862           (sortedIndices[3] == faceVerticesQuadHexSorted[ii+3])) {
2863         for (fVertex = 0; fVertex < faceSizeQuadHex; ++fVertex) {
2864           for (cVertex = 0; cVertex < faceSizeQuadHex; ++cVertex) {
2865             if (indices[cVertex] == faceVerticesQuadHex[ii+fVertex]) {
2866               faceVertices[fVertex] = origVertices[cVertex];
2867               break;
2868             }
2869           }
2870         }
2871         found = PETSC_TRUE;
2872         break;
2873       }
2874     }
2875     PetscCheck(found,comm, PETSC_ERR_ARG_WRONG, "Invalid hex crossface");
2876     if (posOriented) *posOriented = PETSC_TRUE;
2877     PetscFunctionReturn(0);
2878   } else SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Unknown cell type for faceOrientation().");
2879   if (!posOrient) {
2880     if (debug) PetscCall(PetscPrintf(comm, "  Reversing initial face orientation\n"));
2881     for (f = 0; f < faceSize; ++f) faceVertices[f] = origVertices[faceSize-1 - f];
2882   } else {
2883     if (debug) PetscCall(PetscPrintf(comm, "  Keeping initial face orientation\n"));
2884     for (f = 0; f < faceSize; ++f) faceVertices[f] = origVertices[f];
2885   }
2886   if (posOriented) *posOriented = posOrient;
2887   PetscFunctionReturn(0);
2888 }
2889 
2890 /*@
2891   DMPlexGetOrientedFace - Given a cell and a face, as a set of vertices, return the oriented face, as a set of vertices,
2892   in faceVertices. The orientation is such that the face normal points out of the cell
2893 
2894   Not collective
2895 
2896   Input Parameters:
2897 + dm           - The original mesh
2898 . cell         - The cell mesh point
2899 . faceSize     - The number of vertices on the face
2900 . face         - The face vertices
2901 . numCorners   - The number of vertices on the cell
2902 . indices      - Local numbering of face vertices in cell cone
2903 - origVertices - Original face vertices
2904 
2905   Output Parameters:
2906 + faceVertices - The face vertices properly oriented
2907 - posOriented  - PETSC_TRUE if the face was oriented with outward normal
2908 
2909   Level: developer
2910 
2911 .seealso: `DMPlexGetCone()`
2912 @*/
2913 PetscErrorCode DMPlexGetOrientedFace(DM dm, PetscInt cell, PetscInt faceSize, const PetscInt face[], PetscInt numCorners, PetscInt indices[], PetscInt origVertices[], PetscInt faceVertices[], PetscBool *posOriented)
2914 {
2915   const PetscInt *cone = NULL;
2916   PetscInt        coneSize, v, f, v2;
2917   PetscInt        oppositeVertex = -1;
2918 
2919   PetscFunctionBegin;
2920   PetscCall(DMPlexGetConeSize(dm, cell, &coneSize));
2921   PetscCall(DMPlexGetCone(dm, cell, &cone));
2922   for (v = 0, v2 = 0; v < coneSize; ++v) {
2923     PetscBool found = PETSC_FALSE;
2924 
2925     for (f = 0; f < faceSize; ++f) {
2926       if (face[f] == cone[v]) {
2927         found = PETSC_TRUE; break;
2928       }
2929     }
2930     if (found) {
2931       indices[v2]      = v;
2932       origVertices[v2] = cone[v];
2933       ++v2;
2934     } else {
2935       oppositeVertex = v;
2936     }
2937   }
2938   PetscCall(DMPlexGetFaceOrientation(dm, cell, numCorners, indices, oppositeVertex, origVertices, faceVertices, posOriented));
2939   PetscFunctionReturn(0);
2940 }
2941 
2942 /*
2943   DMPlexInsertFace_Internal - Puts a face into the mesh
2944 
2945   Not collective
2946 
2947   Input Parameters:
2948   + dm              - The DMPlex
2949   . numFaceVertex   - The number of vertices in the face
2950   . faceVertices    - The vertices in the face for dm
2951   . subfaceVertices - The vertices in the face for subdm
2952   . numCorners      - The number of vertices in the cell
2953   . cell            - A cell in dm containing the face
2954   . subcell         - A cell in subdm containing the face
2955   . firstFace       - First face in the mesh
2956   - newFacePoint    - Next face in the mesh
2957 
2958   Output Parameters:
2959   . newFacePoint - Contains next face point number on input, updated on output
2960 
2961   Level: developer
2962 */
2963 static PetscErrorCode DMPlexInsertFace_Internal(DM dm, DM subdm, PetscInt numFaceVertices, const PetscInt faceVertices[], const PetscInt subfaceVertices[], PetscInt numCorners, PetscInt cell, PetscInt subcell, PetscInt firstFace, PetscInt *newFacePoint)
2964 {
2965   MPI_Comm        comm;
2966   DM_Plex        *submesh = (DM_Plex*) subdm->data;
2967   const PetscInt *faces;
2968   PetscInt        numFaces, coneSize;
2969 
2970   PetscFunctionBegin;
2971   PetscCall(PetscObjectGetComm((PetscObject)dm,&comm));
2972   PetscCall(DMPlexGetConeSize(subdm, subcell, &coneSize));
2973   PetscCheck(coneSize == 1,comm, PETSC_ERR_ARG_OUTOFRANGE, "Cone size of cell %" PetscInt_FMT " is %" PetscInt_FMT " != 1", cell, coneSize);
2974 #if 0
2975   /* Cannot use this because support() has not been constructed yet */
2976   PetscCall(DMPlexGetJoin(subdm, numFaceVertices, subfaceVertices, &numFaces, &faces));
2977 #else
2978   {
2979     PetscInt f;
2980 
2981     numFaces = 0;
2982     PetscCall(DMGetWorkArray(subdm, 1, MPIU_INT, (void **) &faces));
2983     for (f = firstFace; f < *newFacePoint; ++f) {
2984       PetscInt dof, off, d;
2985 
2986       PetscCall(PetscSectionGetDof(submesh->coneSection, f, &dof));
2987       PetscCall(PetscSectionGetOffset(submesh->coneSection, f, &off));
2988       /* Yes, I know this is quadratic, but I expect the sizes to be <5 */
2989       for (d = 0; d < dof; ++d) {
2990         const PetscInt p = submesh->cones[off+d];
2991         PetscInt       v;
2992 
2993         for (v = 0; v < numFaceVertices; ++v) {
2994           if (subfaceVertices[v] == p) break;
2995         }
2996         if (v == numFaceVertices) break;
2997       }
2998       if (d == dof) {
2999         numFaces               = 1;
3000         ((PetscInt*) faces)[0] = f;
3001       }
3002     }
3003   }
3004 #endif
3005   PetscCheck(numFaces <= 1,comm, PETSC_ERR_ARG_WRONG, "Vertex set had %" PetscInt_FMT " faces, not one", numFaces);
3006   else if (numFaces == 1) {
3007     /* Add the other cell neighbor for this face */
3008     PetscCall(DMPlexSetCone(subdm, subcell, faces));
3009   } else {
3010     PetscInt *indices, *origVertices, *orientedVertices, *orientedSubVertices, v, ov;
3011     PetscBool posOriented;
3012 
3013     PetscCall(DMGetWorkArray(subdm, 4*numFaceVertices * sizeof(PetscInt), MPIU_INT, &orientedVertices));
3014     origVertices        = &orientedVertices[numFaceVertices];
3015     indices             = &orientedVertices[numFaceVertices*2];
3016     orientedSubVertices = &orientedVertices[numFaceVertices*3];
3017     PetscCall(DMPlexGetOrientedFace(dm, cell, numFaceVertices, faceVertices, numCorners, indices, origVertices, orientedVertices, &posOriented));
3018     /* TODO: I know that routine should return a permutation, not the indices */
3019     for (v = 0; v < numFaceVertices; ++v) {
3020       const PetscInt vertex = faceVertices[v], subvertex = subfaceVertices[v];
3021       for (ov = 0; ov < numFaceVertices; ++ov) {
3022         if (orientedVertices[ov] == vertex) {
3023           orientedSubVertices[ov] = subvertex;
3024           break;
3025         }
3026       }
3027       PetscCheck(ov != numFaceVertices,comm, PETSC_ERR_PLIB, "Could not find face vertex %" PetscInt_FMT " in orientated set", vertex);
3028     }
3029     PetscCall(DMPlexSetCone(subdm, *newFacePoint, orientedSubVertices));
3030     PetscCall(DMPlexSetCone(subdm, subcell, newFacePoint));
3031     PetscCall(DMRestoreWorkArray(subdm, 4*numFaceVertices * sizeof(PetscInt), MPIU_INT, &orientedVertices));
3032     ++(*newFacePoint);
3033   }
3034 #if 0
3035   PetscCall(DMPlexRestoreJoin(subdm, numFaceVertices, subfaceVertices, &numFaces, &faces));
3036 #else
3037   PetscCall(DMRestoreWorkArray(subdm, 1, MPIU_INT, (void **) &faces));
3038 #endif
3039   PetscFunctionReturn(0);
3040 }
3041 
3042 static PetscErrorCode DMPlexCreateSubmesh_Uninterpolated(DM dm, DMLabel vertexLabel, PetscInt value, DM subdm)
3043 {
3044   MPI_Comm        comm;
3045   DMLabel         subpointMap;
3046   IS              subvertexIS,  subcellIS;
3047   const PetscInt *subVertices, *subCells;
3048   PetscInt        numSubVertices, firstSubVertex, numSubCells;
3049   PetscInt       *subface, maxConeSize, numSubFaces = 0, firstSubFace, newFacePoint, nFV = 0;
3050   PetscInt        vStart, vEnd, c, f;
3051 
3052   PetscFunctionBegin;
3053   PetscCall(PetscObjectGetComm((PetscObject)dm,&comm));
3054   /* Create subpointMap which marks the submesh */
3055   PetscCall(DMLabelCreate(PETSC_COMM_SELF, "subpoint_map", &subpointMap));
3056   PetscCall(DMPlexSetSubpointMap(subdm, subpointMap));
3057   PetscCall(DMLabelDestroy(&subpointMap));
3058   if (vertexLabel) PetscCall(DMPlexMarkSubmesh_Uninterpolated(dm, vertexLabel, value, subpointMap, &numSubFaces, &nFV, subdm));
3059   /* Setup chart */
3060   PetscCall(DMLabelGetStratumSize(subpointMap, 0, &numSubVertices));
3061   PetscCall(DMLabelGetStratumSize(subpointMap, 2, &numSubCells));
3062   PetscCall(DMPlexSetChart(subdm, 0, numSubCells+numSubFaces+numSubVertices));
3063   PetscCall(DMPlexSetVTKCellHeight(subdm, 1));
3064   /* Set cone sizes */
3065   firstSubVertex = numSubCells;
3066   firstSubFace   = numSubCells+numSubVertices;
3067   newFacePoint   = firstSubFace;
3068   PetscCall(DMLabelGetStratumIS(subpointMap, 0, &subvertexIS));
3069   if (subvertexIS) PetscCall(ISGetIndices(subvertexIS, &subVertices));
3070   PetscCall(DMLabelGetStratumIS(subpointMap, 2, &subcellIS));
3071   if (subcellIS) PetscCall(ISGetIndices(subcellIS, &subCells));
3072   for (c = 0; c < numSubCells; ++c) {
3073     PetscCall(DMPlexSetConeSize(subdm, c, 1));
3074   }
3075   for (f = firstSubFace; f < firstSubFace+numSubFaces; ++f) {
3076     PetscCall(DMPlexSetConeSize(subdm, f, nFV));
3077   }
3078   PetscCall(DMSetUp(subdm));
3079   /* Create face cones */
3080   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
3081   PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, NULL));
3082   PetscCall(DMGetWorkArray(subdm, maxConeSize, MPIU_INT, (void**) &subface));
3083   for (c = 0; c < numSubCells; ++c) {
3084     const PetscInt cell    = subCells[c];
3085     const PetscInt subcell = c;
3086     PetscInt      *closure = NULL;
3087     PetscInt       closureSize, cl, numCorners = 0, faceSize = 0;
3088 
3089     PetscCall(DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
3090     for (cl = 0; cl < closureSize*2; cl += 2) {
3091       const PetscInt point = closure[cl];
3092       PetscInt       subVertex;
3093 
3094       if ((point >= vStart) && (point < vEnd)) {
3095         ++numCorners;
3096         PetscCall(PetscFindInt(point, numSubVertices, subVertices, &subVertex));
3097         if (subVertex >= 0) {
3098           closure[faceSize] = point;
3099           subface[faceSize] = firstSubVertex+subVertex;
3100           ++faceSize;
3101         }
3102       }
3103     }
3104     PetscCheck(faceSize <= nFV,comm, PETSC_ERR_ARG_WRONG, "Invalid submesh: Too many vertices %" PetscInt_FMT " of an element on the surface", faceSize);
3105     if (faceSize == nFV) {
3106       PetscCall(DMPlexInsertFace_Internal(dm, subdm, faceSize, closure, subface, numCorners, cell, subcell, firstSubFace, &newFacePoint));
3107     }
3108     PetscCall(DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
3109   }
3110   PetscCall(DMRestoreWorkArray(subdm, maxConeSize, MPIU_INT, (void**) &subface));
3111   PetscCall(DMPlexSymmetrize(subdm));
3112   PetscCall(DMPlexStratify(subdm));
3113   /* Build coordinates */
3114   {
3115     PetscSection coordSection, subCoordSection;
3116     Vec          coordinates, subCoordinates;
3117     PetscScalar *coords, *subCoords;
3118     PetscInt     numComp, coordSize, v;
3119     const char  *name;
3120 
3121     PetscCall(DMGetCoordinateSection(dm, &coordSection));
3122     PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
3123     PetscCall(DMGetCoordinateSection(subdm, &subCoordSection));
3124     PetscCall(PetscSectionSetNumFields(subCoordSection, 1));
3125     PetscCall(PetscSectionGetFieldComponents(coordSection, 0, &numComp));
3126     PetscCall(PetscSectionSetFieldComponents(subCoordSection, 0, numComp));
3127     PetscCall(PetscSectionSetChart(subCoordSection, firstSubVertex, firstSubVertex+numSubVertices));
3128     for (v = 0; v < numSubVertices; ++v) {
3129       const PetscInt vertex    = subVertices[v];
3130       const PetscInt subvertex = firstSubVertex+v;
3131       PetscInt       dof;
3132 
3133       PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3134       PetscCall(PetscSectionSetDof(subCoordSection, subvertex, dof));
3135       PetscCall(PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof));
3136     }
3137     PetscCall(PetscSectionSetUp(subCoordSection));
3138     PetscCall(PetscSectionGetStorageSize(subCoordSection, &coordSize));
3139     PetscCall(VecCreate(PETSC_COMM_SELF, &subCoordinates));
3140     PetscCall(PetscObjectGetName((PetscObject)coordinates,&name));
3141     PetscCall(PetscObjectSetName((PetscObject)subCoordinates,name));
3142     PetscCall(VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE));
3143     PetscCall(VecSetType(subCoordinates,VECSTANDARD));
3144     if (coordSize) {
3145       PetscCall(VecGetArray(coordinates,    &coords));
3146       PetscCall(VecGetArray(subCoordinates, &subCoords));
3147       for (v = 0; v < numSubVertices; ++v) {
3148         const PetscInt vertex    = subVertices[v];
3149         const PetscInt subvertex = firstSubVertex+v;
3150         PetscInt       dof, off, sdof, soff, d;
3151 
3152         PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3153         PetscCall(PetscSectionGetOffset(coordSection, vertex, &off));
3154         PetscCall(PetscSectionGetDof(subCoordSection, subvertex, &sdof));
3155         PetscCall(PetscSectionGetOffset(subCoordSection, subvertex, &soff));
3156         PetscCheck(dof == sdof,comm, PETSC_ERR_PLIB, "Coordinate dimension %" PetscInt_FMT " on subvertex %" PetscInt_FMT ", vertex %" PetscInt_FMT " should be %" PetscInt_FMT, sdof, subvertex, vertex, dof);
3157         for (d = 0; d < dof; ++d) subCoords[soff+d] = coords[off+d];
3158       }
3159       PetscCall(VecRestoreArray(coordinates,    &coords));
3160       PetscCall(VecRestoreArray(subCoordinates, &subCoords));
3161     }
3162     PetscCall(DMSetCoordinatesLocal(subdm, subCoordinates));
3163     PetscCall(VecDestroy(&subCoordinates));
3164   }
3165   /* Cleanup */
3166   if (subvertexIS) PetscCall(ISRestoreIndices(subvertexIS, &subVertices));
3167   PetscCall(ISDestroy(&subvertexIS));
3168   if (subcellIS) PetscCall(ISRestoreIndices(subcellIS, &subCells));
3169   PetscCall(ISDestroy(&subcellIS));
3170   PetscFunctionReturn(0);
3171 }
3172 
3173 /* TODO: Fix this to properly propogate up error conditions it may find */
3174 static inline PetscInt DMPlexFilterPoint_Internal(PetscInt point, PetscInt firstSubPoint, PetscInt numSubPoints, const PetscInt subPoints[])
3175 {
3176   PetscInt       subPoint;
3177   PetscErrorCode ierr;
3178 
3179   ierr = PetscFindInt(point, numSubPoints, subPoints, &subPoint); if (ierr) return -1;
3180   return subPoint < 0 ? subPoint : firstSubPoint+subPoint;
3181 }
3182 
3183 static PetscErrorCode DMPlexFilterLabels_Internal(DM dm, const PetscInt numSubPoints[], const PetscInt *subpoints[], const PetscInt firstSubPoint[], DM subdm)
3184 {
3185   PetscInt       Nl, l, d;
3186 
3187   PetscFunctionBegin;
3188   PetscCall(DMGetNumLabels(dm, &Nl));
3189   for (l = 0; l < Nl; ++l) {
3190     DMLabel         label, newlabel;
3191     const char     *lname;
3192     PetscBool       isDepth, isDim, isCelltype, isVTK;
3193     IS              valueIS;
3194     const PetscInt *values;
3195     PetscInt        Nv, v;
3196 
3197     PetscCall(DMGetLabelName(dm, l, &lname));
3198     PetscCall(PetscStrcmp(lname, "depth", &isDepth));
3199     PetscCall(PetscStrcmp(lname, "dim", &isDim));
3200     PetscCall(PetscStrcmp(lname, "celltype", &isCelltype));
3201     PetscCall(PetscStrcmp(lname, "vtk", &isVTK));
3202     if (isDepth || isDim || isCelltype || isVTK) continue;
3203     PetscCall(DMCreateLabel(subdm, lname));
3204     PetscCall(DMGetLabel(dm, lname, &label));
3205     PetscCall(DMGetLabel(subdm, lname, &newlabel));
3206     PetscCall(DMLabelGetDefaultValue(label, &v));
3207     PetscCall(DMLabelSetDefaultValue(newlabel, v));
3208     PetscCall(DMLabelGetValueIS(label, &valueIS));
3209     PetscCall(ISGetLocalSize(valueIS, &Nv));
3210     PetscCall(ISGetIndices(valueIS, &values));
3211     for (v = 0; v < Nv; ++v) {
3212       IS              pointIS;
3213       const PetscInt *points;
3214       PetscInt        Np, p;
3215 
3216       PetscCall(DMLabelGetStratumIS(label, values[v], &pointIS));
3217       PetscCall(ISGetLocalSize(pointIS, &Np));
3218       PetscCall(ISGetIndices(pointIS, &points));
3219       for (p = 0; p < Np; ++p) {
3220         const PetscInt point = points[p];
3221         PetscInt       subp;
3222 
3223         PetscCall(DMPlexGetPointDepth(dm, point, &d));
3224         subp = DMPlexFilterPoint_Internal(point, firstSubPoint[d], numSubPoints[d], subpoints[d]);
3225         if (subp >= 0) PetscCall(DMLabelSetValue(newlabel, subp, values[v]));
3226       }
3227       PetscCall(ISRestoreIndices(pointIS, &points));
3228       PetscCall(ISDestroy(&pointIS));
3229     }
3230     PetscCall(ISRestoreIndices(valueIS, &values));
3231     PetscCall(ISDestroy(&valueIS));
3232   }
3233   PetscFunctionReturn(0);
3234 }
3235 
3236 static PetscErrorCode DMPlexCreateSubmeshGeneric_Interpolated(DM dm, DMLabel label, PetscInt value, PetscBool markedFaces, PetscBool isCohesive, PetscInt cellHeight, DM subdm)
3237 {
3238   MPI_Comm         comm;
3239   DMLabel          subpointMap;
3240   IS              *subpointIS;
3241   const PetscInt **subpoints;
3242   PetscInt        *numSubPoints, *firstSubPoint, *coneNew, *orntNew;
3243   PetscInt         totSubPoints = 0, maxConeSize, dim, p, d, v;
3244   PetscMPIInt      rank;
3245 
3246   PetscFunctionBegin;
3247   PetscCall(PetscObjectGetComm((PetscObject)dm,&comm));
3248   PetscCallMPI(MPI_Comm_rank(comm, &rank));
3249   /* Create subpointMap which marks the submesh */
3250   PetscCall(DMLabelCreate(PETSC_COMM_SELF, "subpoint_map", &subpointMap));
3251   PetscCall(DMPlexSetSubpointMap(subdm, subpointMap));
3252   if (cellHeight) {
3253     if (isCohesive) PetscCall(DMPlexMarkCohesiveSubmesh_Interpolated(dm, label, value, subpointMap, subdm));
3254     else            PetscCall(DMPlexMarkSubmesh_Interpolated(dm, label, value, markedFaces, subpointMap, subdm));
3255   } else {
3256     DMLabel         depth;
3257     IS              pointIS;
3258     const PetscInt *points;
3259     PetscInt        numPoints=0;
3260 
3261     PetscCall(DMPlexGetDepthLabel(dm, &depth));
3262     PetscCall(DMLabelGetStratumIS(label, value, &pointIS));
3263     if (pointIS) {
3264       PetscCall(ISGetIndices(pointIS, &points));
3265       PetscCall(ISGetLocalSize(pointIS, &numPoints));
3266     }
3267     for (p = 0; p < numPoints; ++p) {
3268       PetscInt *closure = NULL;
3269       PetscInt  closureSize, c, pdim;
3270 
3271       PetscCall(DMPlexGetTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closure));
3272       for (c = 0; c < closureSize*2; c += 2) {
3273         PetscCall(DMLabelGetValue(depth, closure[c], &pdim));
3274         PetscCall(DMLabelSetValue(subpointMap, closure[c], pdim));
3275       }
3276       PetscCall(DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closure));
3277     }
3278     if (pointIS) PetscCall(ISRestoreIndices(pointIS, &points));
3279     PetscCall(ISDestroy(&pointIS));
3280   }
3281   /* Setup chart */
3282   PetscCall(DMGetDimension(dm, &dim));
3283   PetscCall(PetscMalloc4(dim+1,&numSubPoints,dim+1,&firstSubPoint,dim+1,&subpointIS,dim+1,&subpoints));
3284   for (d = 0; d <= dim; ++d) {
3285     PetscCall(DMLabelGetStratumSize(subpointMap, d, &numSubPoints[d]));
3286     totSubPoints += numSubPoints[d];
3287   }
3288   PetscCall(DMPlexSetChart(subdm, 0, totSubPoints));
3289   PetscCall(DMPlexSetVTKCellHeight(subdm, cellHeight));
3290   /* Set cone sizes */
3291   firstSubPoint[dim] = 0;
3292   firstSubPoint[0]   = firstSubPoint[dim] + numSubPoints[dim];
3293   if (dim > 1) {firstSubPoint[dim-1] = firstSubPoint[0]     + numSubPoints[0];}
3294   if (dim > 2) {firstSubPoint[dim-2] = firstSubPoint[dim-1] + numSubPoints[dim-1];}
3295   for (d = 0; d <= dim; ++d) {
3296     PetscCall(DMLabelGetStratumIS(subpointMap, d, &subpointIS[d]));
3297     if (subpointIS[d]) PetscCall(ISGetIndices(subpointIS[d], &subpoints[d]));
3298   }
3299   /* We do not want this label automatically computed, instead we compute it here */
3300   PetscCall(DMCreateLabel(subdm, "celltype"));
3301   for (d = 0; d <= dim; ++d) {
3302     for (p = 0; p < numSubPoints[d]; ++p) {
3303       const PetscInt  point    = subpoints[d][p];
3304       const PetscInt  subpoint = firstSubPoint[d] + p;
3305       const PetscInt *cone;
3306       PetscInt        coneSize;
3307 
3308       PetscCall(DMPlexGetConeSize(dm, point, &coneSize));
3309       if (cellHeight && (d == dim)) {
3310         PetscInt coneSizeNew, c, val;
3311 
3312         PetscCall(DMPlexGetCone(dm, point, &cone));
3313         for (c = 0, coneSizeNew = 0; c < coneSize; ++c) {
3314           PetscCall(DMLabelGetValue(subpointMap, cone[c], &val));
3315           if (val >= 0) coneSizeNew++;
3316         }
3317         PetscCall(DMPlexSetConeSize(subdm, subpoint, coneSizeNew));
3318         PetscCall(DMPlexSetCellType(subdm, subpoint, DM_POLYTOPE_FV_GHOST));
3319       } else {
3320         DMPolytopeType  ct;
3321 
3322         PetscCall(DMPlexSetConeSize(subdm, subpoint, coneSize));
3323         PetscCall(DMPlexGetCellType(dm, point, &ct));
3324         PetscCall(DMPlexSetCellType(subdm, subpoint, ct));
3325       }
3326     }
3327   }
3328   PetscCall(DMLabelDestroy(&subpointMap));
3329   PetscCall(DMSetUp(subdm));
3330   /* Set cones */
3331   PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, NULL));
3332   PetscCall(PetscMalloc2(maxConeSize,&coneNew,maxConeSize,&orntNew));
3333   for (d = 0; d <= dim; ++d) {
3334     for (p = 0; p < numSubPoints[d]; ++p) {
3335       const PetscInt  point    = subpoints[d][p];
3336       const PetscInt  subpoint = firstSubPoint[d] + p;
3337       const PetscInt *cone, *ornt;
3338       PetscInt        coneSize, subconeSize, coneSizeNew, c, subc, fornt = 0;
3339 
3340       if (d == dim-1) {
3341         const PetscInt *support, *cone, *ornt;
3342         PetscInt        supportSize, coneSize, s, subc;
3343 
3344         PetscCall(DMPlexGetSupport(dm, point, &support));
3345         PetscCall(DMPlexGetSupportSize(dm, point, &supportSize));
3346         for (s = 0; s < supportSize; ++s) {
3347           PetscBool isHybrid;
3348 
3349           PetscCall(DMPlexCellIsHybrid_Internal(dm, support[s], &isHybrid));
3350           if (!isHybrid) continue;
3351           PetscCall(PetscFindInt(support[s], numSubPoints[d+1], subpoints[d+1], &subc));
3352           if (subc >= 0) {
3353             const PetscInt ccell = subpoints[d+1][subc];
3354 
3355             PetscCall(DMPlexGetCone(dm, ccell, &cone));
3356             PetscCall(DMPlexGetConeSize(dm, ccell, &coneSize));
3357             PetscCall(DMPlexGetConeOrientation(dm, ccell, &ornt));
3358             for (c = 0; c < coneSize; ++c) {
3359               if (cone[c] == point) {
3360                 fornt = ornt[c];
3361                 break;
3362               }
3363             }
3364             break;
3365           }
3366         }
3367       }
3368       PetscCall(DMPlexGetConeSize(dm, point, &coneSize));
3369       PetscCall(DMPlexGetConeSize(subdm, subpoint, &subconeSize));
3370       PetscCall(DMPlexGetCone(dm, point, &cone));
3371       PetscCall(DMPlexGetConeOrientation(dm, point, &ornt));
3372       for (c = 0, coneSizeNew = 0; c < coneSize; ++c) {
3373         PetscCall(PetscFindInt(cone[c], numSubPoints[d-1], subpoints[d-1], &subc));
3374         if (subc >= 0) {
3375           coneNew[coneSizeNew] = firstSubPoint[d-1] + subc;
3376           orntNew[coneSizeNew] = ornt[c];
3377           ++coneSizeNew;
3378         }
3379       }
3380       PetscCheck(coneSizeNew == subconeSize,comm, PETSC_ERR_PLIB, "Number of cone points located %" PetscInt_FMT " does not match subcone size %" PetscInt_FMT, coneSizeNew, subconeSize);
3381       PetscCall(DMPlexSetCone(subdm, subpoint, coneNew));
3382       PetscCall(DMPlexSetConeOrientation(subdm, subpoint, orntNew));
3383       if (fornt < 0) PetscCall(DMPlexOrientPoint(subdm, subpoint, fornt));
3384     }
3385   }
3386   PetscCall(PetscFree2(coneNew,orntNew));
3387   PetscCall(DMPlexSymmetrize(subdm));
3388   PetscCall(DMPlexStratify(subdm));
3389   /* Build coordinates */
3390   {
3391     PetscSection coordSection, subCoordSection;
3392     Vec          coordinates, subCoordinates;
3393     PetscScalar *coords, *subCoords;
3394     PetscInt     cdim, numComp, coordSize;
3395     const char  *name;
3396 
3397     PetscCall(DMGetCoordinateDim(dm, &cdim));
3398     PetscCall(DMGetCoordinateSection(dm, &coordSection));
3399     PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
3400     PetscCall(DMGetCoordinateSection(subdm, &subCoordSection));
3401     PetscCall(PetscSectionSetNumFields(subCoordSection, 1));
3402     PetscCall(PetscSectionGetFieldComponents(coordSection, 0, &numComp));
3403     PetscCall(PetscSectionSetFieldComponents(subCoordSection, 0, numComp));
3404     PetscCall(PetscSectionSetChart(subCoordSection, firstSubPoint[0], firstSubPoint[0]+numSubPoints[0]));
3405     for (v = 0; v < numSubPoints[0]; ++v) {
3406       const PetscInt vertex    = subpoints[0][v];
3407       const PetscInt subvertex = firstSubPoint[0]+v;
3408       PetscInt       dof;
3409 
3410       PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3411       PetscCall(PetscSectionSetDof(subCoordSection, subvertex, dof));
3412       PetscCall(PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof));
3413     }
3414     PetscCall(PetscSectionSetUp(subCoordSection));
3415     PetscCall(PetscSectionGetStorageSize(subCoordSection, &coordSize));
3416     PetscCall(VecCreate(PETSC_COMM_SELF, &subCoordinates));
3417     PetscCall(PetscObjectGetName((PetscObject)coordinates,&name));
3418     PetscCall(PetscObjectSetName((PetscObject)subCoordinates,name));
3419     PetscCall(VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE));
3420     PetscCall(VecSetBlockSize(subCoordinates, cdim));
3421     PetscCall(VecSetType(subCoordinates,VECSTANDARD));
3422     PetscCall(VecGetArray(coordinates,    &coords));
3423     PetscCall(VecGetArray(subCoordinates, &subCoords));
3424     for (v = 0; v < numSubPoints[0]; ++v) {
3425       const PetscInt vertex    = subpoints[0][v];
3426       const PetscInt subvertex = firstSubPoint[0]+v;
3427       PetscInt dof, off, sdof, soff, d;
3428 
3429       PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3430       PetscCall(PetscSectionGetOffset(coordSection, vertex, &off));
3431       PetscCall(PetscSectionGetDof(subCoordSection, subvertex, &sdof));
3432       PetscCall(PetscSectionGetOffset(subCoordSection, subvertex, &soff));
3433       PetscCheck(dof == sdof,comm, PETSC_ERR_PLIB, "Coordinate dimension %" PetscInt_FMT " on subvertex %" PetscInt_FMT ", vertex %" PetscInt_FMT " should be %" PetscInt_FMT, sdof, subvertex, vertex, dof);
3434       for (d = 0; d < dof; ++d) subCoords[soff+d] = coords[off+d];
3435     }
3436     PetscCall(VecRestoreArray(coordinates,    &coords));
3437     PetscCall(VecRestoreArray(subCoordinates, &subCoords));
3438     PetscCall(DMSetCoordinatesLocal(subdm, subCoordinates));
3439     PetscCall(VecDestroy(&subCoordinates));
3440   }
3441   /* Build SF: We need this complexity because subpoints might not be selected on the owning process */
3442   {
3443     PetscSF            sfPoint, sfPointSub;
3444     IS                 subpIS;
3445     const PetscSFNode *remotePoints;
3446     PetscSFNode       *sremotePoints, *newLocalPoints, *newOwners;
3447     const PetscInt    *localPoints, *subpoints;
3448     PetscInt          *slocalPoints;
3449     PetscInt           numRoots, numLeaves, numSubpoints = 0, numSubroots, numSubleaves = 0, l, sl, ll, pStart, pEnd, p;
3450     PetscMPIInt        rank;
3451 
3452     PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank));
3453     PetscCall(DMGetPointSF(dm, &sfPoint));
3454     PetscCall(DMGetPointSF(subdm, &sfPointSub));
3455     PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
3456     PetscCall(DMPlexGetChart(subdm, NULL, &numSubroots));
3457     PetscCall(DMPlexGetSubpointIS(subdm, &subpIS));
3458     if (subpIS) {
3459       PetscCall(ISGetIndices(subpIS, &subpoints));
3460       PetscCall(ISGetLocalSize(subpIS, &numSubpoints));
3461     }
3462     PetscCall(PetscSFGetGraph(sfPoint, &numRoots, &numLeaves, &localPoints, &remotePoints));
3463     if (numRoots >= 0) {
3464       PetscCall(PetscMalloc2(pEnd-pStart,&newLocalPoints,numRoots,&newOwners));
3465       for (p = 0; p < pEnd-pStart; ++p) {
3466         newLocalPoints[p].rank  = -2;
3467         newLocalPoints[p].index = -2;
3468       }
3469       /* Set subleaves */
3470       for (l = 0; l < numLeaves; ++l) {
3471         const PetscInt point    = localPoints[l];
3472         const PetscInt subpoint = DMPlexFilterPoint_Internal(point, 0, numSubpoints, subpoints);
3473 
3474         if (subpoint < 0) continue;
3475         newLocalPoints[point-pStart].rank  = rank;
3476         newLocalPoints[point-pStart].index = subpoint;
3477         ++numSubleaves;
3478       }
3479       /* Must put in owned subpoints */
3480       for (p = pStart; p < pEnd; ++p) {
3481         const PetscInt subpoint = DMPlexFilterPoint_Internal(p, 0, numSubpoints, subpoints);
3482 
3483         if (subpoint < 0) {
3484           newOwners[p-pStart].rank  = -3;
3485           newOwners[p-pStart].index = -3;
3486         } else {
3487           newOwners[p-pStart].rank  = rank;
3488           newOwners[p-pStart].index = subpoint;
3489         }
3490       }
3491       PetscCall(PetscSFReduceBegin(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC));
3492       PetscCall(PetscSFReduceEnd(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC));
3493       PetscCall(PetscSFBcastBegin(sfPoint, MPIU_2INT, newOwners, newLocalPoints,MPI_REPLACE));
3494       PetscCall(PetscSFBcastEnd(sfPoint, MPIU_2INT, newOwners, newLocalPoints,MPI_REPLACE));
3495       PetscCall(PetscMalloc1(numSubleaves, &slocalPoints));
3496       PetscCall(PetscMalloc1(numSubleaves, &sremotePoints));
3497       for (l = 0, sl = 0, ll = 0; l < numLeaves; ++l) {
3498         const PetscInt point    = localPoints[l];
3499         const PetscInt subpoint = DMPlexFilterPoint_Internal(point, 0, numSubpoints, subpoints);
3500 
3501         if (subpoint < 0) continue;
3502         if (newLocalPoints[point].rank == rank) {++ll; continue;}
3503         slocalPoints[sl]        = subpoint;
3504         sremotePoints[sl].rank  = newLocalPoints[point].rank;
3505         sremotePoints[sl].index = newLocalPoints[point].index;
3506         PetscCheck(sremotePoints[sl].rank  >= 0,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote rank for local point %" PetscInt_FMT, point);
3507         PetscCheck(sremotePoints[sl].index >= 0,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote subpoint for local point %" PetscInt_FMT, point);
3508         ++sl;
3509       }
3510       PetscCheck(sl + ll == numSubleaves,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatch in number of subleaves %" PetscInt_FMT " + %" PetscInt_FMT " != %" PetscInt_FMT, sl, ll, numSubleaves);
3511       PetscCall(PetscFree2(newLocalPoints,newOwners));
3512       PetscCall(PetscSFSetGraph(sfPointSub, numSubroots, sl, slocalPoints, PETSC_OWN_POINTER, sremotePoints, PETSC_OWN_POINTER));
3513     }
3514     if (subpIS) {
3515       PetscCall(ISRestoreIndices(subpIS, &subpoints));
3516     }
3517   }
3518   /* Filter labels */
3519   PetscCall(DMPlexFilterLabels_Internal(dm, numSubPoints, subpoints, firstSubPoint, subdm));
3520   /* Cleanup */
3521   for (d = 0; d <= dim; ++d) {
3522     if (subpointIS[d]) PetscCall(ISRestoreIndices(subpointIS[d], &subpoints[d]));
3523     PetscCall(ISDestroy(&subpointIS[d]));
3524   }
3525   PetscCall(PetscFree4(numSubPoints,firstSubPoint,subpointIS,subpoints));
3526   PetscFunctionReturn(0);
3527 }
3528 
3529 static PetscErrorCode DMPlexCreateSubmesh_Interpolated(DM dm, DMLabel vertexLabel, PetscInt value, PetscBool markedFaces, DM subdm)
3530 {
3531   PetscFunctionBegin;
3532   PetscCall(DMPlexCreateSubmeshGeneric_Interpolated(dm, vertexLabel, value, markedFaces, PETSC_FALSE, 1, subdm));
3533   PetscFunctionReturn(0);
3534 }
3535 
3536 /*@
3537   DMPlexCreateSubmesh - Extract a hypersurface from the mesh using vertices defined by a label
3538 
3539   Input Parameters:
3540 + dm           - The original mesh
3541 . vertexLabel  - The DMLabel marking points contained in the surface
3542 . value        - The label value to use
3543 - markedFaces  - PETSC_TRUE if surface faces are marked in addition to vertices, PETSC_FALSE if only vertices are marked
3544 
3545   Output Parameter:
3546 . subdm - The surface mesh
3547 
3548   Note: This function produces a DMLabel mapping original points in the submesh to their depth. This can be obtained using DMPlexGetSubpointMap().
3549 
3550   Level: developer
3551 
3552 .seealso: `DMPlexGetSubpointMap()`, `DMGetLabel()`, `DMLabelSetValue()`
3553 @*/
3554 PetscErrorCode DMPlexCreateSubmesh(DM dm, DMLabel vertexLabel, PetscInt value, PetscBool markedFaces, DM *subdm)
3555 {
3556   DMPlexInterpolatedFlag interpolated;
3557   PetscInt       dim, cdim;
3558 
3559   PetscFunctionBegin;
3560   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3561   PetscValidPointer(subdm, 5);
3562   PetscCall(DMGetDimension(dm, &dim));
3563   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), subdm));
3564   PetscCall(DMSetType(*subdm, DMPLEX));
3565   PetscCall(DMSetDimension(*subdm, dim-1));
3566   PetscCall(DMGetCoordinateDim(dm, &cdim));
3567   PetscCall(DMSetCoordinateDim(*subdm, cdim));
3568   PetscCall(DMPlexIsInterpolated(dm, &interpolated));
3569   PetscCheck(interpolated != DMPLEX_INTERPOLATED_PARTIAL,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Not for partially interpolated meshes");
3570   if (interpolated) {
3571     PetscCall(DMPlexCreateSubmesh_Interpolated(dm, vertexLabel, value, markedFaces, *subdm));
3572   } else {
3573     PetscCall(DMPlexCreateSubmesh_Uninterpolated(dm, vertexLabel, value, *subdm));
3574   }
3575   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_TRUE, *subdm));
3576   PetscFunctionReturn(0);
3577 }
3578 
3579 static PetscErrorCode DMPlexCreateCohesiveSubmesh_Uninterpolated(DM dm, PetscBool hasLagrange, const char label[], PetscInt value, DM subdm)
3580 {
3581   MPI_Comm        comm;
3582   DMLabel         subpointMap;
3583   IS              subvertexIS;
3584   const PetscInt *subVertices;
3585   PetscInt        numSubVertices, firstSubVertex, numSubCells, *subCells = NULL;
3586   PetscInt       *subface, maxConeSize, numSubFaces, firstSubFace, newFacePoint, nFV;
3587   PetscInt        c, f;
3588 
3589   PetscFunctionBegin;
3590   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
3591   /* Create subpointMap which marks the submesh */
3592   PetscCall(DMLabelCreate(PETSC_COMM_SELF, "subpoint_map", &subpointMap));
3593   PetscCall(DMPlexSetSubpointMap(subdm, subpointMap));
3594   PetscCall(DMLabelDestroy(&subpointMap));
3595   PetscCall(DMPlexMarkCohesiveSubmesh_Uninterpolated(dm, hasLagrange, label, value, subpointMap, &numSubFaces, &nFV, &subCells, subdm));
3596   /* Setup chart */
3597   PetscCall(DMLabelGetStratumSize(subpointMap, 0, &numSubVertices));
3598   PetscCall(DMLabelGetStratumSize(subpointMap, 2, &numSubCells));
3599   PetscCall(DMPlexSetChart(subdm, 0, numSubCells+numSubFaces+numSubVertices));
3600   PetscCall(DMPlexSetVTKCellHeight(subdm, 1));
3601   /* Set cone sizes */
3602   firstSubVertex = numSubCells;
3603   firstSubFace   = numSubCells+numSubVertices;
3604   newFacePoint   = firstSubFace;
3605   PetscCall(DMLabelGetStratumIS(subpointMap, 0, &subvertexIS));
3606   if (subvertexIS) PetscCall(ISGetIndices(subvertexIS, &subVertices));
3607   for (c = 0; c < numSubCells; ++c) {
3608     PetscCall(DMPlexSetConeSize(subdm, c, 1));
3609   }
3610   for (f = firstSubFace; f < firstSubFace+numSubFaces; ++f) {
3611     PetscCall(DMPlexSetConeSize(subdm, f, nFV));
3612   }
3613   PetscCall(DMSetUp(subdm));
3614   /* Create face cones */
3615   PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, NULL));
3616   PetscCall(DMGetWorkArray(subdm, maxConeSize, MPIU_INT, (void**) &subface));
3617   for (c = 0; c < numSubCells; ++c) {
3618     const PetscInt  cell    = subCells[c];
3619     const PetscInt  subcell = c;
3620     const PetscInt *cone, *cells;
3621     PetscBool       isHybrid;
3622     PetscInt        numCells, subVertex, p, v;
3623 
3624     PetscCall(DMPlexCellIsHybrid_Internal(dm, cell, &isHybrid));
3625     if (!isHybrid) continue;
3626     PetscCall(DMPlexGetCone(dm, cell, &cone));
3627     for (v = 0; v < nFV; ++v) {
3628       PetscCall(PetscFindInt(cone[v], numSubVertices, subVertices, &subVertex));
3629       subface[v] = firstSubVertex+subVertex;
3630     }
3631     PetscCall(DMPlexSetCone(subdm, newFacePoint, subface));
3632     PetscCall(DMPlexSetCone(subdm, subcell, &newFacePoint));
3633     PetscCall(DMPlexGetJoin(dm, nFV, cone, &numCells, &cells));
3634     /* Not true in parallel
3635     PetscCheck(numCells == 2,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive cells should separate two cells"); */
3636     for (p = 0; p < numCells; ++p) {
3637       PetscInt  negsubcell;
3638       PetscBool isHybrid;
3639 
3640       PetscCall(DMPlexCellIsHybrid_Internal(dm, cells[p], &isHybrid));
3641       if (isHybrid) continue;
3642       /* I know this is a crap search */
3643       for (negsubcell = 0; negsubcell < numSubCells; ++negsubcell) {
3644         if (subCells[negsubcell] == cells[p]) break;
3645       }
3646       PetscCheck(negsubcell != numSubCells,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not find negative face neighbor for cohesive cell %" PetscInt_FMT, cell);
3647       PetscCall(DMPlexSetCone(subdm, negsubcell, &newFacePoint));
3648     }
3649     PetscCall(DMPlexRestoreJoin(dm, nFV, cone, &numCells, &cells));
3650     ++newFacePoint;
3651   }
3652   PetscCall(DMRestoreWorkArray(subdm, maxConeSize, MPIU_INT, (void**) &subface));
3653   PetscCall(DMPlexSymmetrize(subdm));
3654   PetscCall(DMPlexStratify(subdm));
3655   /* Build coordinates */
3656   {
3657     PetscSection coordSection, subCoordSection;
3658     Vec          coordinates, subCoordinates;
3659     PetscScalar *coords, *subCoords;
3660     PetscInt     cdim, numComp, coordSize, v;
3661     const char  *name;
3662 
3663     PetscCall(DMGetCoordinateDim(dm, &cdim));
3664     PetscCall(DMGetCoordinateSection(dm, &coordSection));
3665     PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
3666     PetscCall(DMGetCoordinateSection(subdm, &subCoordSection));
3667     PetscCall(PetscSectionSetNumFields(subCoordSection, 1));
3668     PetscCall(PetscSectionGetFieldComponents(coordSection, 0, &numComp));
3669     PetscCall(PetscSectionSetFieldComponents(subCoordSection, 0, numComp));
3670     PetscCall(PetscSectionSetChart(subCoordSection, firstSubVertex, firstSubVertex+numSubVertices));
3671     for (v = 0; v < numSubVertices; ++v) {
3672       const PetscInt vertex    = subVertices[v];
3673       const PetscInt subvertex = firstSubVertex+v;
3674       PetscInt       dof;
3675 
3676       PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3677       PetscCall(PetscSectionSetDof(subCoordSection, subvertex, dof));
3678       PetscCall(PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof));
3679     }
3680     PetscCall(PetscSectionSetUp(subCoordSection));
3681     PetscCall(PetscSectionGetStorageSize(subCoordSection, &coordSize));
3682     PetscCall(VecCreate(PETSC_COMM_SELF, &subCoordinates));
3683     PetscCall(PetscObjectGetName((PetscObject)coordinates,&name));
3684     PetscCall(PetscObjectSetName((PetscObject)subCoordinates,name));
3685     PetscCall(VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE));
3686     PetscCall(VecSetBlockSize(subCoordinates, cdim));
3687     PetscCall(VecSetType(subCoordinates,VECSTANDARD));
3688     PetscCall(VecGetArray(coordinates,    &coords));
3689     PetscCall(VecGetArray(subCoordinates, &subCoords));
3690     for (v = 0; v < numSubVertices; ++v) {
3691       const PetscInt vertex    = subVertices[v];
3692       const PetscInt subvertex = firstSubVertex+v;
3693       PetscInt       dof, off, sdof, soff, d;
3694 
3695       PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3696       PetscCall(PetscSectionGetOffset(coordSection, vertex, &off));
3697       PetscCall(PetscSectionGetDof(subCoordSection, subvertex, &sdof));
3698       PetscCall(PetscSectionGetOffset(subCoordSection, subvertex, &soff));
3699       PetscCheck(dof == sdof,comm, PETSC_ERR_PLIB, "Coordinate dimension %" PetscInt_FMT " on subvertex %" PetscInt_FMT ", vertex %" PetscInt_FMT " should be %" PetscInt_FMT, sdof, subvertex, vertex, dof);
3700       for (d = 0; d < dof; ++d) subCoords[soff+d] = coords[off+d];
3701     }
3702     PetscCall(VecRestoreArray(coordinates,    &coords));
3703     PetscCall(VecRestoreArray(subCoordinates, &subCoords));
3704     PetscCall(DMSetCoordinatesLocal(subdm, subCoordinates));
3705     PetscCall(VecDestroy(&subCoordinates));
3706   }
3707   /* Build SF */
3708   CHKMEMQ;
3709   {
3710     PetscSF            sfPoint, sfPointSub;
3711     const PetscSFNode *remotePoints;
3712     PetscSFNode       *sremotePoints, *newLocalPoints, *newOwners;
3713     const PetscInt    *localPoints;
3714     PetscInt          *slocalPoints;
3715     PetscInt           numRoots, numLeaves, numSubRoots = numSubCells+numSubFaces+numSubVertices, numSubLeaves = 0, l, sl, ll, pStart, pEnd, p, vStart, vEnd;
3716     PetscMPIInt        rank;
3717 
3718     PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank));
3719     PetscCall(DMGetPointSF(dm, &sfPoint));
3720     PetscCall(DMGetPointSF(subdm, &sfPointSub));
3721     PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
3722     PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
3723     PetscCall(PetscSFGetGraph(sfPoint, &numRoots, &numLeaves, &localPoints, &remotePoints));
3724     if (numRoots >= 0) {
3725       /* Only vertices should be shared */
3726       PetscCall(PetscMalloc2(pEnd-pStart,&newLocalPoints,numRoots,&newOwners));
3727       for (p = 0; p < pEnd-pStart; ++p) {
3728         newLocalPoints[p].rank  = -2;
3729         newLocalPoints[p].index = -2;
3730       }
3731       /* Set subleaves */
3732       for (l = 0; l < numLeaves; ++l) {
3733         const PetscInt point    = localPoints[l];
3734         const PetscInt subPoint = DMPlexFilterPoint_Internal(point, firstSubVertex, numSubVertices, subVertices);
3735 
3736         PetscCheck(!(point < vStart) || !(point >= vEnd),PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Should not be mapping anything but vertices, %" PetscInt_FMT, point);
3737         if (subPoint < 0) continue;
3738         newLocalPoints[point-pStart].rank  = rank;
3739         newLocalPoints[point-pStart].index = subPoint;
3740         ++numSubLeaves;
3741       }
3742       /* Must put in owned subpoints */
3743       for (p = pStart; p < pEnd; ++p) {
3744         const PetscInt subPoint = DMPlexFilterPoint_Internal(p, firstSubVertex, numSubVertices, subVertices);
3745 
3746         if (subPoint < 0) {
3747           newOwners[p-pStart].rank  = -3;
3748           newOwners[p-pStart].index = -3;
3749         } else {
3750           newOwners[p-pStart].rank  = rank;
3751           newOwners[p-pStart].index = subPoint;
3752         }
3753       }
3754       PetscCall(PetscSFReduceBegin(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC));
3755       PetscCall(PetscSFReduceEnd(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC));
3756       PetscCall(PetscSFBcastBegin(sfPoint, MPIU_2INT, newOwners, newLocalPoints,MPI_REPLACE));
3757       PetscCall(PetscSFBcastEnd(sfPoint, MPIU_2INT, newOwners, newLocalPoints,MPI_REPLACE));
3758       PetscCall(PetscMalloc1(numSubLeaves,    &slocalPoints));
3759       PetscCall(PetscMalloc1(numSubLeaves, &sremotePoints));
3760       for (l = 0, sl = 0, ll = 0; l < numLeaves; ++l) {
3761         const PetscInt point    = localPoints[l];
3762         const PetscInt subPoint = DMPlexFilterPoint_Internal(point, firstSubVertex, numSubVertices, subVertices);
3763 
3764         if (subPoint < 0) continue;
3765         if (newLocalPoints[point].rank == rank) {++ll; continue;}
3766         slocalPoints[sl]        = subPoint;
3767         sremotePoints[sl].rank  = newLocalPoints[point].rank;
3768         sremotePoints[sl].index = newLocalPoints[point].index;
3769         PetscCheck(sremotePoints[sl].rank  >= 0,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote rank for local point %" PetscInt_FMT, point);
3770         PetscCheck(sremotePoints[sl].index >= 0,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote subpoint for local point %" PetscInt_FMT, point);
3771         ++sl;
3772       }
3773       PetscCall(PetscFree2(newLocalPoints,newOwners));
3774       PetscCheck(sl + ll == numSubLeaves,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatch in number of subleaves %" PetscInt_FMT " + %" PetscInt_FMT " != %" PetscInt_FMT, sl, ll, numSubLeaves);
3775       PetscCall(PetscSFSetGraph(sfPointSub, numSubRoots, sl, slocalPoints, PETSC_OWN_POINTER, sremotePoints, PETSC_OWN_POINTER));
3776     }
3777   }
3778   CHKMEMQ;
3779   /* Cleanup */
3780   if (subvertexIS) PetscCall(ISRestoreIndices(subvertexIS, &subVertices));
3781   PetscCall(ISDestroy(&subvertexIS));
3782   PetscCall(PetscFree(subCells));
3783   PetscFunctionReturn(0);
3784 }
3785 
3786 static PetscErrorCode DMPlexCreateCohesiveSubmesh_Interpolated(DM dm, const char labelname[], PetscInt value, DM subdm)
3787 {
3788   DMLabel        label = NULL;
3789 
3790   PetscFunctionBegin;
3791   if (labelname) PetscCall(DMGetLabel(dm, labelname, &label));
3792   PetscCall(DMPlexCreateSubmeshGeneric_Interpolated(dm, label, value, PETSC_FALSE, PETSC_TRUE, 1, subdm));
3793   PetscFunctionReturn(0);
3794 }
3795 
3796 /*@C
3797   DMPlexCreateCohesiveSubmesh - Extract from a mesh with cohesive cells the hypersurface defined by one face of the cells. Optionally, a Label can be given to restrict the cells.
3798 
3799   Input Parameters:
3800 + dm          - The original mesh
3801 . hasLagrange - The mesh has Lagrange unknowns in the cohesive cells
3802 . label       - A label name, or NULL
3803 - value  - A label value
3804 
3805   Output Parameter:
3806 . subdm - The surface mesh
3807 
3808   Note: This function produces a DMLabel mapping original points in the submesh to their depth. This can be obtained using DMPlexGetSubpointMap().
3809 
3810   Level: developer
3811 
3812 .seealso: `DMPlexGetSubpointMap()`, `DMPlexCreateSubmesh()`
3813 @*/
3814 PetscErrorCode DMPlexCreateCohesiveSubmesh(DM dm, PetscBool hasLagrange, const char label[], PetscInt value, DM *subdm)
3815 {
3816   PetscInt       dim, cdim, depth;
3817 
3818   PetscFunctionBegin;
3819   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3820   PetscValidPointer(subdm, 5);
3821   PetscCall(DMGetDimension(dm, &dim));
3822   PetscCall(DMPlexGetDepth(dm, &depth));
3823   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), subdm));
3824   PetscCall(DMSetType(*subdm, DMPLEX));
3825   PetscCall(DMSetDimension(*subdm, dim-1));
3826   PetscCall(DMGetCoordinateDim(dm, &cdim));
3827   PetscCall(DMSetCoordinateDim(*subdm, cdim));
3828   if (depth == dim) {
3829     PetscCall(DMPlexCreateCohesiveSubmesh_Interpolated(dm, label, value, *subdm));
3830   } else {
3831     PetscCall(DMPlexCreateCohesiveSubmesh_Uninterpolated(dm, hasLagrange, label, value, *subdm));
3832   }
3833   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_TRUE, *subdm));
3834   PetscFunctionReturn(0);
3835 }
3836 
3837 /*@
3838   DMPlexFilter - Extract a subset of mesh cells defined by a label as a separate mesh
3839 
3840   Input Parameters:
3841 + dm        - The original mesh
3842 . cellLabel - The DMLabel marking cells contained in the new mesh
3843 - value     - The label value to use
3844 
3845   Output Parameter:
3846 . subdm - The new mesh
3847 
3848   Note: This function produces a DMLabel mapping original points in the submesh to their depth. This can be obtained using DMPlexGetSubpointMap().
3849 
3850   Level: developer
3851 
3852 .seealso: `DMPlexGetSubpointMap()`, `DMGetLabel()`, `DMLabelSetValue()`
3853 @*/
3854 PetscErrorCode DMPlexFilter(DM dm, DMLabel cellLabel, PetscInt value, DM *subdm)
3855 {
3856   PetscInt       dim;
3857 
3858   PetscFunctionBegin;
3859   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3860   PetscValidPointer(subdm, 4);
3861   PetscCall(DMGetDimension(dm, &dim));
3862   PetscCall(DMCreate(PetscObjectComm((PetscObject) dm), subdm));
3863   PetscCall(DMSetType(*subdm, DMPLEX));
3864   PetscCall(DMSetDimension(*subdm, dim));
3865   /* Extract submesh in place, could be empty on some procs, could have inconsistency if procs do not both extract a shared cell */
3866   PetscCall(DMPlexCreateSubmeshGeneric_Interpolated(dm, cellLabel, value, PETSC_FALSE, PETSC_FALSE, 0, *subdm));
3867   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_TRUE, *subdm));
3868   PetscFunctionReturn(0);
3869 }
3870 
3871 /*@
3872   DMPlexGetSubpointMap - Returns a DMLabel with point dimension as values
3873 
3874   Input Parameter:
3875 . dm - The submesh DM
3876 
3877   Output Parameter:
3878 . subpointMap - The DMLabel of all the points from the original mesh in this submesh, or NULL if this is not a submesh
3879 
3880   Level: developer
3881 
3882 .seealso: `DMPlexCreateSubmesh()`, `DMPlexGetSubpointIS()`
3883 @*/
3884 PetscErrorCode DMPlexGetSubpointMap(DM dm, DMLabel *subpointMap)
3885 {
3886   PetscFunctionBegin;
3887   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3888   PetscValidPointer(subpointMap, 2);
3889   *subpointMap = ((DM_Plex*) dm->data)->subpointMap;
3890   PetscFunctionReturn(0);
3891 }
3892 
3893 /*@
3894   DMPlexSetSubpointMap - Sets the DMLabel with point dimension as values
3895 
3896   Input Parameters:
3897 + dm - The submesh DM
3898 - subpointMap - The DMLabel of all the points from the original mesh in this submesh
3899 
3900   Note: Should normally not be called by the user, since it is set in DMPlexCreateSubmesh()
3901 
3902   Level: developer
3903 
3904 .seealso: `DMPlexCreateSubmesh()`, `DMPlexGetSubpointIS()`
3905 @*/
3906 PetscErrorCode DMPlexSetSubpointMap(DM dm, DMLabel subpointMap)
3907 {
3908   DM_Plex       *mesh = (DM_Plex *) dm->data;
3909   DMLabel        tmp;
3910 
3911   PetscFunctionBegin;
3912   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3913   tmp  = mesh->subpointMap;
3914   mesh->subpointMap = subpointMap;
3915   PetscCall(PetscObjectReference((PetscObject) mesh->subpointMap));
3916   PetscCall(DMLabelDestroy(&tmp));
3917   PetscFunctionReturn(0);
3918 }
3919 
3920 static PetscErrorCode DMPlexCreateSubpointIS_Internal(DM dm, IS *subpointIS)
3921 {
3922   DMLabel        spmap;
3923   PetscInt       depth, d;
3924 
3925   PetscFunctionBegin;
3926   PetscCall(DMPlexGetSubpointMap(dm, &spmap));
3927   PetscCall(DMPlexGetDepth(dm, &depth));
3928   if (spmap && depth >= 0) {
3929     DM_Plex  *mesh = (DM_Plex *) dm->data;
3930     PetscInt *points, *depths;
3931     PetscInt  pStart, pEnd, p, off;
3932 
3933     PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
3934     PetscCheck(!pStart,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Submeshes must start the point numbering at 0, not %" PetscInt_FMT, pStart);
3935     PetscCall(PetscMalloc1(pEnd, &points));
3936     PetscCall(DMGetWorkArray(dm, depth+1, MPIU_INT, &depths));
3937     depths[0] = depth;
3938     depths[1] = 0;
3939     for (d = 2; d <= depth; ++d) {depths[d] = depth+1 - d;}
3940     for (d = 0, off = 0; d <= depth; ++d) {
3941       const PetscInt dep = depths[d];
3942       PetscInt       depStart, depEnd, n;
3943 
3944       PetscCall(DMPlexGetDepthStratum(dm, dep, &depStart, &depEnd));
3945       PetscCall(DMLabelGetStratumSize(spmap, dep, &n));
3946       if (((d < 2) && (depth > 1)) || (d == 1)) { /* Only check vertices and cells for now since the map is broken for others */
3947         PetscCheck(n == depEnd-depStart,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "The number of mapped submesh points %" PetscInt_FMT " at depth %" PetscInt_FMT " should be %" PetscInt_FMT, n, dep, depEnd-depStart);
3948       } else {
3949         if (!n) {
3950           if (d == 0) {
3951             /* Missing cells */
3952             for (p = 0; p < depEnd-depStart; ++p, ++off) points[off] = -1;
3953           } else {
3954             /* Missing faces */
3955             for (p = 0; p < depEnd-depStart; ++p, ++off) points[off] = PETSC_MAX_INT;
3956           }
3957         }
3958       }
3959       if (n) {
3960         IS              is;
3961         const PetscInt *opoints;
3962 
3963         PetscCall(DMLabelGetStratumIS(spmap, dep, &is));
3964         PetscCall(ISGetIndices(is, &opoints));
3965         for (p = 0; p < n; ++p, ++off) points[off] = opoints[p];
3966         PetscCall(ISRestoreIndices(is, &opoints));
3967         PetscCall(ISDestroy(&is));
3968       }
3969     }
3970     PetscCall(DMRestoreWorkArray(dm, depth+1, MPIU_INT, &depths));
3971     PetscCheck(off == pEnd,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "The number of mapped submesh points %" PetscInt_FMT " should be %" PetscInt_FMT, off, pEnd);
3972     PetscCall(ISCreateGeneral(PETSC_COMM_SELF, pEnd, points, PETSC_OWN_POINTER, subpointIS));
3973     PetscCall(PetscObjectStateGet((PetscObject) spmap, &mesh->subpointState));
3974   }
3975   PetscFunctionReturn(0);
3976 }
3977 
3978 /*@
3979   DMPlexGetSubpointIS - Returns an IS covering the entire subdm chart with the original points as data
3980 
3981   Input Parameter:
3982 . dm - The submesh DM
3983 
3984   Output Parameter:
3985 . subpointIS - The IS of all the points from the original mesh in this submesh, or NULL if this is not a submesh
3986 
3987   Note: This IS is guaranteed to be sorted by the construction of the submesh
3988 
3989   Level: developer
3990 
3991 .seealso: `DMPlexCreateSubmesh()`, `DMPlexGetSubpointMap()`
3992 @*/
3993 PetscErrorCode DMPlexGetSubpointIS(DM dm, IS *subpointIS)
3994 {
3995   DM_Plex         *mesh = (DM_Plex *) dm->data;
3996   DMLabel          spmap;
3997   PetscObjectState state;
3998 
3999   PetscFunctionBegin;
4000   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4001   PetscValidPointer(subpointIS, 2);
4002   PetscCall(DMPlexGetSubpointMap(dm, &spmap));
4003   PetscCall(PetscObjectStateGet((PetscObject) spmap, &state));
4004   if (state != mesh->subpointState || !mesh->subpointIS) PetscCall(DMPlexCreateSubpointIS_Internal(dm, &mesh->subpointIS));
4005   *subpointIS = mesh->subpointIS;
4006   PetscFunctionReturn(0);
4007 }
4008 
4009 /*@
4010   DMGetEnclosureRelation - Get the relationship between dmA and dmB
4011 
4012   Input Parameters:
4013 + dmA - The first DM
4014 - dmB - The second DM
4015 
4016   Output Parameter:
4017 . rel - The relation of dmA to dmB
4018 
4019   Level: intermediate
4020 
4021 .seealso: `DMGetEnclosurePoint()`
4022 @*/
4023 PetscErrorCode DMGetEnclosureRelation(DM dmA, DM dmB, DMEnclosureType *rel)
4024 {
4025   DM             plexA, plexB, sdm;
4026   DMLabel        spmap;
4027   PetscInt       pStartA, pEndA, pStartB, pEndB, NpA, NpB;
4028 
4029   PetscFunctionBegin;
4030   PetscValidPointer(rel, 3);
4031   *rel = DM_ENC_NONE;
4032   if (!dmA || !dmB) PetscFunctionReturn(0);
4033   PetscValidHeaderSpecific(dmA, DM_CLASSID, 1);
4034   PetscValidHeaderSpecific(dmB, DM_CLASSID, 2);
4035   if (dmA == dmB) {*rel = DM_ENC_EQUALITY; PetscFunctionReturn(0);}
4036   PetscCall(DMConvert(dmA, DMPLEX, &plexA));
4037   PetscCall(DMConvert(dmB, DMPLEX, &plexB));
4038   PetscCall(DMPlexGetChart(plexA, &pStartA, &pEndA));
4039   PetscCall(DMPlexGetChart(plexB, &pStartB, &pEndB));
4040   /* Assumption 1: subDMs have smaller charts than the DMs that they originate from
4041     - The degenerate case of a subdomain which includes all of the domain on some process can be treated as equality */
4042   if ((pStartA == pStartB) && (pEndA == pEndB)) {
4043     *rel = DM_ENC_EQUALITY;
4044     goto end;
4045   }
4046   NpA = pEndA - pStartA;
4047   NpB = pEndB - pStartB;
4048   if (NpA == NpB) goto end;
4049   sdm = NpA > NpB ? plexB : plexA; /* The other is the original, enclosing dm */
4050   PetscCall(DMPlexGetSubpointMap(sdm, &spmap));
4051   if (!spmap) goto end;
4052   /* TODO Check the space mapped to by subpointMap is same size as dm */
4053   if (NpA > NpB) {
4054     *rel = DM_ENC_SUPERMESH;
4055   } else {
4056     *rel = DM_ENC_SUBMESH;
4057   }
4058   end:
4059   PetscCall(DMDestroy(&plexA));
4060   PetscCall(DMDestroy(&plexB));
4061   PetscFunctionReturn(0);
4062 }
4063 
4064 /*@
4065   DMGetEnclosurePoint - Get the point pA in dmA which corresponds to the point pB in dmB
4066 
4067   Input Parameters:
4068 + dmA   - The first DM
4069 . dmB   - The second DM
4070 . etype - The type of enclosure relation that dmA has to dmB
4071 - pB    - A point of dmB
4072 
4073   Output Parameter:
4074 . pA    - The corresponding point of dmA
4075 
4076   Level: intermediate
4077 
4078 .seealso: `DMGetEnclosureRelation()`
4079 @*/
4080 PetscErrorCode DMGetEnclosurePoint(DM dmA, DM dmB, DMEnclosureType etype, PetscInt pB, PetscInt *pA)
4081 {
4082   DM              sdm;
4083   IS              subpointIS;
4084   const PetscInt *subpoints;
4085   PetscInt        numSubpoints;
4086 
4087   PetscFunctionBegin;
4088   /* TODO Cache the IS, making it look like an index */
4089   switch (etype) {
4090     case DM_ENC_SUPERMESH:
4091     sdm  = dmB;
4092     PetscCall(DMPlexGetSubpointIS(sdm, &subpointIS));
4093     PetscCall(ISGetIndices(subpointIS, &subpoints));
4094     *pA  = subpoints[pB];
4095     PetscCall(ISRestoreIndices(subpointIS, &subpoints));
4096     break;
4097     case DM_ENC_SUBMESH:
4098     sdm  = dmA;
4099     PetscCall(DMPlexGetSubpointIS(sdm, &subpointIS));
4100     PetscCall(ISGetLocalSize(subpointIS, &numSubpoints));
4101     PetscCall(ISGetIndices(subpointIS, &subpoints));
4102     PetscCall(PetscFindInt(pB, numSubpoints, subpoints, pA));
4103     if (*pA < 0) {
4104       PetscCall(DMViewFromOptions(dmA, NULL, "-dm_enc_A_view"));
4105       PetscCall(DMViewFromOptions(dmB, NULL, "-dm_enc_B_view"));
4106       SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Point %" PetscInt_FMT " not found in submesh", pB);
4107     }
4108     PetscCall(ISRestoreIndices(subpointIS, &subpoints));
4109     break;
4110     case DM_ENC_EQUALITY:
4111     case DM_ENC_NONE:
4112     *pA = pB;break;
4113     case DM_ENC_UNKNOWN:
4114     {
4115       DMEnclosureType enc;
4116 
4117       PetscCall(DMGetEnclosureRelation(dmA, dmB, &enc));
4118       PetscCall(DMGetEnclosurePoint(dmA, dmB, enc, pB, pA));
4119     }
4120     break;
4121     default: SETERRQ(PetscObjectComm((PetscObject) dmA), PETSC_ERR_ARG_OUTOFRANGE, "Invalid enclosure type %d", (int) etype);
4122   }
4123   PetscFunctionReturn(0);
4124 }
4125