xref: /petsc/src/dm/impls/plex/plexsubmesh.c (revision d5b43468fb8780a8feea140ccd6fa3e6a50411cc)
1 #include <petsc/private/dmpleximpl.h>  /*I      "petscdmplex.h"    I*/
2 #include <petsc/private/dmlabelimpl.h> /*I      "petscdmlabel.h"   I*/
3 #include <petscsf.h>
4 
5 static PetscErrorCode DMPlexCellIsHybrid_Internal(DM dm, PetscInt p, PetscBool *isHybrid)
6 {
7   DMPolytopeType ct;
8 
9   PetscFunctionBegin;
10   PetscCall(DMPlexGetCellType(dm, p, &ct));
11   switch (ct) {
12   case DM_POLYTOPE_POINT_PRISM_TENSOR:
13   case DM_POLYTOPE_SEG_PRISM_TENSOR:
14   case DM_POLYTOPE_TRI_PRISM_TENSOR:
15   case DM_POLYTOPE_QUAD_PRISM_TENSOR:
16     *isHybrid = PETSC_TRUE;
17     break;
18   default:
19     *isHybrid = PETSC_FALSE;
20     break;
21   }
22   PetscFunctionReturn(0);
23 }
24 
25 static PetscErrorCode DMPlexGetTensorPrismBounds_Internal(DM dm, PetscInt dim, PetscInt *cStart, PetscInt *cEnd)
26 {
27   DMLabel ctLabel;
28 
29   PetscFunctionBegin;
30   if (cStart) *cStart = -1;
31   if (cEnd) *cEnd = -1;
32   PetscCall(DMPlexGetCellTypeLabel(dm, &ctLabel));
33   switch (dim) {
34   case 1:
35     PetscCall(DMLabelGetStratumBounds(ctLabel, DM_POLYTOPE_POINT_PRISM_TENSOR, cStart, cEnd));
36     break;
37   case 2:
38     PetscCall(DMLabelGetStratumBounds(ctLabel, DM_POLYTOPE_SEG_PRISM_TENSOR, cStart, cEnd));
39     break;
40   case 3:
41     PetscCall(DMLabelGetStratumBounds(ctLabel, DM_POLYTOPE_TRI_PRISM_TENSOR, cStart, cEnd));
42     if (*cStart < 0) PetscCall(DMLabelGetStratumBounds(ctLabel, DM_POLYTOPE_QUAD_PRISM_TENSOR, cStart, cEnd));
43     break;
44   default:
45     PetscFunctionReturn(0);
46   }
47   PetscFunctionReturn(0);
48 }
49 
50 static PetscErrorCode DMPlexMarkBoundaryFaces_Internal(DM dm, PetscInt val, PetscInt cellHeight, DMLabel label)
51 {
52   PetscInt fStart, fEnd, f;
53 
54   PetscFunctionBegin;
55   PetscCall(DMPlexGetHeightStratum(dm, cellHeight + 1, &fStart, &fEnd));
56   for (f = fStart; f < fEnd; ++f) {
57     PetscInt supportSize;
58 
59     PetscCall(DMPlexGetSupportSize(dm, f, &supportSize));
60     if (supportSize == 1) {
61       if (val < 0) {
62         PetscInt *closure = NULL;
63         PetscInt  clSize, cl, cval;
64 
65         PetscCall(DMPlexGetTransitiveClosure(dm, f, PETSC_TRUE, &clSize, &closure));
66         for (cl = 0; cl < clSize * 2; cl += 2) {
67           PetscCall(DMLabelGetValue(label, closure[cl], &cval));
68           if (cval < 0) continue;
69           PetscCall(DMLabelSetValue(label, f, cval));
70           break;
71         }
72         if (cl == clSize * 2) PetscCall(DMLabelSetValue(label, f, 1));
73         PetscCall(DMPlexRestoreTransitiveClosure(dm, f, PETSC_TRUE, &clSize, &closure));
74       } else {
75         PetscCall(DMLabelSetValue(label, f, val));
76       }
77     }
78   }
79   PetscFunctionReturn(0);
80 }
81 
82 /*@
83   DMPlexMarkBoundaryFaces - Mark all faces on the boundary
84 
85   Not Collective
86 
87   Input Parameters:
88 + dm - The original DM
89 - val - The marker value, or PETSC_DETERMINE to use some value in the closure (or 1 if none are found)
90 
91   Output Parameter:
92 . label - The DMLabel marking boundary faces with the given value
93 
94   Level: developer
95 
96 .seealso: `DMLabelCreate()`, `DMCreateLabel()`
97 @*/
98 PetscErrorCode DMPlexMarkBoundaryFaces(DM dm, PetscInt val, DMLabel label)
99 {
100   DMPlexInterpolatedFlag flg;
101 
102   PetscFunctionBegin;
103   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
104   PetscCall(DMPlexIsInterpolated(dm, &flg));
105   PetscCheck(flg == DMPLEX_INTERPOLATED_FULL, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM is not fully interpolated on this rank");
106   PetscCall(DMPlexMarkBoundaryFaces_Internal(dm, val, 0, label));
107   PetscFunctionReturn(0);
108 }
109 
110 static PetscErrorCode DMPlexLabelComplete_Internal(DM dm, DMLabel label, PetscBool completeCells)
111 {
112   IS              valueIS;
113   PetscSF         sfPoint;
114   const PetscInt *values;
115   PetscInt        numValues, v, cStart, cEnd, nroots;
116 
117   PetscFunctionBegin;
118   PetscCall(DMLabelGetNumValues(label, &numValues));
119   PetscCall(DMLabelGetValueIS(label, &valueIS));
120   PetscCall(DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd));
121   PetscCall(ISGetIndices(valueIS, &values));
122   for (v = 0; v < numValues; ++v) {
123     IS              pointIS;
124     const PetscInt *points;
125     PetscInt        numPoints, p;
126 
127     PetscCall(DMLabelGetStratumSize(label, values[v], &numPoints));
128     PetscCall(DMLabelGetStratumIS(label, values[v], &pointIS));
129     PetscCall(ISGetIndices(pointIS, &points));
130     for (p = 0; p < numPoints; ++p) {
131       PetscInt  q       = points[p];
132       PetscInt *closure = NULL;
133       PetscInt  closureSize, c;
134 
135       if (cStart <= q && q < cEnd && !completeCells) { /* skip cells */
136         continue;
137       }
138       PetscCall(DMPlexGetTransitiveClosure(dm, q, PETSC_TRUE, &closureSize, &closure));
139       for (c = 0; c < closureSize * 2; c += 2) PetscCall(DMLabelSetValue(label, closure[c], values[v]));
140       PetscCall(DMPlexRestoreTransitiveClosure(dm, q, PETSC_TRUE, &closureSize, &closure));
141     }
142     PetscCall(ISRestoreIndices(pointIS, &points));
143     PetscCall(ISDestroy(&pointIS));
144   }
145   PetscCall(ISRestoreIndices(valueIS, &values));
146   PetscCall(ISDestroy(&valueIS));
147   PetscCall(DMGetPointSF(dm, &sfPoint));
148   PetscCall(PetscSFGetGraph(sfPoint, &nroots, NULL, NULL, NULL));
149   if (nroots >= 0) {
150     DMLabel         lblRoots, lblLeaves;
151     IS              valueIS, pointIS;
152     const PetscInt *values;
153     PetscInt        numValues, v;
154 
155     /* Pull point contributions from remote leaves into local roots */
156     PetscCall(DMLabelGather(label, sfPoint, &lblLeaves));
157     PetscCall(DMLabelGetValueIS(lblLeaves, &valueIS));
158     PetscCall(ISGetLocalSize(valueIS, &numValues));
159     PetscCall(ISGetIndices(valueIS, &values));
160     for (v = 0; v < numValues; ++v) {
161       const PetscInt value = values[v];
162 
163       PetscCall(DMLabelGetStratumIS(lblLeaves, value, &pointIS));
164       PetscCall(DMLabelInsertIS(label, pointIS, value));
165       PetscCall(ISDestroy(&pointIS));
166     }
167     PetscCall(ISRestoreIndices(valueIS, &values));
168     PetscCall(ISDestroy(&valueIS));
169     PetscCall(DMLabelDestroy(&lblLeaves));
170     /* Push point contributions from roots into remote leaves */
171     PetscCall(DMLabelDistribute(label, sfPoint, &lblRoots));
172     PetscCall(DMLabelGetValueIS(lblRoots, &valueIS));
173     PetscCall(ISGetLocalSize(valueIS, &numValues));
174     PetscCall(ISGetIndices(valueIS, &values));
175     for (v = 0; v < numValues; ++v) {
176       const PetscInt value = values[v];
177 
178       PetscCall(DMLabelGetStratumIS(lblRoots, value, &pointIS));
179       PetscCall(DMLabelInsertIS(label, pointIS, value));
180       PetscCall(ISDestroy(&pointIS));
181     }
182     PetscCall(ISRestoreIndices(valueIS, &values));
183     PetscCall(ISDestroy(&valueIS));
184     PetscCall(DMLabelDestroy(&lblRoots));
185   }
186   PetscFunctionReturn(0);
187 }
188 
189 /*@
190   DMPlexLabelComplete - Starting with a label marking points on a surface, we add the transitive closure to the surface
191 
192   Input Parameters:
193 + dm - The DM
194 - label - A DMLabel marking the surface points
195 
196   Output Parameter:
197 . label - A DMLabel marking all surface points in the transitive closure
198 
199   Level: developer
200 
201 .seealso: `DMPlexLabelCohesiveComplete()`
202 @*/
203 PetscErrorCode DMPlexLabelComplete(DM dm, DMLabel label)
204 {
205   PetscFunctionBegin;
206   PetscCall(DMPlexLabelComplete_Internal(dm, label, PETSC_TRUE));
207   PetscFunctionReturn(0);
208 }
209 
210 /*@
211   DMPlexLabelAddCells - Starting with a label marking points on a surface, we add a cell for each point
212 
213   Input Parameters:
214 + dm - The DM
215 - label - A DMLabel marking the surface points
216 
217   Output Parameter:
218 . label - A DMLabel incorporating cells
219 
220   Level: developer
221 
222   Note: The cells allow FEM boundary conditions to be applied using the cell geometry
223 
224 .seealso: `DMPlexLabelAddFaceCells()`, `DMPlexLabelComplete()`, `DMPlexLabelCohesiveComplete()`
225 @*/
226 PetscErrorCode DMPlexLabelAddCells(DM dm, DMLabel label)
227 {
228   IS              valueIS;
229   const PetscInt *values;
230   PetscInt        numValues, v, cStart, cEnd;
231 
232   PetscFunctionBegin;
233   PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd));
234   PetscCall(DMLabelGetNumValues(label, &numValues));
235   PetscCall(DMLabelGetValueIS(label, &valueIS));
236   PetscCall(ISGetIndices(valueIS, &values));
237   for (v = 0; v < numValues; ++v) {
238     IS              pointIS;
239     const PetscInt *points;
240     PetscInt        numPoints, p;
241 
242     PetscCall(DMLabelGetStratumSize(label, values[v], &numPoints));
243     PetscCall(DMLabelGetStratumIS(label, values[v], &pointIS));
244     PetscCall(ISGetIndices(pointIS, &points));
245     for (p = 0; p < numPoints; ++p) {
246       PetscInt *closure = NULL;
247       PetscInt  closureSize, cl;
248 
249       PetscCall(DMPlexGetTransitiveClosure(dm, points[p], PETSC_FALSE, &closureSize, &closure));
250       for (cl = closureSize - 1; cl > 0; --cl) {
251         const PetscInt cell = closure[cl * 2];
252         if ((cell >= cStart) && (cell < cEnd)) {
253           PetscCall(DMLabelSetValue(label, cell, values[v]));
254           break;
255         }
256       }
257       PetscCall(DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_FALSE, &closureSize, &closure));
258     }
259     PetscCall(ISRestoreIndices(pointIS, &points));
260     PetscCall(ISDestroy(&pointIS));
261   }
262   PetscCall(ISRestoreIndices(valueIS, &values));
263   PetscCall(ISDestroy(&valueIS));
264   PetscFunctionReturn(0);
265 }
266 
267 /*@
268   DMPlexLabelAddFaceCells - Starting with a label marking faces on a surface, we add a cell for each face
269 
270   Input Parameters:
271 + dm - The DM
272 - label - A DMLabel marking the surface points
273 
274   Output Parameter:
275 . label - A DMLabel incorporating cells
276 
277   Level: developer
278 
279   Note: The cells allow FEM boundary conditions to be applied using the cell geometry
280 
281 .seealso: `DMPlexLabelAddCells()`, `DMPlexLabelComplete()`, `DMPlexLabelCohesiveComplete()`
282 @*/
283 PetscErrorCode DMPlexLabelAddFaceCells(DM dm, DMLabel label)
284 {
285   IS              valueIS;
286   const PetscInt *values;
287   PetscInt        numValues, v, cStart, cEnd, fStart, fEnd;
288 
289   PetscFunctionBegin;
290   PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd));
291   PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
292   PetscCall(DMLabelGetNumValues(label, &numValues));
293   PetscCall(DMLabelGetValueIS(label, &valueIS));
294   PetscCall(ISGetIndices(valueIS, &values));
295   for (v = 0; v < numValues; ++v) {
296     IS              pointIS;
297     const PetscInt *points;
298     PetscInt        numPoints, p;
299 
300     PetscCall(DMLabelGetStratumSize(label, values[v], &numPoints));
301     PetscCall(DMLabelGetStratumIS(label, values[v], &pointIS));
302     PetscCall(ISGetIndices(pointIS, &points));
303     for (p = 0; p < numPoints; ++p) {
304       const PetscInt face    = points[p];
305       PetscInt      *closure = NULL;
306       PetscInt       closureSize, cl;
307 
308       if ((face < fStart) || (face >= fEnd)) continue;
309       PetscCall(DMPlexGetTransitiveClosure(dm, face, PETSC_FALSE, &closureSize, &closure));
310       for (cl = closureSize - 1; cl > 0; --cl) {
311         const PetscInt cell = closure[cl * 2];
312         if ((cell >= cStart) && (cell < cEnd)) {
313           PetscCall(DMLabelSetValue(label, cell, values[v]));
314           break;
315         }
316       }
317       PetscCall(DMPlexRestoreTransitiveClosure(dm, face, PETSC_FALSE, &closureSize, &closure));
318     }
319     PetscCall(ISRestoreIndices(pointIS, &points));
320     PetscCall(ISDestroy(&pointIS));
321   }
322   PetscCall(ISRestoreIndices(valueIS, &values));
323   PetscCall(ISDestroy(&valueIS));
324   PetscFunctionReturn(0);
325 }
326 
327 /*@
328   DMPlexLabelClearCells - Remove cells from a label
329 
330   Input Parameters:
331 + dm - The DM
332 - label - A DMLabel marking surface points and their adjacent cells
333 
334   Output Parameter:
335 . label - A DMLabel without cells
336 
337   Level: developer
338 
339   Note: This undoes DMPlexLabelAddCells() or DMPlexLabelAddFaceCells()
340 
341 .seealso: `DMPlexLabelComplete()`, `DMPlexLabelCohesiveComplete()`, `DMPlexLabelAddCells()`
342 @*/
343 PetscErrorCode DMPlexLabelClearCells(DM dm, DMLabel label)
344 {
345   IS              valueIS;
346   const PetscInt *values;
347   PetscInt        numValues, v, cStart, cEnd;
348 
349   PetscFunctionBegin;
350   PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd));
351   PetscCall(DMLabelGetNumValues(label, &numValues));
352   PetscCall(DMLabelGetValueIS(label, &valueIS));
353   PetscCall(ISGetIndices(valueIS, &values));
354   for (v = 0; v < numValues; ++v) {
355     IS              pointIS;
356     const PetscInt *points;
357     PetscInt        numPoints, p;
358 
359     PetscCall(DMLabelGetStratumSize(label, values[v], &numPoints));
360     PetscCall(DMLabelGetStratumIS(label, values[v], &pointIS));
361     PetscCall(ISGetIndices(pointIS, &points));
362     for (p = 0; p < numPoints; ++p) {
363       PetscInt point = points[p];
364 
365       if (point >= cStart && point < cEnd) PetscCall(DMLabelClearValue(label, point, values[v]));
366     }
367     PetscCall(ISRestoreIndices(pointIS, &points));
368     PetscCall(ISDestroy(&pointIS));
369   }
370   PetscCall(ISRestoreIndices(valueIS, &values));
371   PetscCall(ISDestroy(&valueIS));
372   PetscFunctionReturn(0);
373 }
374 
375 /* take (oldEnd, added) pairs, ordered by height and convert them to (oldstart, newstart) pairs, ordered by ascending
376  * index (skipping first, which is (0,0)) */
377 static inline PetscErrorCode DMPlexShiftPointSetUp_Internal(PetscInt depth, PetscInt depthShift[])
378 {
379   PetscInt d, off = 0;
380 
381   PetscFunctionBegin;
382   /* sort by (oldend): yes this is an O(n^2) sort, we expect depth <= 3 */
383   for (d = 0; d < depth; d++) {
384     PetscInt firstd     = d;
385     PetscInt firstStart = depthShift[2 * d];
386     PetscInt e;
387 
388     for (e = d + 1; e <= depth; e++) {
389       if (depthShift[2 * e] < firstStart) {
390         firstd     = e;
391         firstStart = depthShift[2 * d];
392       }
393     }
394     if (firstd != d) {
395       PetscInt swap[2];
396 
397       e                     = firstd;
398       swap[0]               = depthShift[2 * d];
399       swap[1]               = depthShift[2 * d + 1];
400       depthShift[2 * d]     = depthShift[2 * e];
401       depthShift[2 * d + 1] = depthShift[2 * e + 1];
402       depthShift[2 * e]     = swap[0];
403       depthShift[2 * e + 1] = swap[1];
404     }
405   }
406   /* convert (oldstart, added) to (oldstart, newstart) */
407   for (d = 0; d <= depth; d++) {
408     off += depthShift[2 * d + 1];
409     depthShift[2 * d + 1] = depthShift[2 * d] + off;
410   }
411   PetscFunctionReturn(0);
412 }
413 
414 /* depthShift is a list of (old, new) pairs */
415 static inline PetscInt DMPlexShiftPoint_Internal(PetscInt p, PetscInt depth, PetscInt depthShift[])
416 {
417   PetscInt d;
418   PetscInt newOff = 0;
419 
420   for (d = 0; d <= depth; d++) {
421     if (p < depthShift[2 * d]) return p + newOff;
422     else newOff = depthShift[2 * d + 1] - depthShift[2 * d];
423   }
424   return p + newOff;
425 }
426 
427 /* depthShift is a list of (old, new) pairs */
428 static inline PetscInt DMPlexShiftPointInverse_Internal(PetscInt p, PetscInt depth, PetscInt depthShift[])
429 {
430   PetscInt d;
431   PetscInt newOff = 0;
432 
433   for (d = 0; d <= depth; d++) {
434     if (p < depthShift[2 * d + 1]) return p + newOff;
435     else newOff = depthShift[2 * d] - depthShift[2 * d + 1];
436   }
437   return p + newOff;
438 }
439 
440 static PetscErrorCode DMPlexShiftSizes_Internal(DM dm, PetscInt depthShift[], DM dmNew)
441 {
442   PetscInt depth = 0, d, pStart, pEnd, p;
443   DMLabel  depthLabel;
444 
445   PetscFunctionBegin;
446   PetscCall(DMPlexGetDepth(dm, &depth));
447   if (depth < 0) PetscFunctionReturn(0);
448   /* Step 1: Expand chart */
449   PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
450   pEnd = DMPlexShiftPoint_Internal(pEnd, depth, depthShift);
451   PetscCall(DMPlexSetChart(dmNew, pStart, pEnd));
452   PetscCall(DMCreateLabel(dmNew, "depth"));
453   PetscCall(DMPlexGetDepthLabel(dmNew, &depthLabel));
454   PetscCall(DMCreateLabel(dmNew, "celltype"));
455   /* Step 2: Set cone and support sizes */
456   for (d = 0; d <= depth; ++d) {
457     PetscInt pStartNew, pEndNew;
458     IS       pIS;
459 
460     PetscCall(DMPlexGetDepthStratum(dm, d, &pStart, &pEnd));
461     pStartNew = DMPlexShiftPoint_Internal(pStart, depth, depthShift);
462     pEndNew   = DMPlexShiftPoint_Internal(pEnd, depth, depthShift);
463     PetscCall(ISCreateStride(PETSC_COMM_SELF, pEndNew - pStartNew, pStartNew, 1, &pIS));
464     PetscCall(DMLabelSetStratumIS(depthLabel, d, pIS));
465     PetscCall(ISDestroy(&pIS));
466     for (p = pStart; p < pEnd; ++p) {
467       PetscInt       newp = DMPlexShiftPoint_Internal(p, depth, depthShift);
468       PetscInt       size;
469       DMPolytopeType ct;
470 
471       PetscCall(DMPlexGetConeSize(dm, p, &size));
472       PetscCall(DMPlexSetConeSize(dmNew, newp, size));
473       PetscCall(DMPlexGetSupportSize(dm, p, &size));
474       PetscCall(DMPlexSetSupportSize(dmNew, newp, size));
475       PetscCall(DMPlexGetCellType(dm, p, &ct));
476       PetscCall(DMPlexSetCellType(dmNew, newp, ct));
477     }
478   }
479   PetscFunctionReturn(0);
480 }
481 
482 static PetscErrorCode DMPlexShiftPoints_Internal(DM dm, PetscInt depthShift[], DM dmNew)
483 {
484   PetscInt *newpoints;
485   PetscInt  depth = 0, maxConeSize, maxSupportSize, maxConeSizeNew, maxSupportSizeNew, pStart, pEnd, p;
486 
487   PetscFunctionBegin;
488   PetscCall(DMPlexGetDepth(dm, &depth));
489   if (depth < 0) PetscFunctionReturn(0);
490   PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize));
491   PetscCall(DMPlexGetMaxSizes(dmNew, &maxConeSizeNew, &maxSupportSizeNew));
492   PetscCall(PetscMalloc1(PetscMax(PetscMax(maxConeSize, maxSupportSize), PetscMax(maxConeSizeNew, maxSupportSizeNew)), &newpoints));
493   /* Step 5: Set cones and supports */
494   PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
495   for (p = pStart; p < pEnd; ++p) {
496     const PetscInt *points = NULL, *orientations = NULL;
497     PetscInt        size, sizeNew, i, newp = DMPlexShiftPoint_Internal(p, depth, depthShift);
498 
499     PetscCall(DMPlexGetConeSize(dm, p, &size));
500     PetscCall(DMPlexGetCone(dm, p, &points));
501     PetscCall(DMPlexGetConeOrientation(dm, p, &orientations));
502     for (i = 0; i < size; ++i) newpoints[i] = DMPlexShiftPoint_Internal(points[i], depth, depthShift);
503     PetscCall(DMPlexSetCone(dmNew, newp, newpoints));
504     PetscCall(DMPlexSetConeOrientation(dmNew, newp, orientations));
505     PetscCall(DMPlexGetSupportSize(dm, p, &size));
506     PetscCall(DMPlexGetSupportSize(dmNew, newp, &sizeNew));
507     PetscCall(DMPlexGetSupport(dm, p, &points));
508     for (i = 0; i < size; ++i) newpoints[i] = DMPlexShiftPoint_Internal(points[i], depth, depthShift);
509     for (i = size; i < sizeNew; ++i) newpoints[i] = 0;
510     PetscCall(DMPlexSetSupport(dmNew, newp, newpoints));
511   }
512   PetscCall(PetscFree(newpoints));
513   PetscFunctionReturn(0);
514 }
515 
516 static PetscErrorCode DMPlexShiftCoordinates_Internal(DM dm, PetscInt depthShift[], DM dmNew)
517 {
518   PetscSection coordSection, newCoordSection;
519   Vec          coordinates, newCoordinates;
520   PetscScalar *coords, *newCoords;
521   PetscInt     coordSize, sStart, sEnd;
522   PetscInt     dim, depth = 0, cStart, cEnd, cStartNew, cEndNew, c, vStart, vEnd, vStartNew, vEndNew, v;
523   PetscBool    hasCells;
524 
525   PetscFunctionBegin;
526   PetscCall(DMGetCoordinateDim(dm, &dim));
527   PetscCall(DMSetCoordinateDim(dmNew, dim));
528   PetscCall(DMPlexGetDepth(dm, &depth));
529   /* Step 8: Convert coordinates */
530   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
531   PetscCall(DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd));
532   PetscCall(DMPlexGetDepthStratum(dmNew, 0, &vStartNew, &vEndNew));
533   PetscCall(DMPlexGetHeightStratum(dmNew, 0, &cStartNew, &cEndNew));
534   PetscCall(DMGetCoordinateSection(dm, &coordSection));
535   PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dm), &newCoordSection));
536   PetscCall(PetscSectionSetNumFields(newCoordSection, 1));
537   PetscCall(PetscSectionSetFieldComponents(newCoordSection, 0, dim));
538   PetscCall(PetscSectionGetChart(coordSection, &sStart, &sEnd));
539   hasCells = sStart == cStart ? PETSC_TRUE : PETSC_FALSE;
540   PetscCall(PetscSectionSetChart(newCoordSection, hasCells ? cStartNew : vStartNew, vEndNew));
541   if (hasCells) {
542     for (c = cStart; c < cEnd; ++c) {
543       PetscInt cNew = DMPlexShiftPoint_Internal(c, depth, depthShift), dof;
544 
545       PetscCall(PetscSectionGetDof(coordSection, c, &dof));
546       PetscCall(PetscSectionSetDof(newCoordSection, cNew, dof));
547       PetscCall(PetscSectionSetFieldDof(newCoordSection, cNew, 0, dof));
548     }
549   }
550   for (v = vStartNew; v < vEndNew; ++v) {
551     PetscCall(PetscSectionSetDof(newCoordSection, v, dim));
552     PetscCall(PetscSectionSetFieldDof(newCoordSection, v, 0, dim));
553   }
554   PetscCall(PetscSectionSetUp(newCoordSection));
555   PetscCall(DMSetCoordinateSection(dmNew, PETSC_DETERMINE, newCoordSection));
556   PetscCall(PetscSectionGetStorageSize(newCoordSection, &coordSize));
557   PetscCall(VecCreate(PETSC_COMM_SELF, &newCoordinates));
558   PetscCall(PetscObjectSetName((PetscObject)newCoordinates, "coordinates"));
559   PetscCall(VecSetSizes(newCoordinates, coordSize, PETSC_DETERMINE));
560   PetscCall(VecSetBlockSize(newCoordinates, dim));
561   PetscCall(VecSetType(newCoordinates, VECSTANDARD));
562   PetscCall(DMSetCoordinatesLocal(dmNew, newCoordinates));
563   PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
564   PetscCall(VecGetArray(coordinates, &coords));
565   PetscCall(VecGetArray(newCoordinates, &newCoords));
566   if (hasCells) {
567     for (c = cStart; c < cEnd; ++c) {
568       PetscInt cNew = DMPlexShiftPoint_Internal(c, depth, depthShift), dof, off, noff, d;
569 
570       PetscCall(PetscSectionGetDof(coordSection, c, &dof));
571       PetscCall(PetscSectionGetOffset(coordSection, c, &off));
572       PetscCall(PetscSectionGetOffset(newCoordSection, cNew, &noff));
573       for (d = 0; d < dof; ++d) newCoords[noff + d] = coords[off + d];
574     }
575   }
576   for (v = vStart; v < vEnd; ++v) {
577     PetscInt dof, off, noff, d;
578 
579     PetscCall(PetscSectionGetDof(coordSection, v, &dof));
580     PetscCall(PetscSectionGetOffset(coordSection, v, &off));
581     PetscCall(PetscSectionGetOffset(newCoordSection, DMPlexShiftPoint_Internal(v, depth, depthShift), &noff));
582     for (d = 0; d < dof; ++d) newCoords[noff + d] = coords[off + d];
583   }
584   PetscCall(VecRestoreArray(coordinates, &coords));
585   PetscCall(VecRestoreArray(newCoordinates, &newCoords));
586   PetscCall(VecDestroy(&newCoordinates));
587   PetscCall(PetscSectionDestroy(&newCoordSection));
588   PetscFunctionReturn(0);
589 }
590 
591 static PetscErrorCode DMPlexShiftSF_Single(DM dm, PetscInt depthShift[], PetscSF sf, PetscSF sfNew)
592 {
593   const PetscSFNode *remotePoints;
594   PetscSFNode       *gremotePoints;
595   const PetscInt    *localPoints;
596   PetscInt          *glocalPoints, *newLocation, *newRemoteLocation;
597   PetscInt           numRoots, numLeaves, l, pStart, pEnd, depth = 0, totShift = 0;
598 
599   PetscFunctionBegin;
600   PetscCall(DMPlexGetDepth(dm, &depth));
601   PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
602   PetscCall(PetscSFGetGraph(sf, &numRoots, &numLeaves, &localPoints, &remotePoints));
603   totShift = DMPlexShiftPoint_Internal(pEnd, depth, depthShift) - pEnd;
604   if (numRoots >= 0) {
605     PetscCall(PetscMalloc2(numRoots, &newLocation, pEnd - pStart, &newRemoteLocation));
606     for (l = 0; l < numRoots; ++l) newLocation[l] = DMPlexShiftPoint_Internal(l, depth, depthShift);
607     PetscCall(PetscSFBcastBegin(sf, MPIU_INT, newLocation, newRemoteLocation, MPI_REPLACE));
608     PetscCall(PetscSFBcastEnd(sf, MPIU_INT, newLocation, newRemoteLocation, MPI_REPLACE));
609     PetscCall(PetscMalloc1(numLeaves, &glocalPoints));
610     PetscCall(PetscMalloc1(numLeaves, &gremotePoints));
611     for (l = 0; l < numLeaves; ++l) {
612       glocalPoints[l]        = DMPlexShiftPoint_Internal(localPoints[l], depth, depthShift);
613       gremotePoints[l].rank  = remotePoints[l].rank;
614       gremotePoints[l].index = newRemoteLocation[localPoints[l]];
615     }
616     PetscCall(PetscFree2(newLocation, newRemoteLocation));
617     PetscCall(PetscSFSetGraph(sfNew, numRoots + totShift, numLeaves, glocalPoints, PETSC_OWN_POINTER, gremotePoints, PETSC_OWN_POINTER));
618   }
619   PetscFunctionReturn(0);
620 }
621 
622 static PetscErrorCode DMPlexShiftSF_Internal(DM dm, PetscInt depthShift[], DM dmNew)
623 {
624   PetscSF   sfPoint, sfPointNew;
625   PetscBool useNatural;
626 
627   PetscFunctionBegin;
628   /* Step 9: Convert pointSF */
629   PetscCall(DMGetPointSF(dm, &sfPoint));
630   PetscCall(DMGetPointSF(dmNew, &sfPointNew));
631   PetscCall(DMPlexShiftSF_Single(dm, depthShift, sfPoint, sfPointNew));
632   /* Step 9b: Convert naturalSF */
633   PetscCall(DMGetUseNatural(dm, &useNatural));
634   if (useNatural) {
635     PetscSF sfNat, sfNatNew;
636 
637     PetscCall(DMSetUseNatural(dmNew, useNatural));
638     PetscCall(DMGetNaturalSF(dm, &sfNat));
639     PetscCall(DMGetNaturalSF(dmNew, &sfNatNew));
640     PetscCall(DMPlexShiftSF_Single(dm, depthShift, sfNat, sfNatNew));
641   }
642   PetscFunctionReturn(0);
643 }
644 
645 static PetscErrorCode DMPlexShiftLabels_Internal(DM dm, PetscInt depthShift[], DM dmNew)
646 {
647   PetscInt depth = 0, numLabels, l;
648 
649   PetscFunctionBegin;
650   PetscCall(DMPlexGetDepth(dm, &depth));
651   /* Step 10: Convert labels */
652   PetscCall(DMGetNumLabels(dm, &numLabels));
653   for (l = 0; l < numLabels; ++l) {
654     DMLabel         label, newlabel;
655     const char     *lname;
656     PetscBool       isDepth, isDim;
657     IS              valueIS;
658     const PetscInt *values;
659     PetscInt        numValues, val;
660 
661     PetscCall(DMGetLabelName(dm, l, &lname));
662     PetscCall(PetscStrcmp(lname, "depth", &isDepth));
663     if (isDepth) continue;
664     PetscCall(PetscStrcmp(lname, "dim", &isDim));
665     if (isDim) continue;
666     PetscCall(DMCreateLabel(dmNew, lname));
667     PetscCall(DMGetLabel(dm, lname, &label));
668     PetscCall(DMGetLabel(dmNew, lname, &newlabel));
669     PetscCall(DMLabelGetDefaultValue(label, &val));
670     PetscCall(DMLabelSetDefaultValue(newlabel, val));
671     PetscCall(DMLabelGetValueIS(label, &valueIS));
672     PetscCall(ISGetLocalSize(valueIS, &numValues));
673     PetscCall(ISGetIndices(valueIS, &values));
674     for (val = 0; val < numValues; ++val) {
675       IS              pointIS;
676       const PetscInt *points;
677       PetscInt        numPoints, p;
678 
679       PetscCall(DMLabelGetStratumIS(label, values[val], &pointIS));
680       PetscCall(ISGetLocalSize(pointIS, &numPoints));
681       PetscCall(ISGetIndices(pointIS, &points));
682       for (p = 0; p < numPoints; ++p) {
683         const PetscInt newpoint = DMPlexShiftPoint_Internal(points[p], depth, depthShift);
684 
685         PetscCall(DMLabelSetValue(newlabel, newpoint, values[val]));
686       }
687       PetscCall(ISRestoreIndices(pointIS, &points));
688       PetscCall(ISDestroy(&pointIS));
689     }
690     PetscCall(ISRestoreIndices(valueIS, &values));
691     PetscCall(ISDestroy(&valueIS));
692   }
693   PetscFunctionReturn(0);
694 }
695 
696 static PetscErrorCode DMPlexCreateVTKLabel_Internal(DM dm, PetscBool createGhostLabel, DM dmNew)
697 {
698   PetscSF            sfPoint;
699   DMLabel            vtkLabel, ghostLabel = NULL;
700   const PetscSFNode *leafRemote;
701   const PetscInt    *leafLocal;
702   PetscInt           cellHeight, cStart, cEnd, c, fStart, fEnd, f, numLeaves, l;
703   PetscMPIInt        rank;
704 
705   PetscFunctionBegin;
706   /* Step 11: Make label for output (vtk) and to mark ghost points (ghost) */
707   PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
708   PetscCall(DMGetPointSF(dm, &sfPoint));
709   PetscCall(DMPlexGetVTKCellHeight(dmNew, &cellHeight));
710   PetscCall(DMPlexGetHeightStratum(dm, cellHeight, &cStart, &cEnd));
711   PetscCall(PetscSFGetGraph(sfPoint, NULL, &numLeaves, &leafLocal, &leafRemote));
712   PetscCall(DMCreateLabel(dmNew, "vtk"));
713   PetscCall(DMGetLabel(dmNew, "vtk", &vtkLabel));
714   if (createGhostLabel) {
715     PetscCall(DMCreateLabel(dmNew, "ghost"));
716     PetscCall(DMGetLabel(dmNew, "ghost", &ghostLabel));
717   }
718   for (l = 0, c = cStart; l < numLeaves && c < cEnd; ++l, ++c) {
719     for (; c < leafLocal[l] && c < cEnd; ++c) PetscCall(DMLabelSetValue(vtkLabel, c, 1));
720     if (leafLocal[l] >= cEnd) break;
721     if (leafRemote[l].rank == rank) {
722       PetscCall(DMLabelSetValue(vtkLabel, c, 1));
723     } else if (ghostLabel) PetscCall(DMLabelSetValue(ghostLabel, c, 2));
724   }
725   for (; c < cEnd; ++c) PetscCall(DMLabelSetValue(vtkLabel, c, 1));
726   if (ghostLabel) {
727     PetscCall(DMPlexGetHeightStratum(dmNew, 1, &fStart, &fEnd));
728     for (f = fStart; f < fEnd; ++f) {
729       PetscInt numCells;
730 
731       PetscCall(DMPlexGetSupportSize(dmNew, f, &numCells));
732       if (numCells < 2) {
733         PetscCall(DMLabelSetValue(ghostLabel, f, 1));
734       } else {
735         const PetscInt *cells = NULL;
736         PetscInt        vA, vB;
737 
738         PetscCall(DMPlexGetSupport(dmNew, f, &cells));
739         PetscCall(DMLabelGetValue(vtkLabel, cells[0], &vA));
740         PetscCall(DMLabelGetValue(vtkLabel, cells[1], &vB));
741         if (vA != 1 && vB != 1) PetscCall(DMLabelSetValue(ghostLabel, f, 1));
742       }
743     }
744   }
745   PetscFunctionReturn(0);
746 }
747 
748 static PetscErrorCode DMPlexShiftTree_Internal(DM dm, PetscInt depthShift[], DM dmNew)
749 {
750   DM           refTree;
751   PetscSection pSec;
752   PetscInt    *parents, *childIDs;
753 
754   PetscFunctionBegin;
755   PetscCall(DMPlexGetReferenceTree(dm, &refTree));
756   PetscCall(DMPlexSetReferenceTree(dmNew, refTree));
757   PetscCall(DMPlexGetTree(dm, &pSec, &parents, &childIDs, NULL, NULL));
758   if (pSec) {
759     PetscInt     p, pStart, pEnd, *parentsShifted, pStartShifted, pEndShifted, depth;
760     PetscInt    *childIDsShifted;
761     PetscSection pSecShifted;
762 
763     PetscCall(PetscSectionGetChart(pSec, &pStart, &pEnd));
764     PetscCall(DMPlexGetDepth(dm, &depth));
765     pStartShifted = DMPlexShiftPoint_Internal(pStart, depth, depthShift);
766     pEndShifted   = DMPlexShiftPoint_Internal(pEnd, depth, depthShift);
767     PetscCall(PetscMalloc2(pEndShifted - pStartShifted, &parentsShifted, pEndShifted - pStartShifted, &childIDsShifted));
768     PetscCall(PetscSectionCreate(PetscObjectComm((PetscObject)dmNew), &pSecShifted));
769     PetscCall(PetscSectionSetChart(pSecShifted, pStartShifted, pEndShifted));
770     for (p = pStartShifted; p < pEndShifted; p++) {
771       /* start off assuming no children */
772       PetscCall(PetscSectionSetDof(pSecShifted, p, 0));
773     }
774     for (p = pStart; p < pEnd; p++) {
775       PetscInt dof;
776       PetscInt pNew = DMPlexShiftPoint_Internal(p, depth, depthShift);
777 
778       PetscCall(PetscSectionGetDof(pSec, p, &dof));
779       PetscCall(PetscSectionSetDof(pSecShifted, pNew, dof));
780     }
781     PetscCall(PetscSectionSetUp(pSecShifted));
782     for (p = pStart; p < pEnd; p++) {
783       PetscInt dof;
784       PetscInt pNew = DMPlexShiftPoint_Internal(p, depth, depthShift);
785 
786       PetscCall(PetscSectionGetDof(pSec, p, &dof));
787       if (dof) {
788         PetscInt off, offNew;
789 
790         PetscCall(PetscSectionGetOffset(pSec, p, &off));
791         PetscCall(PetscSectionGetOffset(pSecShifted, pNew, &offNew));
792         parentsShifted[offNew]  = DMPlexShiftPoint_Internal(parents[off], depth, depthShift);
793         childIDsShifted[offNew] = childIDs[off];
794       }
795     }
796     PetscCall(DMPlexSetTree(dmNew, pSecShifted, parentsShifted, childIDsShifted));
797     PetscCall(PetscFree2(parentsShifted, childIDsShifted));
798     PetscCall(PetscSectionDestroy(&pSecShifted));
799   }
800   PetscFunctionReturn(0);
801 }
802 
803 static PetscErrorCode DMPlexConstructGhostCells_Internal(DM dm, DMLabel label, PetscInt *numGhostCells, DM gdm)
804 {
805   PetscSF          sf;
806   IS               valueIS;
807   const PetscInt  *values, *leaves;
808   PetscInt        *depthShift;
809   PetscInt         d, depth = 0, nleaves, loc, Ng, numFS, fs, fStart, fEnd, ghostCell, cEnd, c;
810   const PetscReal *maxCell, *Lstart, *L;
811 
812   PetscFunctionBegin;
813   PetscCall(DMGetPointSF(dm, &sf));
814   PetscCall(PetscSFGetGraph(sf, NULL, &nleaves, &leaves, NULL));
815   nleaves = PetscMax(0, nleaves);
816   PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
817   /* Count ghost cells */
818   PetscCall(DMLabelGetValueIS(label, &valueIS));
819   PetscCall(ISGetLocalSize(valueIS, &numFS));
820   PetscCall(ISGetIndices(valueIS, &values));
821   Ng = 0;
822   for (fs = 0; fs < numFS; ++fs) {
823     IS              faceIS;
824     const PetscInt *faces;
825     PetscInt        numFaces, f, numBdFaces = 0;
826 
827     PetscCall(DMLabelGetStratumIS(label, values[fs], &faceIS));
828     PetscCall(ISGetLocalSize(faceIS, &numFaces));
829     PetscCall(ISGetIndices(faceIS, &faces));
830     for (f = 0; f < numFaces; ++f) {
831       PetscInt numChildren;
832 
833       PetscCall(PetscFindInt(faces[f], nleaves, leaves, &loc));
834       PetscCall(DMPlexGetTreeChildren(dm, faces[f], &numChildren, NULL));
835       /* non-local and ancestors points don't get to register ghosts */
836       if (loc >= 0 || numChildren) continue;
837       if ((faces[f] >= fStart) && (faces[f] < fEnd)) ++numBdFaces;
838     }
839     Ng += numBdFaces;
840     PetscCall(ISRestoreIndices(faceIS, &faces));
841     PetscCall(ISDestroy(&faceIS));
842   }
843   PetscCall(DMPlexGetDepth(dm, &depth));
844   PetscCall(PetscMalloc1(2 * (depth + 1), &depthShift));
845   for (d = 0; d <= depth; d++) {
846     PetscInt dEnd;
847 
848     PetscCall(DMPlexGetDepthStratum(dm, d, NULL, &dEnd));
849     depthShift[2 * d]     = dEnd;
850     depthShift[2 * d + 1] = 0;
851   }
852   if (depth >= 0) depthShift[2 * depth + 1] = Ng;
853   PetscCall(DMPlexShiftPointSetUp_Internal(depth, depthShift));
854   PetscCall(DMPlexShiftSizes_Internal(dm, depthShift, gdm));
855   /* Step 3: Set cone/support sizes for new points */
856   PetscCall(DMPlexGetHeightStratum(dm, 0, NULL, &cEnd));
857   for (c = cEnd; c < cEnd + Ng; ++c) PetscCall(DMPlexSetConeSize(gdm, c, 1));
858   for (fs = 0; fs < numFS; ++fs) {
859     IS              faceIS;
860     const PetscInt *faces;
861     PetscInt        numFaces, f;
862 
863     PetscCall(DMLabelGetStratumIS(label, values[fs], &faceIS));
864     PetscCall(ISGetLocalSize(faceIS, &numFaces));
865     PetscCall(ISGetIndices(faceIS, &faces));
866     for (f = 0; f < numFaces; ++f) {
867       PetscInt size, numChildren;
868 
869       PetscCall(PetscFindInt(faces[f], nleaves, leaves, &loc));
870       PetscCall(DMPlexGetTreeChildren(dm, faces[f], &numChildren, NULL));
871       if (loc >= 0 || numChildren) continue;
872       if ((faces[f] < fStart) || (faces[f] >= fEnd)) continue;
873       PetscCall(DMPlexGetSupportSize(dm, faces[f], &size));
874       PetscCheck(size == 1, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM has boundary face %" PetscInt_FMT " with %" PetscInt_FMT " support cells", faces[f], size);
875       PetscCall(DMPlexSetSupportSize(gdm, faces[f] + Ng, 2));
876     }
877     PetscCall(ISRestoreIndices(faceIS, &faces));
878     PetscCall(ISDestroy(&faceIS));
879   }
880   /* Step 4: Setup ghosted DM */
881   PetscCall(DMSetUp(gdm));
882   PetscCall(DMPlexShiftPoints_Internal(dm, depthShift, gdm));
883   /* Step 6: Set cones and supports for new points */
884   ghostCell = cEnd;
885   for (fs = 0; fs < numFS; ++fs) {
886     IS              faceIS;
887     const PetscInt *faces;
888     PetscInt        numFaces, f;
889 
890     PetscCall(DMLabelGetStratumIS(label, values[fs], &faceIS));
891     PetscCall(ISGetLocalSize(faceIS, &numFaces));
892     PetscCall(ISGetIndices(faceIS, &faces));
893     for (f = 0; f < numFaces; ++f) {
894       PetscInt newFace = faces[f] + Ng, numChildren;
895 
896       PetscCall(PetscFindInt(faces[f], nleaves, leaves, &loc));
897       PetscCall(DMPlexGetTreeChildren(dm, faces[f], &numChildren, NULL));
898       if (loc >= 0 || numChildren) continue;
899       if ((faces[f] < fStart) || (faces[f] >= fEnd)) continue;
900       PetscCall(DMPlexSetCone(gdm, ghostCell, &newFace));
901       PetscCall(DMPlexInsertSupport(gdm, newFace, 1, ghostCell));
902       ++ghostCell;
903     }
904     PetscCall(ISRestoreIndices(faceIS, &faces));
905     PetscCall(ISDestroy(&faceIS));
906   }
907   PetscCall(ISRestoreIndices(valueIS, &values));
908   PetscCall(ISDestroy(&valueIS));
909   PetscCall(DMPlexShiftCoordinates_Internal(dm, depthShift, gdm));
910   PetscCall(DMPlexShiftSF_Internal(dm, depthShift, gdm));
911   PetscCall(DMPlexShiftLabels_Internal(dm, depthShift, gdm));
912   PetscCall(DMPlexCreateVTKLabel_Internal(dm, PETSC_TRUE, gdm));
913   PetscCall(DMPlexShiftTree_Internal(dm, depthShift, gdm));
914   PetscCall(PetscFree(depthShift));
915   for (c = cEnd; c < cEnd + Ng; ++c) PetscCall(DMPlexSetCellType(gdm, c, DM_POLYTOPE_FV_GHOST));
916   /* Step 7: Periodicity */
917   PetscCall(DMGetPeriodicity(dm, &maxCell, &Lstart, &L));
918   PetscCall(DMSetPeriodicity(gdm, maxCell, Lstart, L));
919   if (numGhostCells) *numGhostCells = Ng;
920   PetscFunctionReturn(0);
921 }
922 
923 /*@C
924   DMPlexConstructGhostCells - Construct ghost cells which connect to every boundary face
925 
926   Collective on dm
927 
928   Input Parameters:
929 + dm - The original DM
930 - labelName - The label specifying the boundary faces, or "Face Sets" if this is NULL
931 
932   Output Parameters:
933 + numGhostCells - The number of ghost cells added to the DM
934 - dmGhosted - The new DM
935 
936   Note: If no label exists of that name, one will be created marking all boundary faces
937 
938   Level: developer
939 
940 .seealso: `DMCreate()`
941 @*/
942 PetscErrorCode DMPlexConstructGhostCells(DM dm, const char labelName[], PetscInt *numGhostCells, DM *dmGhosted)
943 {
944   DM          gdm;
945   DMLabel     label;
946   const char *name = labelName ? labelName : "Face Sets";
947   PetscInt    dim, Ng = 0;
948   PetscBool   useCone, useClosure;
949 
950   PetscFunctionBegin;
951   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
952   if (numGhostCells) PetscValidIntPointer(numGhostCells, 3);
953   PetscValidPointer(dmGhosted, 4);
954   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &gdm));
955   PetscCall(DMSetType(gdm, DMPLEX));
956   PetscCall(DMGetDimension(dm, &dim));
957   PetscCall(DMSetDimension(gdm, dim));
958   PetscCall(DMGetBasicAdjacency(dm, &useCone, &useClosure));
959   PetscCall(DMSetBasicAdjacency(gdm, useCone, useClosure));
960   PetscCall(DMGetLabel(dm, name, &label));
961   if (!label) {
962     /* Get label for boundary faces */
963     PetscCall(DMCreateLabel(dm, name));
964     PetscCall(DMGetLabel(dm, name, &label));
965     PetscCall(DMPlexMarkBoundaryFaces(dm, 1, label));
966   }
967   PetscCall(DMPlexConstructGhostCells_Internal(dm, label, &Ng, gdm));
968   PetscCall(DMCopyDisc(dm, gdm));
969   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_TRUE, gdm));
970   gdm->setfromoptionscalled = dm->setfromoptionscalled;
971   if (numGhostCells) *numGhostCells = Ng;
972   *dmGhosted = gdm;
973   PetscFunctionReturn(0);
974 }
975 
976 static PetscErrorCode DivideCells_Private(DM dm, DMLabel label, DMPlexPointQueue queue)
977 {
978   PetscInt dim, d, shift = 100, *pStart, *pEnd;
979 
980   PetscFunctionBegin;
981   PetscCall(DMGetDimension(dm, &dim));
982   PetscCall(PetscMalloc2(dim, &pStart, dim, &pEnd));
983   for (d = 0; d < dim; ++d) PetscCall(DMPlexGetDepthStratum(dm, d, &pStart[d], &pEnd[d]));
984   while (!DMPlexPointQueueEmpty(queue)) {
985     PetscInt  cell    = -1;
986     PetscInt *closure = NULL;
987     PetscInt  closureSize, cl, cval;
988 
989     PetscCall(DMPlexPointQueueDequeue(queue, &cell));
990     PetscCall(DMLabelGetValue(label, cell, &cval));
991     PetscCall(DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
992     /* Mark points in the cell closure that touch the fault */
993     for (d = 0; d < dim; ++d) {
994       for (cl = 0; cl < closureSize * 2; cl += 2) {
995         const PetscInt clp = closure[cl];
996         PetscInt       clval;
997 
998         if ((clp < pStart[d]) || (clp >= pEnd[d])) continue;
999         PetscCall(DMLabelGetValue(label, clp, &clval));
1000         if (clval == -1) {
1001           const PetscInt *cone;
1002           PetscInt        coneSize, c;
1003 
1004           /* If a cone point touches the fault, then this point touches the fault */
1005           PetscCall(DMPlexGetCone(dm, clp, &cone));
1006           PetscCall(DMPlexGetConeSize(dm, clp, &coneSize));
1007           for (c = 0; c < coneSize; ++c) {
1008             PetscInt cpval;
1009 
1010             PetscCall(DMLabelGetValue(label, cone[c], &cpval));
1011             if (cpval != -1) {
1012               PetscInt dep;
1013 
1014               PetscCall(DMPlexGetPointDepth(dm, clp, &dep));
1015               clval = cval < 0 ? -(shift + dep) : shift + dep;
1016               PetscCall(DMLabelSetValue(label, clp, clval));
1017               break;
1018             }
1019           }
1020         }
1021         /* Mark neighbor cells through marked faces (these cells must also touch the fault) */
1022         if (d == dim - 1 && clval != -1) {
1023           const PetscInt *support;
1024           PetscInt        supportSize, s, nval;
1025 
1026           PetscCall(DMPlexGetSupport(dm, clp, &support));
1027           PetscCall(DMPlexGetSupportSize(dm, clp, &supportSize));
1028           for (s = 0; s < supportSize; ++s) {
1029             PetscCall(DMLabelGetValue(label, support[s], &nval));
1030             if (nval == -1) {
1031               PetscCall(DMLabelSetValue(label, support[s], clval < 0 ? clval - 1 : clval + 1));
1032               PetscCall(DMPlexPointQueueEnqueue(queue, support[s]));
1033             }
1034           }
1035         }
1036       }
1037     }
1038     PetscCall(DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
1039   }
1040   PetscCall(PetscFree2(pStart, pEnd));
1041   PetscFunctionReturn(0);
1042 }
1043 
1044 typedef struct {
1045   DM               dm;
1046   DMPlexPointQueue queue;
1047 } PointDivision;
1048 
1049 static PetscErrorCode divideCell(DMLabel label, PetscInt p, PetscInt val, void *ctx)
1050 {
1051   PointDivision  *div  = (PointDivision *)ctx;
1052   PetscInt        cval = val < 0 ? val - 1 : val + 1;
1053   const PetscInt *support;
1054   PetscInt        supportSize, s;
1055 
1056   PetscFunctionBegin;
1057   PetscCall(DMPlexGetSupport(div->dm, p, &support));
1058   PetscCall(DMPlexGetSupportSize(div->dm, p, &supportSize));
1059   for (s = 0; s < supportSize; ++s) {
1060     PetscCall(DMLabelSetValue(label, support[s], cval));
1061     PetscCall(DMPlexPointQueueEnqueue(div->queue, support[s]));
1062   }
1063   PetscFunctionReturn(0);
1064 }
1065 
1066 /* Mark cells by label propagation */
1067 static PetscErrorCode DMPlexLabelFaultHalo(DM dm, DMLabel faultLabel)
1068 {
1069   DMPlexPointQueue queue = NULL;
1070   PointDivision    div;
1071   PetscSF          pointSF;
1072   IS               pointIS;
1073   const PetscInt  *points;
1074   PetscBool        empty;
1075   PetscInt         dim, shift = 100, n, i;
1076 
1077   PetscFunctionBegin;
1078   PetscCall(DMGetDimension(dm, &dim));
1079   PetscCall(DMPlexPointQueueCreate(1024, &queue));
1080   div.dm    = dm;
1081   div.queue = queue;
1082   /* Enqueue cells on fault */
1083   PetscCall(DMLabelGetStratumIS(faultLabel, shift + dim, &pointIS));
1084   if (pointIS) {
1085     PetscCall(ISGetLocalSize(pointIS, &n));
1086     PetscCall(ISGetIndices(pointIS, &points));
1087     for (i = 0; i < n; ++i) PetscCall(DMPlexPointQueueEnqueue(queue, points[i]));
1088     PetscCall(ISRestoreIndices(pointIS, &points));
1089     PetscCall(ISDestroy(&pointIS));
1090   }
1091   PetscCall(DMLabelGetStratumIS(faultLabel, -(shift + dim), &pointIS));
1092   if (pointIS) {
1093     PetscCall(ISGetLocalSize(pointIS, &n));
1094     PetscCall(ISGetIndices(pointIS, &points));
1095     for (i = 0; i < n; ++i) PetscCall(DMPlexPointQueueEnqueue(queue, points[i]));
1096     PetscCall(ISRestoreIndices(pointIS, &points));
1097     PetscCall(ISDestroy(&pointIS));
1098   }
1099 
1100   PetscCall(DMGetPointSF(dm, &pointSF));
1101   PetscCall(DMLabelPropagateBegin(faultLabel, pointSF));
1102   /* While edge queue is not empty: */
1103   PetscCall(DMPlexPointQueueEmptyCollective((PetscObject)dm, queue, &empty));
1104   while (!empty) {
1105     PetscCall(DivideCells_Private(dm, faultLabel, queue));
1106     PetscCall(DMLabelPropagatePush(faultLabel, pointSF, divideCell, &div));
1107     PetscCall(DMPlexPointQueueEmptyCollective((PetscObject)dm, queue, &empty));
1108   }
1109   PetscCall(DMLabelPropagateEnd(faultLabel, pointSF));
1110   PetscCall(DMPlexPointQueueDestroy(&queue));
1111   PetscFunctionReturn(0);
1112 }
1113 
1114 /*
1115   We are adding three kinds of points here:
1116     Replicated:     Copies of points which exist in the mesh, such as vertices identified across a fault
1117     Non-replicated: Points which exist on the fault, but are not replicated
1118     Ghost:          These are shared fault faces which are not owned by this process. These do not produce hybrid cells and do not replicate
1119     Hybrid:         Entirely new points, such as cohesive cells
1120 
1121   When creating subsequent cohesive cells, we shift the old hybrid cells to the end of the numbering at
1122   each depth so that the new split/hybrid points can be inserted as a block.
1123 */
1124 static PetscErrorCode DMPlexConstructCohesiveCells_Internal(DM dm, DMLabel label, DMLabel splitLabel, DM sdm)
1125 {
1126   MPI_Comm         comm;
1127   IS               valueIS;
1128   PetscInt         numSP = 0; /* The number of depths for which we have replicated points */
1129   const PetscInt  *values;    /* List of depths for which we have replicated points */
1130   IS              *splitIS;
1131   IS              *unsplitIS;
1132   IS               ghostIS;
1133   PetscInt        *numSplitPoints;     /* The number of replicated points at each depth */
1134   PetscInt        *numUnsplitPoints;   /* The number of non-replicated points at each depth which still give rise to hybrid points */
1135   PetscInt        *numHybridPoints;    /* The number of new hybrid points at each depth */
1136   PetscInt        *numHybridPointsOld; /* The number of existing hybrid points at each depth */
1137   PetscInt         numGhostPoints;     /* The number of unowned, shared fault faces */
1138   const PetscInt **splitPoints;        /* Replicated points for each depth */
1139   const PetscInt **unsplitPoints;      /* Non-replicated points for each depth */
1140   const PetscInt  *ghostPoints;        /* Ghost fault faces */
1141   PetscSection     coordSection;
1142   Vec              coordinates;
1143   PetscScalar     *coords;
1144   PetscInt        *depthMax;   /* The first hybrid point at each depth in the original mesh */
1145   PetscInt        *depthEnd;   /* The point limit at each depth in the original mesh */
1146   PetscInt        *depthShift; /* Number of replicated+hybrid points at each depth */
1147   PetscInt        *pMaxNew;    /* The first replicated point at each depth in the new mesh, hybrids come after this */
1148   PetscInt        *coneNew, *coneONew, *supportNew;
1149   PetscInt         shift = 100, shift2 = 200, depth = 0, dep, dim, d, sp, maxConeSize, maxSupportSize, maxConeSizeNew, maxSupportSizeNew, numLabels, vStart, vEnd, pEnd, p, v;
1150 
1151   PetscFunctionBegin;
1152   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
1153   PetscCall(DMGetDimension(dm, &dim));
1154   PetscCall(DMPlexGetDepth(dm, &depth));
1155   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
1156   /* We do not want this label automatically computed, instead we compute it here */
1157   PetscCall(DMCreateLabel(sdm, "celltype"));
1158   /* Count split points and add cohesive cells */
1159   PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize));
1160   PetscCall(PetscMalloc5(depth + 1, &depthMax, depth + 1, &depthEnd, 2 * (depth + 1), &depthShift, depth + 1, &pMaxNew, depth + 1, &numHybridPointsOld));
1161   PetscCall(PetscMalloc7(depth + 1, &splitIS, depth + 1, &unsplitIS, depth + 1, &numSplitPoints, depth + 1, &numUnsplitPoints, depth + 1, &numHybridPoints, depth + 1, &splitPoints, depth + 1, &unsplitPoints));
1162   for (d = 0; d <= depth; ++d) {
1163     PetscCall(DMPlexGetDepthStratum(dm, d, NULL, &pMaxNew[d]));
1164     PetscCall(DMPlexGetTensorPrismBounds_Internal(dm, d, &depthMax[d], NULL));
1165     depthEnd[d]           = pMaxNew[d];
1166     depthMax[d]           = depthMax[d] < 0 ? depthEnd[d] : depthMax[d];
1167     numSplitPoints[d]     = 0;
1168     numUnsplitPoints[d]   = 0;
1169     numHybridPoints[d]    = 0;
1170     numHybridPointsOld[d] = depthMax[d] < 0 ? 0 : depthEnd[d] - depthMax[d];
1171     splitPoints[d]        = NULL;
1172     unsplitPoints[d]      = NULL;
1173     splitIS[d]            = NULL;
1174     unsplitIS[d]          = NULL;
1175     /* we are shifting the existing hybrid points with the stratum behind them, so
1176      * the split comes at the end of the normal points, i.e., at depthMax[d] */
1177     depthShift[2 * d]     = depthMax[d];
1178     depthShift[2 * d + 1] = 0;
1179   }
1180   numGhostPoints = 0;
1181   ghostPoints    = NULL;
1182   if (label) {
1183     PetscCall(DMLabelGetValueIS(label, &valueIS));
1184     PetscCall(ISGetLocalSize(valueIS, &numSP));
1185     PetscCall(ISGetIndices(valueIS, &values));
1186   }
1187   for (sp = 0; sp < numSP; ++sp) {
1188     const PetscInt dep = values[sp];
1189 
1190     if ((dep < 0) || (dep > depth)) continue;
1191     PetscCall(DMLabelGetStratumIS(label, dep, &splitIS[dep]));
1192     if (splitIS[dep]) {
1193       PetscCall(ISGetLocalSize(splitIS[dep], &numSplitPoints[dep]));
1194       PetscCall(ISGetIndices(splitIS[dep], &splitPoints[dep]));
1195     }
1196     PetscCall(DMLabelGetStratumIS(label, shift2 + dep, &unsplitIS[dep]));
1197     if (unsplitIS[dep]) {
1198       PetscCall(ISGetLocalSize(unsplitIS[dep], &numUnsplitPoints[dep]));
1199       PetscCall(ISGetIndices(unsplitIS[dep], &unsplitPoints[dep]));
1200     }
1201   }
1202   PetscCall(DMLabelGetStratumIS(label, shift2 + dim - 1, &ghostIS));
1203   if (ghostIS) {
1204     PetscCall(ISGetLocalSize(ghostIS, &numGhostPoints));
1205     PetscCall(ISGetIndices(ghostIS, &ghostPoints));
1206   }
1207   /* Calculate number of hybrid points */
1208   for (d = 1; d <= depth; ++d) numHybridPoints[d] = numSplitPoints[d - 1] + numUnsplitPoints[d - 1]; /* There is a hybrid cell/face/edge for every split face/edge/vertex   */
1209   for (d = 0; d <= depth; ++d) depthShift[2 * d + 1] = numSplitPoints[d] + numHybridPoints[d];
1210   PetscCall(DMPlexShiftPointSetUp_Internal(depth, depthShift));
1211   /* the end of the points in this stratum that come before the new points:
1212    * shifting pMaxNew[d] gets the new start of the next stratum, then count back the old hybrid points and the newly
1213    * added points */
1214   for (d = 0; d <= depth; ++d) pMaxNew[d] = DMPlexShiftPoint_Internal(pMaxNew[d], depth, depthShift) - (numHybridPointsOld[d] + numSplitPoints[d] + numHybridPoints[d]);
1215   PetscCall(DMPlexShiftSizes_Internal(dm, depthShift, sdm));
1216   /* Step 3: Set cone/support sizes for new points */
1217   for (dep = 0; dep <= depth; ++dep) {
1218     for (p = 0; p < numSplitPoints[dep]; ++p) {
1219       const PetscInt  oldp   = splitPoints[dep][p];
1220       const PetscInt  newp   = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*oldp + depthOffset[dep]*/;
1221       const PetscInt  splitp = p + pMaxNew[dep];
1222       const PetscInt *support;
1223       DMPolytopeType  ct;
1224       PetscInt        coneSize, supportSize, qf, qn, qp, e;
1225 
1226       PetscCall(DMPlexGetConeSize(dm, oldp, &coneSize));
1227       PetscCall(DMPlexSetConeSize(sdm, splitp, coneSize));
1228       PetscCall(DMPlexGetSupportSize(dm, oldp, &supportSize));
1229       PetscCall(DMPlexSetSupportSize(sdm, splitp, supportSize));
1230       PetscCall(DMPlexGetCellType(dm, oldp, &ct));
1231       PetscCall(DMPlexSetCellType(sdm, splitp, ct));
1232       if (dep == depth - 1) {
1233         const PetscInt hybcell = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1];
1234 
1235         /* Add cohesive cells, they are prisms */
1236         PetscCall(DMPlexSetConeSize(sdm, hybcell, 2 + coneSize));
1237         switch (coneSize) {
1238         case 2:
1239           PetscCall(DMPlexSetCellType(sdm, hybcell, DM_POLYTOPE_SEG_PRISM_TENSOR));
1240           break;
1241         case 3:
1242           PetscCall(DMPlexSetCellType(sdm, hybcell, DM_POLYTOPE_TRI_PRISM_TENSOR));
1243           break;
1244         case 4:
1245           PetscCall(DMPlexSetCellType(sdm, hybcell, DM_POLYTOPE_QUAD_PRISM_TENSOR));
1246           break;
1247         }
1248         /* Shared fault faces with only one support cell now have two with the cohesive cell */
1249         /*   TODO Check thaat oldp has rootdegree == 1 */
1250         if (supportSize == 1) {
1251           const PetscInt *support;
1252           PetscInt        val;
1253 
1254           PetscCall(DMPlexGetSupport(dm, oldp, &support));
1255           PetscCall(DMLabelGetValue(label, support[0], &val));
1256           if (val < 0) PetscCall(DMPlexSetSupportSize(sdm, splitp, 2));
1257           else PetscCall(DMPlexSetSupportSize(sdm, newp, 2));
1258         }
1259       } else if (dep == 0) {
1260         const PetscInt hybedge = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1];
1261 
1262         PetscCall(DMPlexGetSupport(dm, oldp, &support));
1263         for (e = 0, qn = 0, qp = 0, qf = 0; e < supportSize; ++e) {
1264           PetscInt val;
1265 
1266           PetscCall(DMLabelGetValue(label, support[e], &val));
1267           if (val == 1) ++qf;
1268           if ((val == 1) || (val == (shift + 1))) ++qn;
1269           if ((val == 1) || (val == -(shift + 1))) ++qp;
1270         }
1271         /* Split old vertex: Edges into original vertex and new cohesive edge */
1272         PetscCall(DMPlexSetSupportSize(sdm, newp, qn + 1));
1273         /* Split new vertex: Edges into split vertex and new cohesive edge */
1274         PetscCall(DMPlexSetSupportSize(sdm, splitp, qp + 1));
1275         /* Add hybrid edge */
1276         PetscCall(DMPlexSetConeSize(sdm, hybedge, 2));
1277         PetscCall(DMPlexSetSupportSize(sdm, hybedge, qf));
1278         PetscCall(DMPlexSetCellType(sdm, hybedge, DM_POLYTOPE_POINT_PRISM_TENSOR));
1279       } else if (dep == dim - 2) {
1280         const PetscInt hybface = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1];
1281 
1282         PetscCall(DMPlexGetSupport(dm, oldp, &support));
1283         for (e = 0, qn = 0, qp = 0, qf = 0; e < supportSize; ++e) {
1284           PetscInt val;
1285 
1286           PetscCall(DMLabelGetValue(label, support[e], &val));
1287           if (val == dim - 1) ++qf;
1288           if ((val == dim - 1) || (val == (shift + dim - 1))) ++qn;
1289           if ((val == dim - 1) || (val == -(shift + dim - 1))) ++qp;
1290         }
1291         /* Split old edge: Faces into original edge and cohesive face (positive side?) */
1292         PetscCall(DMPlexSetSupportSize(sdm, newp, qn + 1));
1293         /* Split new edge: Faces into split edge and cohesive face (negative side?) */
1294         PetscCall(DMPlexSetSupportSize(sdm, splitp, qp + 1));
1295         /* Add hybrid face */
1296         PetscCall(DMPlexSetConeSize(sdm, hybface, 4));
1297         PetscCall(DMPlexSetSupportSize(sdm, hybface, qf));
1298         PetscCall(DMPlexSetCellType(sdm, hybface, DM_POLYTOPE_SEG_PRISM_TENSOR));
1299       }
1300     }
1301   }
1302   for (dep = 0; dep <= depth; ++dep) {
1303     for (p = 0; p < numUnsplitPoints[dep]; ++p) {
1304       const PetscInt  oldp = unsplitPoints[dep][p];
1305       const PetscInt  newp = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*oldp + depthOffset[dep]*/;
1306       const PetscInt *support;
1307       PetscInt        coneSize, supportSize, qf, e, s;
1308 
1309       PetscCall(DMPlexGetConeSize(dm, oldp, &coneSize));
1310       PetscCall(DMPlexGetSupportSize(dm, oldp, &supportSize));
1311       PetscCall(DMPlexGetSupport(dm, oldp, &support));
1312       if (dep == 0) {
1313         const PetscInt hybedge = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1] + numSplitPoints[dep];
1314 
1315         /* Unsplit vertex: Edges into original vertex, split edges, and new cohesive edge twice */
1316         for (s = 0, qf = 0; s < supportSize; ++s, ++qf) {
1317           PetscCall(PetscFindInt(support[s], numSplitPoints[dep + 1], splitPoints[dep + 1], &e));
1318           if (e >= 0) ++qf;
1319         }
1320         PetscCall(DMPlexSetSupportSize(sdm, newp, qf + 2));
1321         /* Add hybrid edge */
1322         PetscCall(DMPlexSetConeSize(sdm, hybedge, 2));
1323         for (e = 0, qf = 0; e < supportSize; ++e) {
1324           PetscInt val;
1325 
1326           PetscCall(DMLabelGetValue(label, support[e], &val));
1327           /* Split and unsplit edges produce hybrid faces */
1328           if (val == 1) ++qf;
1329           if (val == (shift2 + 1)) ++qf;
1330         }
1331         PetscCall(DMPlexSetSupportSize(sdm, hybedge, qf));
1332         PetscCall(DMPlexSetCellType(sdm, hybedge, DM_POLYTOPE_POINT_PRISM_TENSOR));
1333       } else if (dep == dim - 2) {
1334         const PetscInt hybface = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1] + numSplitPoints[dep];
1335         PetscInt       val;
1336 
1337         for (e = 0, qf = 0; e < supportSize; ++e) {
1338           PetscCall(DMLabelGetValue(label, support[e], &val));
1339           if (val == dim - 1) qf += 2;
1340           else ++qf;
1341         }
1342         /* Unsplit edge: Faces into original edge, split face, and cohesive face twice */
1343         PetscCall(DMPlexSetSupportSize(sdm, newp, qf + 2));
1344         /* Add hybrid face */
1345         for (e = 0, qf = 0; e < supportSize; ++e) {
1346           PetscCall(DMLabelGetValue(label, support[e], &val));
1347           if (val == dim - 1) ++qf;
1348         }
1349         PetscCall(DMPlexSetConeSize(sdm, hybface, 4));
1350         PetscCall(DMPlexSetSupportSize(sdm, hybface, qf));
1351         PetscCall(DMPlexSetCellType(sdm, hybface, DM_POLYTOPE_SEG_PRISM_TENSOR));
1352       }
1353     }
1354   }
1355   /* Step 4: Setup split DM */
1356   PetscCall(DMSetUp(sdm));
1357   PetscCall(DMPlexShiftPoints_Internal(dm, depthShift, sdm));
1358   PetscCall(DMPlexGetMaxSizes(sdm, &maxConeSizeNew, &maxSupportSizeNew));
1359   PetscCall(PetscMalloc3(PetscMax(maxConeSize, maxConeSizeNew) * 3, &coneNew, PetscMax(maxConeSize, maxConeSizeNew) * 3, &coneONew, PetscMax(maxSupportSize, maxSupportSizeNew), &supportNew));
1360   /* Step 6: Set cones and supports for new points */
1361   for (dep = 0; dep <= depth; ++dep) {
1362     for (p = 0; p < numSplitPoints[dep]; ++p) {
1363       const PetscInt  oldp   = splitPoints[dep][p];
1364       const PetscInt  newp   = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*oldp + depthOffset[dep]*/;
1365       const PetscInt  splitp = p + pMaxNew[dep];
1366       const PetscInt *cone, *support, *ornt;
1367       DMPolytopeType  ct;
1368       PetscInt        coneSize, supportSize, q, qf, qn, qp, v, e, s;
1369 
1370       PetscCall(DMPlexGetCellType(dm, oldp, &ct));
1371       PetscCall(DMPlexGetConeSize(dm, oldp, &coneSize));
1372       PetscCall(DMPlexGetCone(dm, oldp, &cone));
1373       PetscCall(DMPlexGetConeOrientation(dm, oldp, &ornt));
1374       PetscCall(DMPlexGetSupportSize(dm, oldp, &supportSize));
1375       PetscCall(DMPlexGetSupport(dm, oldp, &support));
1376       if (dep == depth - 1) {
1377         PetscBool       hasUnsplit = PETSC_FALSE;
1378         const PetscInt  hybcell    = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1];
1379         const PetscInt *supportF;
1380 
1381         coneONew[0] = coneONew[1] = -1000;
1382         /* Split face:       copy in old face to new face to start */
1383         PetscCall(DMPlexGetSupport(sdm, newp, &supportF));
1384         PetscCall(DMPlexSetSupport(sdm, splitp, supportF));
1385         /* Split old face:   old vertices/edges in cone so no change */
1386         /* Split new face:   new vertices/edges in cone */
1387         for (q = 0; q < coneSize; ++q) {
1388           PetscCall(PetscFindInt(cone[q], numSplitPoints[dep - 1], splitPoints[dep - 1], &v));
1389           if (v < 0) {
1390             PetscCall(PetscFindInt(cone[q], numUnsplitPoints[dep - 1], unsplitPoints[dep - 1], &v));
1391             PetscCheck(v >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %" PetscInt_FMT " in split or unsplit points of depth %" PetscInt_FMT, cone[q], dep - 1);
1392             coneNew[2 + q] = DMPlexShiftPoint_Internal(cone[q], depth, depthShift) /*cone[q] + depthOffset[dep-1]*/;
1393             hasUnsplit     = PETSC_TRUE;
1394           } else {
1395             coneNew[2 + q] = v + pMaxNew[dep - 1];
1396             if (dep > 1) {
1397               const PetscInt *econe;
1398               PetscInt        econeSize, r, vs, vu;
1399 
1400               PetscCall(DMPlexGetConeSize(dm, cone[q], &econeSize));
1401               PetscCall(DMPlexGetCone(dm, cone[q], &econe));
1402               for (r = 0; r < econeSize; ++r) {
1403                 PetscCall(PetscFindInt(econe[r], numSplitPoints[dep - 2], splitPoints[dep - 2], &vs));
1404                 PetscCall(PetscFindInt(econe[r], numUnsplitPoints[dep - 2], unsplitPoints[dep - 2], &vu));
1405                 if (vs >= 0) continue;
1406                 PetscCheck(vu >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %" PetscInt_FMT " in split or unsplit points of depth %" PetscInt_FMT, econe[r], dep - 2);
1407                 hasUnsplit = PETSC_TRUE;
1408               }
1409             }
1410           }
1411         }
1412         PetscCall(DMPlexSetCone(sdm, splitp, &coneNew[2]));
1413         PetscCall(DMPlexSetConeOrientation(sdm, splitp, ornt));
1414         /* Face support */
1415         PetscInt vals[2];
1416 
1417         PetscCall(DMLabelGetValue(label, support[0], &vals[0]));
1418         if (supportSize > 1) PetscCall(DMLabelGetValue(label, support[1], &vals[1]));
1419         else vals[1] = -vals[0];
1420         PetscCheck(vals[0] * vals[1] < 0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid support labels %" PetscInt_FMT " %" PetscInt_FMT, vals[0], vals[1]);
1421 
1422         for (s = 0; s < 2; ++s) {
1423           if (s >= supportSize) {
1424             if (vals[s] < 0) {
1425               /* Ghost old face:   Replace negative side cell with cohesive cell */
1426               PetscCall(DMPlexInsertSupport(sdm, newp, s, hybcell));
1427             } else {
1428               /* Ghost new face:   Replace positive side cell with cohesive cell */
1429               PetscCall(DMPlexInsertSupport(sdm, splitp, s, hybcell));
1430             }
1431           } else {
1432             if (vals[s] < 0) {
1433               /* Split old face:   Replace negative side cell with cohesive cell */
1434               PetscCall(DMPlexInsertSupport(sdm, newp, s, hybcell));
1435             } else {
1436               /* Split new face:   Replace positive side cell with cohesive cell */
1437               PetscCall(DMPlexInsertSupport(sdm, splitp, s, hybcell));
1438             }
1439           }
1440         }
1441         /* Get orientation for cohesive face using the positive side cell */
1442         {
1443           const PetscInt *ncone, *nconeO;
1444           PetscInt        nconeSize, nc, ocell;
1445           PetscBool       flip = PETSC_FALSE;
1446 
1447           if (supportSize > 1) {
1448             ocell = vals[0] < 0 ? support[1] : support[0];
1449           } else {
1450             ocell = support[0];
1451             flip  = vals[0] < 0 ? PETSC_TRUE : PETSC_FALSE;
1452           }
1453           PetscCall(DMPlexGetConeSize(dm, ocell, &nconeSize));
1454           PetscCall(DMPlexGetCone(dm, ocell, &ncone));
1455           PetscCall(DMPlexGetConeOrientation(dm, ocell, &nconeO));
1456           for (nc = 0; nc < nconeSize; ++nc) {
1457             if (ncone[nc] == oldp) {
1458               coneONew[0] = flip ? -(nconeO[nc] + 1) : nconeO[nc];
1459               break;
1460             }
1461           }
1462           PetscCheck(nc < nconeSize, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate face %" PetscInt_FMT " in neighboring cell %" PetscInt_FMT, oldp, ocell);
1463         }
1464         /* Cohesive cell:    Old and new split face, then new cohesive faces */
1465         {
1466           const PetscInt No = DMPolytopeTypeGetNumArrangments(ct) / 2;
1467           PetscCheck((coneONew[0] >= -No) && (coneONew[0] < No), PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid %s orientation %" PetscInt_FMT, DMPolytopeTypes[ct], coneONew[0]);
1468         }
1469         const PetscInt *arr = DMPolytopeTypeGetArrangment(ct, coneONew[0]);
1470 
1471         coneNew[0]  = newp; /* Extracted negative side orientation above */
1472         coneNew[1]  = splitp;
1473         coneONew[1] = coneONew[0];
1474         for (q = 0; q < coneSize; ++q) {
1475           /* Hybrid faces must follow order from oriented end face */
1476           const PetscInt qa = arr[q * 2 + 0];
1477           const PetscInt qo = arr[q * 2 + 1];
1478           DMPolytopeType ft = dep == 2 ? DM_POLYTOPE_SEGMENT : DM_POLYTOPE_POINT;
1479 
1480           PetscCall(PetscFindInt(cone[qa], numSplitPoints[dep - 1], splitPoints[dep - 1], &v));
1481           if (v < 0) {
1482             PetscCall(PetscFindInt(cone[qa], numUnsplitPoints[dep - 1], unsplitPoints[dep - 1], &v));
1483             coneNew[2 + q] = v + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep - 1];
1484           } else {
1485             coneNew[2 + q] = v + pMaxNew[dep] + numSplitPoints[dep];
1486           }
1487           coneONew[2 + q] = DMPolytopeTypeComposeOrientation(ft, qo, ornt[qa]);
1488         }
1489         PetscCall(DMPlexSetCone(sdm, hybcell, coneNew));
1490         PetscCall(DMPlexSetConeOrientation(sdm, hybcell, coneONew));
1491         /* Label the hybrid cells on the boundary of the split */
1492         if (hasUnsplit) PetscCall(DMLabelSetValue(label, -hybcell, dim));
1493       } else if (dep == 0) {
1494         const PetscInt hybedge = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1];
1495 
1496         /* Split old vertex: Edges in old split faces and new cohesive edge */
1497         for (e = 0, qn = 0; e < supportSize; ++e) {
1498           PetscInt val;
1499 
1500           PetscCall(DMLabelGetValue(label, support[e], &val));
1501           if ((val == 1) || (val == (shift + 1))) supportNew[qn++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1502         }
1503         supportNew[qn] = hybedge;
1504         PetscCall(DMPlexSetSupport(sdm, newp, supportNew));
1505         /* Split new vertex: Edges in new split faces and new cohesive edge */
1506         for (e = 0, qp = 0; e < supportSize; ++e) {
1507           PetscInt val, edge;
1508 
1509           PetscCall(DMLabelGetValue(label, support[e], &val));
1510           if (val == 1) {
1511             PetscCall(PetscFindInt(support[e], numSplitPoints[dep + 1], splitPoints[dep + 1], &edge));
1512             PetscCheck(edge >= 0, comm, PETSC_ERR_ARG_WRONG, "Edge %" PetscInt_FMT " is not a split edge", support[e]);
1513             supportNew[qp++] = edge + pMaxNew[dep + 1];
1514           } else if (val == -(shift + 1)) {
1515             supportNew[qp++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1516           }
1517         }
1518         supportNew[qp] = hybedge;
1519         PetscCall(DMPlexSetSupport(sdm, splitp, supportNew));
1520         /* Hybrid edge:    Old and new split vertex */
1521         coneNew[0] = newp;
1522         coneNew[1] = splitp;
1523         PetscCall(DMPlexSetCone(sdm, hybedge, coneNew));
1524         for (e = 0, qf = 0; e < supportSize; ++e) {
1525           PetscInt val, edge;
1526 
1527           PetscCall(DMLabelGetValue(label, support[e], &val));
1528           if (val == 1) {
1529             PetscCall(PetscFindInt(support[e], numSplitPoints[dep + 1], splitPoints[dep + 1], &edge));
1530             PetscCheck(edge >= 0, comm, PETSC_ERR_ARG_WRONG, "Edge %" PetscInt_FMT " is not a split edge", support[e]);
1531             supportNew[qf++] = edge + pMaxNew[dep + 2] + numSplitPoints[dep + 2];
1532           }
1533         }
1534         PetscCall(DMPlexSetSupport(sdm, hybedge, supportNew));
1535       } else if (dep == dim - 2) {
1536         const PetscInt hybface = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1];
1537 
1538         /* Split old edge:   old vertices in cone so no change */
1539         /* Split new edge:   new vertices in cone */
1540         for (q = 0; q < coneSize; ++q) {
1541           PetscCall(PetscFindInt(cone[q], numSplitPoints[dep - 1], splitPoints[dep - 1], &v));
1542           if (v < 0) {
1543             PetscCall(PetscFindInt(cone[q], numUnsplitPoints[dep - 1], unsplitPoints[dep - 1], &v));
1544             PetscCheck(v >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %" PetscInt_FMT " in split or unsplit points of depth %" PetscInt_FMT, cone[q], dep - 1);
1545             coneNew[q] = DMPlexShiftPoint_Internal(cone[q], depth, depthShift) /*cone[q] + depthOffset[dep-1]*/;
1546           } else {
1547             coneNew[q] = v + pMaxNew[dep - 1];
1548           }
1549         }
1550         PetscCall(DMPlexSetCone(sdm, splitp, coneNew));
1551         /* Split old edge: Faces in positive side cells and old split faces */
1552         for (e = 0, q = 0; e < supportSize; ++e) {
1553           PetscInt val;
1554 
1555           PetscCall(DMLabelGetValue(label, support[e], &val));
1556           if (val == dim - 1) {
1557             supportNew[q++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1558           } else if (val == (shift + dim - 1)) {
1559             supportNew[q++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1560           }
1561         }
1562         supportNew[q++] = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1];
1563         PetscCall(DMPlexSetSupport(sdm, newp, supportNew));
1564         /* Split new edge: Faces in negative side cells and new split faces */
1565         for (e = 0, q = 0; e < supportSize; ++e) {
1566           PetscInt val, face;
1567 
1568           PetscCall(DMLabelGetValue(label, support[e], &val));
1569           if (val == dim - 1) {
1570             PetscCall(PetscFindInt(support[e], numSplitPoints[dep + 1], splitPoints[dep + 1], &face));
1571             PetscCheck(face >= 0, comm, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " is not a split face", support[e]);
1572             supportNew[q++] = face + pMaxNew[dep + 1];
1573           } else if (val == -(shift + dim - 1)) {
1574             supportNew[q++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1575           }
1576         }
1577         supportNew[q++] = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1];
1578         PetscCall(DMPlexSetSupport(sdm, splitp, supportNew));
1579         /* Hybrid face */
1580         coneNew[0] = newp;
1581         coneNew[1] = splitp;
1582         for (v = 0; v < coneSize; ++v) {
1583           PetscInt vertex;
1584           PetscCall(PetscFindInt(cone[v], numSplitPoints[dep - 1], splitPoints[dep - 1], &vertex));
1585           if (vertex < 0) {
1586             PetscCall(PetscFindInt(cone[v], numUnsplitPoints[dep - 1], unsplitPoints[dep - 1], &vertex));
1587             PetscCheck(vertex >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %" PetscInt_FMT " in split or unsplit points of depth %" PetscInt_FMT, cone[v], dep - 1);
1588             coneNew[2 + v] = vertex + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep - 1];
1589           } else {
1590             coneNew[2 + v] = vertex + pMaxNew[dep] + numSplitPoints[dep];
1591           }
1592         }
1593         PetscCall(DMPlexSetCone(sdm, hybface, coneNew));
1594         for (e = 0, qf = 0; e < supportSize; ++e) {
1595           PetscInt val, face;
1596 
1597           PetscCall(DMLabelGetValue(label, support[e], &val));
1598           if (val == dim - 1) {
1599             PetscCall(PetscFindInt(support[e], numSplitPoints[dep + 1], splitPoints[dep + 1], &face));
1600             PetscCheck(face >= 0, comm, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " is not a split face", support[e]);
1601             supportNew[qf++] = face + pMaxNew[dep + 2] + numSplitPoints[dep + 2];
1602           }
1603         }
1604         PetscCall(DMPlexSetSupport(sdm, hybface, supportNew));
1605       }
1606     }
1607   }
1608   for (dep = 0; dep <= depth; ++dep) {
1609     for (p = 0; p < numUnsplitPoints[dep]; ++p) {
1610       const PetscInt  oldp = unsplitPoints[dep][p];
1611       const PetscInt  newp = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*oldp + depthOffset[dep]*/;
1612       const PetscInt *cone, *support;
1613       PetscInt        coneSize, supportSize, supportSizeNew, q, qf, e, f, s;
1614 
1615       PetscCall(DMPlexGetConeSize(dm, oldp, &coneSize));
1616       PetscCall(DMPlexGetCone(dm, oldp, &cone));
1617       PetscCall(DMPlexGetSupportSize(dm, oldp, &supportSize));
1618       PetscCall(DMPlexGetSupport(dm, oldp, &support));
1619       if (dep == 0) {
1620         const PetscInt hybedge = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1] + numSplitPoints[dep];
1621 
1622         /* Unsplit vertex */
1623         PetscCall(DMPlexGetSupportSize(sdm, newp, &supportSizeNew));
1624         for (s = 0, q = 0; s < supportSize; ++s) {
1625           supportNew[q++] = DMPlexShiftPoint_Internal(support[s], depth, depthShift) /*support[s] + depthOffset[dep+1]*/;
1626           PetscCall(PetscFindInt(support[s], numSplitPoints[dep + 1], splitPoints[dep + 1], &e));
1627           if (e >= 0) supportNew[q++] = e + pMaxNew[dep + 1];
1628         }
1629         supportNew[q++] = hybedge;
1630         supportNew[q++] = hybedge;
1631         PetscCheck(q == supportSizeNew, comm, PETSC_ERR_ARG_WRONG, "Support size %" PetscInt_FMT " != %" PetscInt_FMT " for vertex %" PetscInt_FMT, q, supportSizeNew, newp);
1632         PetscCall(DMPlexSetSupport(sdm, newp, supportNew));
1633         /* Hybrid edge */
1634         coneNew[0] = newp;
1635         coneNew[1] = newp;
1636         PetscCall(DMPlexSetCone(sdm, hybedge, coneNew));
1637         for (e = 0, qf = 0; e < supportSize; ++e) {
1638           PetscInt val, edge;
1639 
1640           PetscCall(DMLabelGetValue(label, support[e], &val));
1641           if (val == 1) {
1642             PetscCall(PetscFindInt(support[e], numSplitPoints[dep + 1], splitPoints[dep + 1], &edge));
1643             PetscCheck(edge >= 0, comm, PETSC_ERR_ARG_WRONG, "Edge %" PetscInt_FMT " is not a split edge", support[e]);
1644             supportNew[qf++] = edge + pMaxNew[dep + 2] + numSplitPoints[dep + 2];
1645           } else if (val == (shift2 + 1)) {
1646             PetscCall(PetscFindInt(support[e], numUnsplitPoints[dep + 1], unsplitPoints[dep + 1], &edge));
1647             PetscCheck(edge >= 0, comm, PETSC_ERR_ARG_WRONG, "Edge %" PetscInt_FMT " is not a unsplit edge", support[e]);
1648             supportNew[qf++] = edge + pMaxNew[dep + 2] + numSplitPoints[dep + 2] + numSplitPoints[dep + 1];
1649           }
1650         }
1651         PetscCall(DMPlexSetSupport(sdm, hybedge, supportNew));
1652       } else if (dep == dim - 2) {
1653         const PetscInt hybface = p + pMaxNew[dep + 1] + numSplitPoints[dep + 1] + numSplitPoints[dep];
1654 
1655         /* Unsplit edge: Faces into original edge, split face, and hybrid face twice */
1656         for (f = 0, qf = 0; f < supportSize; ++f) {
1657           PetscInt val, face;
1658 
1659           PetscCall(DMLabelGetValue(label, support[f], &val));
1660           if (val == dim - 1) {
1661             PetscCall(PetscFindInt(support[f], numSplitPoints[dep + 1], splitPoints[dep + 1], &face));
1662             PetscCheck(face >= 0, comm, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " is not a split face", support[f]);
1663             supportNew[qf++] = DMPlexShiftPoint_Internal(support[f], depth, depthShift) /*support[f] + depthOffset[dep+1]*/;
1664             supportNew[qf++] = face + pMaxNew[dep + 1];
1665           } else {
1666             supportNew[qf++] = DMPlexShiftPoint_Internal(support[f], depth, depthShift) /*support[f] + depthOffset[dep+1]*/;
1667           }
1668         }
1669         supportNew[qf++] = hybface;
1670         supportNew[qf++] = hybface;
1671         PetscCall(DMPlexGetSupportSize(sdm, newp, &supportSizeNew));
1672         PetscCheck(qf == supportSizeNew, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Support size for unsplit edge %" PetscInt_FMT " is %" PetscInt_FMT " != %" PetscInt_FMT, newp, qf, supportSizeNew);
1673         PetscCall(DMPlexSetSupport(sdm, newp, supportNew));
1674         /* Add hybrid face */
1675         coneNew[0] = newp;
1676         coneNew[1] = newp;
1677         PetscCall(PetscFindInt(cone[0], numUnsplitPoints[dep - 1], unsplitPoints[dep - 1], &v));
1678         PetscCheck(v >= 0, comm, PETSC_ERR_ARG_WRONG, "Vertex %" PetscInt_FMT " is not an unsplit vertex", cone[0]);
1679         coneNew[2] = v + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep - 1];
1680         PetscCall(PetscFindInt(cone[1], numUnsplitPoints[dep - 1], unsplitPoints[dep - 1], &v));
1681         PetscCheck(v >= 0, comm, PETSC_ERR_ARG_WRONG, "Vertex %" PetscInt_FMT " is not an unsplit vertex", cone[1]);
1682         coneNew[3] = v + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep - 1];
1683         PetscCall(DMPlexSetCone(sdm, hybface, coneNew));
1684         for (f = 0, qf = 0; f < supportSize; ++f) {
1685           PetscInt val, face;
1686 
1687           PetscCall(DMLabelGetValue(label, support[f], &val));
1688           if (val == dim - 1) {
1689             PetscCall(PetscFindInt(support[f], numSplitPoints[dep + 1], splitPoints[dep + 1], &face));
1690             supportNew[qf++] = face + pMaxNew[dep + 2] + numSplitPoints[dep + 2];
1691           }
1692         }
1693         PetscCall(DMPlexGetSupportSize(sdm, hybface, &supportSizeNew));
1694         PetscCheck(qf == supportSizeNew, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Support size for hybrid face %" PetscInt_FMT " is %" PetscInt_FMT " != %" PetscInt_FMT, hybface, qf, supportSizeNew);
1695         PetscCall(DMPlexSetSupport(sdm, hybface, supportNew));
1696       }
1697     }
1698   }
1699   /* Step 6b: Replace split points in negative side cones */
1700   for (sp = 0; sp < numSP; ++sp) {
1701     PetscInt        dep = values[sp];
1702     IS              pIS;
1703     PetscInt        numPoints;
1704     const PetscInt *points;
1705 
1706     if (dep >= 0) continue;
1707     PetscCall(DMLabelGetStratumIS(label, dep, &pIS));
1708     if (!pIS) continue;
1709     dep = -dep - shift;
1710     PetscCall(ISGetLocalSize(pIS, &numPoints));
1711     PetscCall(ISGetIndices(pIS, &points));
1712     for (p = 0; p < numPoints; ++p) {
1713       const PetscInt  oldp = points[p];
1714       const PetscInt  newp = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*depthOffset[dep] + oldp*/;
1715       const PetscInt *cone;
1716       PetscInt        coneSize, c;
1717       /* PetscBool       replaced = PETSC_FALSE; */
1718 
1719       /* Negative edge: replace split vertex */
1720       /* Negative cell: replace split face */
1721       PetscCall(DMPlexGetConeSize(sdm, newp, &coneSize));
1722       PetscCall(DMPlexGetCone(sdm, newp, &cone));
1723       for (c = 0; c < coneSize; ++c) {
1724         const PetscInt coldp = DMPlexShiftPointInverse_Internal(cone[c], depth, depthShift);
1725         PetscInt       csplitp, cp, val;
1726 
1727         PetscCall(DMLabelGetValue(label, coldp, &val));
1728         if (val == dep - 1) {
1729           PetscCall(PetscFindInt(coldp, numSplitPoints[dep - 1], splitPoints[dep - 1], &cp));
1730           PetscCheck(cp >= 0, comm, PETSC_ERR_ARG_WRONG, "Point %" PetscInt_FMT " is not a split point of dimension %" PetscInt_FMT, oldp, dep - 1);
1731           csplitp = pMaxNew[dep - 1] + cp;
1732           PetscCall(DMPlexInsertCone(sdm, newp, c, csplitp));
1733           /* replaced = PETSC_TRUE; */
1734         }
1735       }
1736       /* Cells with only a vertex or edge on the submesh have no replacement */
1737       /* PetscCheck(replaced,comm, PETSC_ERR_ARG_WRONG, "The cone of point %d does not contain split points", oldp); */
1738     }
1739     PetscCall(ISRestoreIndices(pIS, &points));
1740     PetscCall(ISDestroy(&pIS));
1741   }
1742   /* Step 7: Coordinates */
1743   PetscCall(DMPlexShiftCoordinates_Internal(dm, depthShift, sdm));
1744   PetscCall(DMGetCoordinateSection(sdm, &coordSection));
1745   PetscCall(DMGetCoordinatesLocal(sdm, &coordinates));
1746   PetscCall(VecGetArray(coordinates, &coords));
1747   for (v = 0; v < (numSplitPoints ? numSplitPoints[0] : 0); ++v) {
1748     const PetscInt newp   = DMPlexShiftPoint_Internal(splitPoints[0][v], depth, depthShift) /*depthOffset[0] + splitPoints[0][v]*/;
1749     const PetscInt splitp = pMaxNew[0] + v;
1750     PetscInt       dof, off, soff, d;
1751 
1752     PetscCall(PetscSectionGetDof(coordSection, newp, &dof));
1753     PetscCall(PetscSectionGetOffset(coordSection, newp, &off));
1754     PetscCall(PetscSectionGetOffset(coordSection, splitp, &soff));
1755     for (d = 0; d < dof; ++d) coords[soff + d] = coords[off + d];
1756   }
1757   PetscCall(VecRestoreArray(coordinates, &coords));
1758   /* Step 8: SF, if I can figure this out we can split the mesh in parallel */
1759   PetscCall(DMPlexShiftSF_Internal(dm, depthShift, sdm));
1760   /*   TODO We need to associate the ghost points with the correct replica */
1761   /* Step 9: Labels */
1762   PetscCall(DMPlexShiftLabels_Internal(dm, depthShift, sdm));
1763   PetscCall(DMPlexCreateVTKLabel_Internal(dm, PETSC_FALSE, sdm));
1764   PetscCall(DMGetNumLabels(sdm, &numLabels));
1765   for (dep = 0; dep <= depth; ++dep) {
1766     for (p = 0; p < numSplitPoints[dep]; ++p) {
1767       const PetscInt newp   = DMPlexShiftPoint_Internal(splitPoints[dep][p], depth, depthShift) /*depthOffset[dep] + splitPoints[dep][p]*/;
1768       const PetscInt splitp = pMaxNew[dep] + p;
1769       PetscInt       l;
1770 
1771       if (splitLabel) {
1772         const PetscInt val = 100 + dep;
1773 
1774         PetscCall(DMLabelSetValue(splitLabel, newp, val));
1775         PetscCall(DMLabelSetValue(splitLabel, splitp, -val));
1776       }
1777       for (l = 0; l < numLabels; ++l) {
1778         DMLabel     mlabel;
1779         const char *lname;
1780         PetscInt    val;
1781         PetscBool   isDepth;
1782 
1783         PetscCall(DMGetLabelName(sdm, l, &lname));
1784         PetscCall(PetscStrcmp(lname, "depth", &isDepth));
1785         if (isDepth) continue;
1786         PetscCall(DMGetLabel(sdm, lname, &mlabel));
1787         PetscCall(DMLabelGetValue(mlabel, newp, &val));
1788         if (val >= 0) PetscCall(DMLabelSetValue(mlabel, splitp, val));
1789       }
1790     }
1791   }
1792   for (sp = 0; sp < numSP; ++sp) {
1793     const PetscInt dep = values[sp];
1794 
1795     if ((dep < 0) || (dep > depth)) continue;
1796     if (splitIS[dep]) PetscCall(ISRestoreIndices(splitIS[dep], &splitPoints[dep]));
1797     PetscCall(ISDestroy(&splitIS[dep]));
1798     if (unsplitIS[dep]) PetscCall(ISRestoreIndices(unsplitIS[dep], &unsplitPoints[dep]));
1799     PetscCall(ISDestroy(&unsplitIS[dep]));
1800   }
1801   if (ghostIS) PetscCall(ISRestoreIndices(ghostIS, &ghostPoints));
1802   PetscCall(ISDestroy(&ghostIS));
1803   if (label) {
1804     PetscCall(ISRestoreIndices(valueIS, &values));
1805     PetscCall(ISDestroy(&valueIS));
1806   }
1807   for (d = 0; d <= depth; ++d) {
1808     PetscCall(DMPlexGetDepthStratum(sdm, d, NULL, &pEnd));
1809     pMaxNew[d] = pEnd - numHybridPoints[d] - numHybridPointsOld[d];
1810   }
1811   PetscCall(PetscFree3(coneNew, coneONew, supportNew));
1812   PetscCall(PetscFree5(depthMax, depthEnd, depthShift, pMaxNew, numHybridPointsOld));
1813   PetscCall(PetscFree7(splitIS, unsplitIS, numSplitPoints, numUnsplitPoints, numHybridPoints, splitPoints, unsplitPoints));
1814   PetscFunctionReturn(0);
1815 }
1816 
1817 /*@C
1818   DMPlexConstructCohesiveCells - Construct cohesive cells which split the face along an internal interface
1819 
1820   Collective on dm
1821 
1822   Input Parameters:
1823 + dm - The original DM
1824 - label - The label specifying the boundary faces (this could be auto-generated)
1825 
1826   Output Parameters:
1827 + splitLabel - The label containing the split points, or NULL if no output is desired
1828 - dmSplit - The new DM
1829 
1830   Level: developer
1831 
1832 .seealso: `DMCreate()`, `DMPlexLabelCohesiveComplete()`
1833 @*/
1834 PetscErrorCode DMPlexConstructCohesiveCells(DM dm, DMLabel label, DMLabel splitLabel, DM *dmSplit)
1835 {
1836   DM       sdm;
1837   PetscInt dim;
1838 
1839   PetscFunctionBegin;
1840   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1841   PetscValidPointer(dmSplit, 4);
1842   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), &sdm));
1843   PetscCall(DMSetType(sdm, DMPLEX));
1844   PetscCall(DMGetDimension(dm, &dim));
1845   PetscCall(DMSetDimension(sdm, dim));
1846   switch (dim) {
1847   case 2:
1848   case 3:
1849     PetscCall(DMPlexConstructCohesiveCells_Internal(dm, label, splitLabel, sdm));
1850     break;
1851   default:
1852     SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cannot construct cohesive cells for dimension %" PetscInt_FMT, dim);
1853   }
1854   *dmSplit = sdm;
1855   PetscFunctionReturn(0);
1856 }
1857 
1858 /* Returns the side of the surface for a given cell with a face on the surface */
1859 static PetscErrorCode GetSurfaceSide_Static(DM dm, DM subdm, PetscInt numSubpoints, const PetscInt *subpoints, PetscInt cell, PetscInt face, PetscBool *pos)
1860 {
1861   const PetscInt *cone, *ornt;
1862   PetscInt        dim, coneSize, c;
1863 
1864   PetscFunctionBegin;
1865   *pos = PETSC_TRUE;
1866   PetscCall(DMGetDimension(dm, &dim));
1867   PetscCall(DMPlexGetConeSize(dm, cell, &coneSize));
1868   PetscCall(DMPlexGetCone(dm, cell, &cone));
1869   PetscCall(DMPlexGetConeOrientation(dm, cell, &ornt));
1870   for (c = 0; c < coneSize; ++c) {
1871     if (cone[c] == face) {
1872       PetscInt o = ornt[c];
1873 
1874       if (subdm) {
1875         const PetscInt *subcone, *subornt;
1876         PetscInt        subpoint, subface, subconeSize, sc;
1877 
1878         PetscCall(PetscFindInt(cell, numSubpoints, subpoints, &subpoint));
1879         PetscCall(PetscFindInt(face, numSubpoints, subpoints, &subface));
1880         PetscCall(DMPlexGetConeSize(subdm, subpoint, &subconeSize));
1881         PetscCall(DMPlexGetCone(subdm, subpoint, &subcone));
1882         PetscCall(DMPlexGetConeOrientation(subdm, subpoint, &subornt));
1883         for (sc = 0; sc < subconeSize; ++sc) {
1884           if (subcone[sc] == subface) {
1885             o = subornt[0];
1886             break;
1887           }
1888         }
1889         PetscCheck(sc < subconeSize, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not find subpoint %" PetscInt_FMT " (%" PetscInt_FMT ") in cone for subpoint %" PetscInt_FMT " (%" PetscInt_FMT ")", subface, face, subpoint, cell);
1890       }
1891       if (o >= 0) *pos = PETSC_TRUE;
1892       else *pos = PETSC_FALSE;
1893       break;
1894     }
1895   }
1896   PetscCheck(c != coneSize, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Cell %" PetscInt_FMT " in split face %" PetscInt_FMT " support does not have it in the cone", cell, face);
1897   PetscFunctionReturn(0);
1898 }
1899 
1900 static PetscErrorCode CheckFaultEdge_Private(DM dm, DMLabel label)
1901 {
1902   IS              facePosIS, faceNegIS, dimIS;
1903   const PetscInt *points;
1904   PetscInt        dim, numPoints, p, shift = 100, shift2 = 200;
1905 
1906   PetscFunctionBegin;
1907   PetscCall(DMGetDimension(dm, &dim));
1908   /* If any faces touching the fault divide cells on either side, split them */
1909   PetscCall(DMLabelGetStratumIS(label, shift + dim - 1, &facePosIS));
1910   PetscCall(DMLabelGetStratumIS(label, -(shift + dim - 1), &faceNegIS));
1911   if (!facePosIS || !faceNegIS) {
1912     PetscCall(ISDestroy(&facePosIS));
1913     PetscCall(ISDestroy(&faceNegIS));
1914     PetscFunctionReturn(0);
1915   }
1916   PetscCall(ISExpand(facePosIS, faceNegIS, &dimIS));
1917   PetscCall(ISDestroy(&facePosIS));
1918   PetscCall(ISDestroy(&faceNegIS));
1919   PetscCall(ISGetLocalSize(dimIS, &numPoints));
1920   PetscCall(ISGetIndices(dimIS, &points));
1921   for (p = 0; p < numPoints; ++p) {
1922     const PetscInt  point = points[p];
1923     const PetscInt *support;
1924     PetscInt        supportSize, valA, valB;
1925 
1926     PetscCall(DMPlexGetSupportSize(dm, point, &supportSize));
1927     if (supportSize != 2) continue;
1928     PetscCall(DMPlexGetSupport(dm, point, &support));
1929     PetscCall(DMLabelGetValue(label, support[0], &valA));
1930     PetscCall(DMLabelGetValue(label, support[1], &valB));
1931     if ((valA == -1) || (valB == -1)) continue;
1932     if (valA * valB > 0) continue;
1933     /* Check that this face is not incident on only unsplit faces, meaning has at least one split face */
1934     {
1935       PetscInt *closure = NULL;
1936       PetscBool split   = PETSC_FALSE;
1937       PetscInt  closureSize, cl;
1938 
1939       PetscCall(DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
1940       for (cl = 0; cl < closureSize * 2; cl += 2) {
1941         PetscCall(DMLabelGetValue(label, closure[cl], &valA));
1942         if ((valA >= 0) && (valA <= dim)) {
1943           split = PETSC_TRUE;
1944           break;
1945         }
1946       }
1947       PetscCall(DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
1948       if (!split) continue;
1949     }
1950     /* Split the face */
1951     PetscCall(DMLabelGetValue(label, point, &valA));
1952     PetscCall(DMLabelClearValue(label, point, valA));
1953     PetscCall(DMLabelSetValue(label, point, dim - 1));
1954     /* Label its closure:
1955       unmarked: label as unsplit
1956       incident: relabel as split
1957       split:    do nothing
1958     */
1959     {
1960       PetscInt *closure = NULL;
1961       PetscInt  closureSize, cl, dep;
1962 
1963       PetscCall(DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
1964       for (cl = 0; cl < closureSize * 2; cl += 2) {
1965         PetscCall(DMLabelGetValue(label, closure[cl], &valA));
1966         if (valA == -1) { /* Mark as unsplit */
1967           PetscCall(DMPlexGetPointDepth(dm, closure[cl], &dep));
1968           PetscCall(DMLabelSetValue(label, closure[cl], shift2 + dep));
1969         } else if (((valA >= shift) && (valA < shift2)) || ((valA <= -shift) && (valA > -shift2))) {
1970           PetscCall(DMPlexGetPointDepth(dm, closure[cl], &dep));
1971           PetscCall(DMLabelClearValue(label, closure[cl], valA));
1972           PetscCall(DMLabelSetValue(label, closure[cl], dep));
1973         }
1974       }
1975       PetscCall(DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
1976     }
1977   }
1978   PetscCall(ISRestoreIndices(dimIS, &points));
1979   PetscCall(ISDestroy(&dimIS));
1980   PetscFunctionReturn(0);
1981 }
1982 
1983 /*@
1984   DMPlexLabelCohesiveComplete - Starting with a label marking points on an internal surface, we add all other mesh pieces
1985   to complete the surface
1986 
1987   Input Parameters:
1988 + dm     - The DM
1989 . label  - A DMLabel marking the surface
1990 . blabel - A DMLabel marking the vertices on the boundary which will not be duplicated, or NULL to find them automatically
1991 . bvalue - Value of DMLabel marking the vertices on the boundary
1992 . flip   - Flag to flip the submesh normal and replace points on the other side
1993 - subdm  - The subDM associated with the label, or NULL
1994 
1995   Output Parameter:
1996 . label - A DMLabel marking all surface points
1997 
1998   Note: The vertices in blabel are called "unsplit" in the terminology from hybrid cell creation.
1999 
2000   Level: developer
2001 
2002 .seealso: `DMPlexConstructCohesiveCells()`, `DMPlexLabelComplete()`
2003 @*/
2004 PetscErrorCode DMPlexLabelCohesiveComplete(DM dm, DMLabel label, DMLabel blabel, PetscInt bvalue, PetscBool flip, DM subdm)
2005 {
2006   DMLabel         depthLabel;
2007   IS              dimIS, subpointIS = NULL;
2008   const PetscInt *points, *subpoints;
2009   const PetscInt  rev   = flip ? -1 : 1;
2010   PetscInt        shift = 100, shift2 = 200, shift3 = 300, dim, depth, numPoints, numSubpoints, p, val;
2011 
2012   PetscFunctionBegin;
2013   PetscCall(DMPlexGetDepth(dm, &depth));
2014   PetscCall(DMGetDimension(dm, &dim));
2015   PetscCall(DMPlexGetDepthLabel(dm, &depthLabel));
2016   if (subdm) {
2017     PetscCall(DMPlexGetSubpointIS(subdm, &subpointIS));
2018     if (subpointIS) {
2019       PetscCall(ISGetLocalSize(subpointIS, &numSubpoints));
2020       PetscCall(ISGetIndices(subpointIS, &subpoints));
2021     }
2022   }
2023   /* Mark cell on the fault, and its faces which touch the fault: cell orientation for face gives the side of the fault */
2024   PetscCall(DMLabelGetStratumIS(label, dim - 1, &dimIS));
2025   if (!dimIS) goto divide;
2026   PetscCall(ISGetLocalSize(dimIS, &numPoints));
2027   PetscCall(ISGetIndices(dimIS, &points));
2028   for (p = 0; p < numPoints; ++p) { /* Loop over fault faces */
2029     const PetscInt *support;
2030     PetscInt        supportSize, s;
2031 
2032     PetscCall(DMPlexGetSupportSize(dm, points[p], &supportSize));
2033 #if 0
2034     if (supportSize != 2) {
2035       const PetscInt *lp;
2036       PetscInt        Nlp, pind;
2037 
2038       /* Check that for a cell with a single support face, that face is in the SF */
2039       /*   THis check only works for the remote side. We would need root side information */
2040       PetscCall(PetscSFGetGraph(dm->sf, NULL, &Nlp, &lp, NULL));
2041       PetscCall(PetscFindInt(points[p], Nlp, lp, &pind));
2042       PetscCheck(pind >= 0,PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Split face %" PetscInt_FMT " has %" PetscInt_FMT " != 2 supports, and the face is not shared with another process", points[p], supportSize);
2043     }
2044 #endif
2045     PetscCall(DMPlexGetSupport(dm, points[p], &support));
2046     for (s = 0; s < supportSize; ++s) {
2047       const PetscInt *cone;
2048       PetscInt        coneSize, c;
2049       PetscBool       pos;
2050 
2051       PetscCall(GetSurfaceSide_Static(dm, subdm, numSubpoints, subpoints, support[s], points[p], &pos));
2052       if (pos) PetscCall(DMLabelSetValue(label, support[s], rev * (shift + dim)));
2053       else PetscCall(DMLabelSetValue(label, support[s], -rev * (shift + dim)));
2054       if (rev < 0) pos = !pos ? PETSC_TRUE : PETSC_FALSE;
2055       /* Put faces touching the fault in the label */
2056       PetscCall(DMPlexGetConeSize(dm, support[s], &coneSize));
2057       PetscCall(DMPlexGetCone(dm, support[s], &cone));
2058       for (c = 0; c < coneSize; ++c) {
2059         const PetscInt point = cone[c];
2060 
2061         PetscCall(DMLabelGetValue(label, point, &val));
2062         if (val == -1) {
2063           PetscInt *closure = NULL;
2064           PetscInt  closureSize, cl;
2065 
2066           PetscCall(DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
2067           for (cl = 0; cl < closureSize * 2; cl += 2) {
2068             const PetscInt clp  = closure[cl];
2069             PetscInt       bval = -1;
2070 
2071             PetscCall(DMLabelGetValue(label, clp, &val));
2072             if (blabel) PetscCall(DMLabelGetValue(blabel, clp, &bval));
2073             if ((val >= 0) && (val < dim - 1) && (bval < 0)) {
2074               PetscCall(DMLabelSetValue(label, point, pos == PETSC_TRUE ? shift + dim - 1 : -(shift + dim - 1)));
2075               break;
2076             }
2077           }
2078           PetscCall(DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
2079         }
2080       }
2081     }
2082   }
2083   PetscCall(ISRestoreIndices(dimIS, &points));
2084   PetscCall(ISDestroy(&dimIS));
2085   /* Mark boundary points as unsplit */
2086   if (blabel) {
2087     IS bdIS;
2088 
2089     PetscCall(DMLabelGetStratumIS(blabel, bvalue, &bdIS));
2090     PetscCall(ISGetLocalSize(bdIS, &numPoints));
2091     PetscCall(ISGetIndices(bdIS, &points));
2092     for (p = 0; p < numPoints; ++p) {
2093       const PetscInt point = points[p];
2094       PetscInt       val, bval;
2095 
2096       PetscCall(DMLabelGetValue(blabel, point, &bval));
2097       if (bval >= 0) {
2098         PetscCall(DMLabelGetValue(label, point, &val));
2099         if ((val < 0) || (val > dim)) {
2100           /* This could be a point added from splitting a vertex on an adjacent fault, otherwise its just wrong */
2101           PetscCall(DMLabelClearValue(blabel, point, bval));
2102         }
2103       }
2104     }
2105     for (p = 0; p < numPoints; ++p) {
2106       const PetscInt point = points[p];
2107       PetscInt       val, bval;
2108 
2109       PetscCall(DMLabelGetValue(blabel, point, &bval));
2110       if (bval >= 0) {
2111         const PetscInt *cone, *support;
2112         PetscInt        coneSize, supportSize, s, valA, valB, valE;
2113 
2114         /* Mark as unsplit */
2115         PetscCall(DMLabelGetValue(label, point, &val));
2116         PetscCheck(val >= 0 && val <= dim, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %" PetscInt_FMT " has label value %" PetscInt_FMT ", should be part of the fault", point, val);
2117         PetscCall(DMLabelClearValue(label, point, val));
2118         PetscCall(DMLabelSetValue(label, point, shift2 + val));
2119         /* Check for cross-edge
2120              A cross-edge has endpoints which are both on the boundary of the surface, but the edge itself is not. */
2121         if (val != 0) continue;
2122         PetscCall(DMPlexGetSupport(dm, point, &support));
2123         PetscCall(DMPlexGetSupportSize(dm, point, &supportSize));
2124         for (s = 0; s < supportSize; ++s) {
2125           PetscCall(DMPlexGetCone(dm, support[s], &cone));
2126           PetscCall(DMPlexGetConeSize(dm, support[s], &coneSize));
2127           PetscCheck(coneSize == 2, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Edge %" PetscInt_FMT " has %" PetscInt_FMT " vertices != 2", support[s], coneSize);
2128           PetscCall(DMLabelGetValue(blabel, cone[0], &valA));
2129           PetscCall(DMLabelGetValue(blabel, cone[1], &valB));
2130           PetscCall(DMLabelGetValue(blabel, support[s], &valE));
2131           if ((valE < 0) && (valA >= 0) && (valB >= 0) && (cone[0] != cone[1])) PetscCall(DMLabelSetValue(blabel, support[s], 2));
2132         }
2133       }
2134     }
2135     PetscCall(ISRestoreIndices(bdIS, &points));
2136     PetscCall(ISDestroy(&bdIS));
2137   }
2138   /* Mark ghost fault cells */
2139   {
2140     PetscSF         sf;
2141     const PetscInt *leaves;
2142     PetscInt        Nl, l;
2143 
2144     PetscCall(DMGetPointSF(dm, &sf));
2145     PetscCall(PetscSFGetGraph(sf, NULL, &Nl, &leaves, NULL));
2146     PetscCall(DMLabelGetStratumIS(label, dim - 1, &dimIS));
2147     if (!dimIS) goto divide;
2148     PetscCall(ISGetLocalSize(dimIS, &numPoints));
2149     PetscCall(ISGetIndices(dimIS, &points));
2150     if (Nl > 0) {
2151       for (p = 0; p < numPoints; ++p) {
2152         const PetscInt point = points[p];
2153         PetscInt       val;
2154 
2155         PetscCall(PetscFindInt(point, Nl, leaves, &l));
2156         if (l >= 0) {
2157           PetscInt *closure = NULL;
2158           PetscInt  closureSize, cl;
2159 
2160           PetscCall(DMLabelGetValue(label, point, &val));
2161           PetscCheck((val == dim - 1) || (val == shift2 + dim - 1), PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %" PetscInt_FMT " has label value %" PetscInt_FMT ", should be a fault face", point, val);
2162           PetscCall(DMLabelClearValue(label, point, val));
2163           PetscCall(DMLabelSetValue(label, point, shift3 + val));
2164           PetscCall(DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
2165           for (cl = 2; cl < closureSize * 2; cl += 2) {
2166             const PetscInt clp = closure[cl];
2167 
2168             PetscCall(DMLabelGetValue(label, clp, &val));
2169             PetscCheck(val != -1, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %" PetscInt_FMT " is missing from label, but is in the closure of a fault face", point);
2170             PetscCall(DMLabelClearValue(label, clp, val));
2171             PetscCall(DMLabelSetValue(label, clp, shift3 + val));
2172           }
2173           PetscCall(DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure));
2174         }
2175       }
2176     }
2177     PetscCall(ISRestoreIndices(dimIS, &points));
2178     PetscCall(ISDestroy(&dimIS));
2179   }
2180 divide:
2181   if (subpointIS) PetscCall(ISRestoreIndices(subpointIS, &subpoints));
2182   PetscCall(DMPlexLabelFaultHalo(dm, label));
2183   PetscCall(CheckFaultEdge_Private(dm, label));
2184   PetscFunctionReturn(0);
2185 }
2186 
2187 /* Check that no cell have all vertices on the fault */
2188 PetscErrorCode DMPlexCheckValidSubmesh_Private(DM dm, DMLabel label, DM subdm)
2189 {
2190   IS              subpointIS;
2191   const PetscInt *dmpoints;
2192   PetscInt        defaultValue, cStart, cEnd, c, vStart, vEnd;
2193 
2194   PetscFunctionBegin;
2195   if (!label) PetscFunctionReturn(0);
2196   PetscCall(DMLabelGetDefaultValue(label, &defaultValue));
2197   PetscCall(DMPlexGetSubpointIS(subdm, &subpointIS));
2198   if (!subpointIS) PetscFunctionReturn(0);
2199   PetscCall(DMPlexGetHeightStratum(subdm, 0, &cStart, &cEnd));
2200   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
2201   PetscCall(ISGetIndices(subpointIS, &dmpoints));
2202   for (c = cStart; c < cEnd; ++c) {
2203     PetscBool invalidCell = PETSC_TRUE;
2204     PetscInt *closure     = NULL;
2205     PetscInt  closureSize, cl;
2206 
2207     PetscCall(DMPlexGetTransitiveClosure(dm, dmpoints[c], PETSC_TRUE, &closureSize, &closure));
2208     for (cl = 0; cl < closureSize * 2; cl += 2) {
2209       PetscInt value = 0;
2210 
2211       if ((closure[cl] < vStart) || (closure[cl] >= vEnd)) continue;
2212       PetscCall(DMLabelGetValue(label, closure[cl], &value));
2213       if (value == defaultValue) {
2214         invalidCell = PETSC_FALSE;
2215         break;
2216       }
2217     }
2218     PetscCall(DMPlexRestoreTransitiveClosure(dm, dmpoints[c], PETSC_TRUE, &closureSize, &closure));
2219     if (invalidCell) {
2220       PetscCall(ISRestoreIndices(subpointIS, &dmpoints));
2221       PetscCall(ISDestroy(&subpointIS));
2222       PetscCall(DMDestroy(&subdm));
2223       SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Ambiguous submesh. Cell %" PetscInt_FMT " has all of its vertices on the submesh.", dmpoints[c]);
2224     }
2225   }
2226   PetscCall(ISRestoreIndices(subpointIS, &dmpoints));
2227   PetscFunctionReturn(0);
2228 }
2229 
2230 /*@
2231   DMPlexCreateHybridMesh - Create a mesh with hybrid cells along an internal interface
2232 
2233   Collective on dm
2234 
2235   Input Parameters:
2236 + dm - The original DM
2237 . label - The label specifying the interface vertices
2238 . bdlabel - The optional label specifying the interface boundary vertices
2239 - bdvalue - Value of optional label specifying the interface boundary vertices
2240 
2241   Output Parameters:
2242 + hybridLabel - The label fully marking the interface, or NULL if no output is desired
2243 . splitLabel - The label containing the split points, or NULL if no output is desired
2244 . dmInterface - The new interface DM, or NULL
2245 - dmHybrid - The new DM with cohesive cells
2246 
2247   Note: The hybridLabel indicates what parts of the original mesh impinged on the on division surface. For points
2248   directly on the division surface, they are labeled with their dimension, so an edge 7 on the division surface would be
2249   7 (1) in hybridLabel. For points that impinge from the positive side, they are labeled with 100+dim, so an edge 6 with
2250   one vertex 3 on the surface would be 6 (101) and 3 (0) in hybridLabel. If an edge 9 from the negative side of the
2251   surface also hits vertex 3, it would be 9 (-101) in hybridLabel.
2252 
2253   The splitLabel indicates what points in the new hybrid mesh were the result of splitting points in the original
2254   mesh. The label value is +=100+dim for each point. For example, if two edges 10 and 14 in the hybrid resulting from
2255   splitting an edge in the original mesh, you would have 10 (101) and 14 (-101) in the splitLabel.
2256 
2257   The dmInterface is a DM built from the original division surface. It has a label which can be retrieved using
2258   DMPlexGetSubpointMap() which maps each point back to the point in the surface of the original mesh.
2259 
2260   Level: developer
2261 
2262 .seealso: `DMPlexConstructCohesiveCells()`, `DMPlexLabelCohesiveComplete()`, `DMPlexGetSubpointMap()`, `DMCreate()`
2263 @*/
2264 PetscErrorCode DMPlexCreateHybridMesh(DM dm, DMLabel label, DMLabel bdlabel, PetscInt bdvalue, DMLabel *hybridLabel, DMLabel *splitLabel, DM *dmInterface, DM *dmHybrid)
2265 {
2266   DM       idm;
2267   DMLabel  subpointMap, hlabel, slabel = NULL;
2268   PetscInt dim;
2269 
2270   PetscFunctionBegin;
2271   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2272   if (label) PetscValidPointer(label, 2);
2273   if (bdlabel) PetscValidPointer(bdlabel, 3);
2274   if (hybridLabel) PetscValidPointer(hybridLabel, 5);
2275   if (splitLabel) PetscValidPointer(splitLabel, 6);
2276   if (dmInterface) PetscValidPointer(dmInterface, 7);
2277   PetscValidPointer(dmHybrid, 8);
2278   PetscCall(DMGetDimension(dm, &dim));
2279   PetscCall(DMPlexCreateSubmesh(dm, label, 1, PETSC_FALSE, &idm));
2280   PetscCall(DMPlexCheckValidSubmesh_Private(dm, label, idm));
2281   PetscCall(DMPlexOrient(idm));
2282   PetscCall(DMPlexGetSubpointMap(idm, &subpointMap));
2283   PetscCall(DMLabelDuplicate(subpointMap, &hlabel));
2284   PetscCall(DMLabelClearStratum(hlabel, dim));
2285   if (splitLabel) {
2286     const char *name;
2287     char        sname[PETSC_MAX_PATH_LEN];
2288 
2289     PetscCall(PetscObjectGetName((PetscObject)hlabel, &name));
2290     PetscCall(PetscStrncpy(sname, name, PETSC_MAX_PATH_LEN));
2291     PetscCall(PetscStrcat(sname, " split"));
2292     PetscCall(DMLabelCreate(PETSC_COMM_SELF, sname, &slabel));
2293   }
2294   PetscCall(DMPlexLabelCohesiveComplete(dm, hlabel, bdlabel, bdvalue, PETSC_FALSE, idm));
2295   if (dmInterface) {
2296     *dmInterface = idm;
2297   } else PetscCall(DMDestroy(&idm));
2298   PetscCall(DMPlexConstructCohesiveCells(dm, hlabel, slabel, dmHybrid));
2299   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_TRUE, *dmHybrid));
2300   if (hybridLabel) *hybridLabel = hlabel;
2301   else PetscCall(DMLabelDestroy(&hlabel));
2302   if (splitLabel) *splitLabel = slabel;
2303   {
2304     DM      cdm;
2305     DMLabel ctLabel;
2306 
2307     /* We need to somehow share the celltype label with the coordinate dm */
2308     PetscCall(DMGetCoordinateDM(*dmHybrid, &cdm));
2309     PetscCall(DMPlexGetCellTypeLabel(*dmHybrid, &ctLabel));
2310     PetscCall(DMSetLabel(cdm, ctLabel));
2311   }
2312   PetscFunctionReturn(0);
2313 }
2314 
2315 /* Here we need the explicit assumption that:
2316 
2317      For any marked cell, the marked vertices constitute a single face
2318 */
2319 static PetscErrorCode DMPlexMarkSubmesh_Uninterpolated(DM dm, DMLabel vertexLabel, PetscInt value, DMLabel subpointMap, PetscInt *numFaces, PetscInt *nFV, DM subdm)
2320 {
2321   IS              subvertexIS = NULL;
2322   const PetscInt *subvertices;
2323   PetscInt       *pStart, *pEnd, pSize;
2324   PetscInt        depth, dim, d, numSubVerticesInitial = 0, v;
2325 
2326   PetscFunctionBegin;
2327   *numFaces = 0;
2328   *nFV      = 0;
2329   PetscCall(DMPlexGetDepth(dm, &depth));
2330   PetscCall(DMGetDimension(dm, &dim));
2331   pSize = PetscMax(depth, dim) + 1;
2332   PetscCall(PetscMalloc2(pSize, &pStart, pSize, &pEnd));
2333   for (d = 0; d <= depth; ++d) PetscCall(DMPlexGetSimplexOrBoxCells(dm, depth - d, &pStart[d], &pEnd[d]));
2334   /* Loop over initial vertices and mark all faces in the collective star() */
2335   if (vertexLabel) PetscCall(DMLabelGetStratumIS(vertexLabel, value, &subvertexIS));
2336   if (subvertexIS) {
2337     PetscCall(ISGetSize(subvertexIS, &numSubVerticesInitial));
2338     PetscCall(ISGetIndices(subvertexIS, &subvertices));
2339   }
2340   for (v = 0; v < numSubVerticesInitial; ++v) {
2341     const PetscInt vertex = subvertices[v];
2342     PetscInt      *star   = NULL;
2343     PetscInt       starSize, s, numCells = 0, c;
2344 
2345     PetscCall(DMPlexGetTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star));
2346     for (s = 0; s < starSize * 2; s += 2) {
2347       const PetscInt point = star[s];
2348       if ((point >= pStart[depth]) && (point < pEnd[depth])) star[numCells++] = point;
2349     }
2350     for (c = 0; c < numCells; ++c) {
2351       const PetscInt cell    = star[c];
2352       PetscInt      *closure = NULL;
2353       PetscInt       closureSize, cl;
2354       PetscInt       cellLoc, numCorners = 0, faceSize = 0;
2355 
2356       PetscCall(DMLabelGetValue(subpointMap, cell, &cellLoc));
2357       if (cellLoc == 2) continue;
2358       PetscCheck(cellLoc < 0, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Cell %" PetscInt_FMT " has dimension %" PetscInt_FMT " in the surface label", cell, cellLoc);
2359       PetscCall(DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
2360       for (cl = 0; cl < closureSize * 2; cl += 2) {
2361         const PetscInt point = closure[cl];
2362         PetscInt       vertexLoc;
2363 
2364         if ((point >= pStart[0]) && (point < pEnd[0])) {
2365           ++numCorners;
2366           PetscCall(DMLabelGetValue(vertexLabel, point, &vertexLoc));
2367           if (vertexLoc == value) closure[faceSize++] = point;
2368         }
2369       }
2370       if (!(*nFV)) PetscCall(DMPlexGetNumFaceVertices(dm, dim, numCorners, nFV));
2371       PetscCheck(faceSize <= *nFV, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Invalid submesh: Too many vertices %" PetscInt_FMT " of an element on the surface", faceSize);
2372       if (faceSize == *nFV) {
2373         const PetscInt *cells = NULL;
2374         PetscInt        numCells, nc;
2375 
2376         ++(*numFaces);
2377         for (cl = 0; cl < faceSize; ++cl) PetscCall(DMLabelSetValue(subpointMap, closure[cl], 0));
2378         PetscCall(DMPlexGetJoin(dm, faceSize, closure, &numCells, &cells));
2379         for (nc = 0; nc < numCells; ++nc) PetscCall(DMLabelSetValue(subpointMap, cells[nc], 2));
2380         PetscCall(DMPlexRestoreJoin(dm, faceSize, closure, &numCells, &cells));
2381       }
2382       PetscCall(DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
2383     }
2384     PetscCall(DMPlexRestoreTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star));
2385   }
2386   if (subvertexIS) PetscCall(ISRestoreIndices(subvertexIS, &subvertices));
2387   PetscCall(ISDestroy(&subvertexIS));
2388   PetscCall(PetscFree2(pStart, pEnd));
2389   PetscFunctionReturn(0);
2390 }
2391 
2392 static PetscErrorCode DMPlexMarkSubmesh_Interpolated(DM dm, DMLabel vertexLabel, PetscInt value, PetscBool markedFaces, DMLabel subpointMap, DM subdm)
2393 {
2394   IS              subvertexIS = NULL;
2395   const PetscInt *subvertices;
2396   PetscInt       *pStart, *pEnd;
2397   PetscInt        dim, d, numSubVerticesInitial = 0, v;
2398 
2399   PetscFunctionBegin;
2400   PetscCall(DMGetDimension(dm, &dim));
2401   PetscCall(PetscMalloc2(dim + 1, &pStart, dim + 1, &pEnd));
2402   for (d = 0; d <= dim; ++d) PetscCall(DMPlexGetSimplexOrBoxCells(dm, dim - d, &pStart[d], &pEnd[d]));
2403   /* Loop over initial vertices and mark all faces in the collective star() */
2404   if (vertexLabel) {
2405     PetscCall(DMLabelGetStratumIS(vertexLabel, value, &subvertexIS));
2406     if (subvertexIS) {
2407       PetscCall(ISGetSize(subvertexIS, &numSubVerticesInitial));
2408       PetscCall(ISGetIndices(subvertexIS, &subvertices));
2409     }
2410   }
2411   for (v = 0; v < numSubVerticesInitial; ++v) {
2412     const PetscInt vertex = subvertices[v];
2413     PetscInt      *star   = NULL;
2414     PetscInt       starSize, s, numFaces = 0, f;
2415 
2416     PetscCall(DMPlexGetTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star));
2417     for (s = 0; s < starSize * 2; s += 2) {
2418       const PetscInt point = star[s];
2419       PetscInt       faceLoc;
2420 
2421       if ((point >= pStart[dim - 1]) && (point < pEnd[dim - 1])) {
2422         if (markedFaces) {
2423           PetscCall(DMLabelGetValue(vertexLabel, point, &faceLoc));
2424           if (faceLoc < 0) continue;
2425         }
2426         star[numFaces++] = point;
2427       }
2428     }
2429     for (f = 0; f < numFaces; ++f) {
2430       const PetscInt face    = star[f];
2431       PetscInt      *closure = NULL;
2432       PetscInt       closureSize, c;
2433       PetscInt       faceLoc;
2434 
2435       PetscCall(DMLabelGetValue(subpointMap, face, &faceLoc));
2436       if (faceLoc == dim - 1) continue;
2437       PetscCheck(faceLoc < 0, PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Face %" PetscInt_FMT " has dimension %" PetscInt_FMT " in the surface label", face, faceLoc);
2438       PetscCall(DMPlexGetTransitiveClosure(dm, face, PETSC_TRUE, &closureSize, &closure));
2439       for (c = 0; c < closureSize * 2; c += 2) {
2440         const PetscInt point = closure[c];
2441         PetscInt       vertexLoc;
2442 
2443         if ((point >= pStart[0]) && (point < pEnd[0])) {
2444           PetscCall(DMLabelGetValue(vertexLabel, point, &vertexLoc));
2445           if (vertexLoc != value) break;
2446         }
2447       }
2448       if (c == closureSize * 2) {
2449         const PetscInt *support;
2450         PetscInt        supportSize, s;
2451 
2452         for (c = 0; c < closureSize * 2; c += 2) {
2453           const PetscInt point = closure[c];
2454 
2455           for (d = 0; d < dim; ++d) {
2456             if ((point >= pStart[d]) && (point < pEnd[d])) {
2457               PetscCall(DMLabelSetValue(subpointMap, point, d));
2458               break;
2459             }
2460           }
2461         }
2462         PetscCall(DMPlexGetSupportSize(dm, face, &supportSize));
2463         PetscCall(DMPlexGetSupport(dm, face, &support));
2464         for (s = 0; s < supportSize; ++s) PetscCall(DMLabelSetValue(subpointMap, support[s], dim));
2465       }
2466       PetscCall(DMPlexRestoreTransitiveClosure(dm, face, PETSC_TRUE, &closureSize, &closure));
2467     }
2468     PetscCall(DMPlexRestoreTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star));
2469   }
2470   if (subvertexIS) PetscCall(ISRestoreIndices(subvertexIS, &subvertices));
2471   PetscCall(ISDestroy(&subvertexIS));
2472   PetscCall(PetscFree2(pStart, pEnd));
2473   PetscFunctionReturn(0);
2474 }
2475 
2476 static PetscErrorCode DMPlexMarkCohesiveSubmesh_Uninterpolated(DM dm, PetscBool hasLagrange, const char labelname[], PetscInt value, DMLabel subpointMap, PetscInt *numFaces, PetscInt *nFV, PetscInt *subCells[], DM subdm)
2477 {
2478   DMLabel         label = NULL;
2479   const PetscInt *cone;
2480   PetscInt        dim, cMax, cEnd, c, subc = 0, p, coneSize = -1;
2481 
2482   PetscFunctionBegin;
2483   *numFaces = 0;
2484   *nFV      = 0;
2485   if (labelname) PetscCall(DMGetLabel(dm, labelname, &label));
2486   *subCells = NULL;
2487   PetscCall(DMGetDimension(dm, &dim));
2488   PetscCall(DMPlexGetTensorPrismBounds_Internal(dm, dim, &cMax, &cEnd));
2489   if (cMax < 0) PetscFunctionReturn(0);
2490   if (label) {
2491     for (c = cMax; c < cEnd; ++c) {
2492       PetscInt val;
2493 
2494       PetscCall(DMLabelGetValue(label, c, &val));
2495       if (val == value) {
2496         ++(*numFaces);
2497         PetscCall(DMPlexGetConeSize(dm, c, &coneSize));
2498       }
2499     }
2500   } else {
2501     *numFaces = cEnd - cMax;
2502     PetscCall(DMPlexGetConeSize(dm, cMax, &coneSize));
2503   }
2504   PetscCall(PetscMalloc1(*numFaces * 2, subCells));
2505   if (!(*numFaces)) PetscFunctionReturn(0);
2506   *nFV = hasLagrange ? coneSize / 3 : coneSize / 2;
2507   for (c = cMax; c < cEnd; ++c) {
2508     const PetscInt *cells;
2509     PetscInt        numCells;
2510 
2511     if (label) {
2512       PetscInt val;
2513 
2514       PetscCall(DMLabelGetValue(label, c, &val));
2515       if (val != value) continue;
2516     }
2517     PetscCall(DMPlexGetCone(dm, c, &cone));
2518     for (p = 0; p < *nFV; ++p) PetscCall(DMLabelSetValue(subpointMap, cone[p], 0));
2519     /* Negative face */
2520     PetscCall(DMPlexGetJoin(dm, *nFV, cone, &numCells, &cells));
2521     /* Not true in parallel
2522     PetscCheck(numCells == 2,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive cells should separate two cells"); */
2523     for (p = 0; p < numCells; ++p) {
2524       PetscCall(DMLabelSetValue(subpointMap, cells[p], 2));
2525       (*subCells)[subc++] = cells[p];
2526     }
2527     PetscCall(DMPlexRestoreJoin(dm, *nFV, cone, &numCells, &cells));
2528     /* Positive face is not included */
2529   }
2530   PetscFunctionReturn(0);
2531 }
2532 
2533 static PetscErrorCode DMPlexMarkCohesiveSubmesh_Interpolated(DM dm, DMLabel label, PetscInt value, DMLabel subpointMap, DM subdm)
2534 {
2535   PetscInt *pStart, *pEnd;
2536   PetscInt  dim, cMax, cEnd, c, d;
2537 
2538   PetscFunctionBegin;
2539   PetscCall(DMGetDimension(dm, &dim));
2540   PetscCall(DMPlexGetTensorPrismBounds_Internal(dm, dim, &cMax, &cEnd));
2541   if (cMax < 0) PetscFunctionReturn(0);
2542   PetscCall(PetscMalloc2(dim + 1, &pStart, dim + 1, &pEnd));
2543   for (d = 0; d <= dim; ++d) PetscCall(DMPlexGetDepthStratum(dm, d, &pStart[d], &pEnd[d]));
2544   for (c = cMax; c < cEnd; ++c) {
2545     const PetscInt *cone;
2546     PetscInt       *closure = NULL;
2547     PetscInt        fconeSize, coneSize, closureSize, cl, val;
2548 
2549     if (label) {
2550       PetscCall(DMLabelGetValue(label, c, &val));
2551       if (val != value) continue;
2552     }
2553     PetscCall(DMPlexGetConeSize(dm, c, &coneSize));
2554     PetscCall(DMPlexGetCone(dm, c, &cone));
2555     PetscCall(DMPlexGetConeSize(dm, cone[0], &fconeSize));
2556     PetscCheck(coneSize == (fconeSize ? fconeSize : 1) + 2, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive cells should separate two cells");
2557     /* Negative face */
2558     PetscCall(DMPlexGetTransitiveClosure(dm, cone[0], PETSC_TRUE, &closureSize, &closure));
2559     for (cl = 0; cl < closureSize * 2; cl += 2) {
2560       const PetscInt point = closure[cl];
2561 
2562       for (d = 0; d <= dim; ++d) {
2563         if ((point >= pStart[d]) && (point < pEnd[d])) {
2564           PetscCall(DMLabelSetValue(subpointMap, point, d));
2565           break;
2566         }
2567       }
2568     }
2569     PetscCall(DMPlexRestoreTransitiveClosure(dm, cone[0], PETSC_TRUE, &closureSize, &closure));
2570     /* Cells -- positive face is not included */
2571     for (cl = 0; cl < 1; ++cl) {
2572       const PetscInt *support;
2573       PetscInt        supportSize, s;
2574 
2575       PetscCall(DMPlexGetSupportSize(dm, cone[cl], &supportSize));
2576       /* PetscCheck(supportSize == 2,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive faces should separate two cells"); */
2577       PetscCall(DMPlexGetSupport(dm, cone[cl], &support));
2578       for (s = 0; s < supportSize; ++s) PetscCall(DMLabelSetValue(subpointMap, support[s], dim));
2579     }
2580   }
2581   PetscCall(PetscFree2(pStart, pEnd));
2582   PetscFunctionReturn(0);
2583 }
2584 
2585 static PetscErrorCode DMPlexGetFaceOrientation(DM dm, PetscInt cell, PetscInt numCorners, PetscInt indices[], PetscInt oppositeVertex, PetscInt origVertices[], PetscInt faceVertices[], PetscBool *posOriented)
2586 {
2587   MPI_Comm       comm;
2588   PetscBool      posOrient = PETSC_FALSE;
2589   const PetscInt debug     = 0;
2590   PetscInt       cellDim, faceSize, f;
2591 
2592   PetscFunctionBegin;
2593   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
2594   PetscCall(DMGetDimension(dm, &cellDim));
2595   if (debug) PetscCall(PetscPrintf(comm, "cellDim: %" PetscInt_FMT " numCorners: %" PetscInt_FMT "\n", cellDim, numCorners));
2596 
2597   if (cellDim == 1 && numCorners == 2) {
2598     /* Triangle */
2599     faceSize  = numCorners - 1;
2600     posOrient = !(oppositeVertex % 2) ? PETSC_TRUE : PETSC_FALSE;
2601   } else if (cellDim == 2 && numCorners == 3) {
2602     /* Triangle */
2603     faceSize  = numCorners - 1;
2604     posOrient = !(oppositeVertex % 2) ? PETSC_TRUE : PETSC_FALSE;
2605   } else if (cellDim == 3 && numCorners == 4) {
2606     /* Tetrahedron */
2607     faceSize  = numCorners - 1;
2608     posOrient = (oppositeVertex % 2) ? PETSC_TRUE : PETSC_FALSE;
2609   } else if (cellDim == 1 && numCorners == 3) {
2610     /* Quadratic line */
2611     faceSize  = 1;
2612     posOrient = PETSC_TRUE;
2613   } else if (cellDim == 2 && numCorners == 4) {
2614     /* Quads */
2615     faceSize = 2;
2616     if ((indices[1] > indices[0]) && (indices[1] - indices[0] == 1)) {
2617       posOrient = PETSC_TRUE;
2618     } else if ((indices[0] == 3) && (indices[1] == 0)) {
2619       posOrient = PETSC_TRUE;
2620     } else {
2621       if (((indices[0] > indices[1]) && (indices[0] - indices[1] == 1)) || ((indices[0] == 0) && (indices[1] == 3))) {
2622         posOrient = PETSC_FALSE;
2623       } else SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid quad crossedge");
2624     }
2625   } else if (cellDim == 2 && numCorners == 6) {
2626     /* Quadratic triangle (I hate this) */
2627     /* Edges are determined by the first 2 vertices (corners of edges) */
2628     const PetscInt faceSizeTri = 3;
2629     PetscInt       sortedIndices[3], i, iFace;
2630     PetscBool      found                    = PETSC_FALSE;
2631     PetscInt       faceVerticesTriSorted[9] = {
2632       0, 3, 4, /* bottom */
2633       1, 4, 5, /* right */
2634       2, 3, 5, /* left */
2635     };
2636     PetscInt faceVerticesTri[9] = {
2637       0, 3, 4, /* bottom */
2638       1, 4, 5, /* right */
2639       2, 5, 3, /* left */
2640     };
2641 
2642     for (i = 0; i < faceSizeTri; ++i) sortedIndices[i] = indices[i];
2643     PetscCall(PetscSortInt(faceSizeTri, sortedIndices));
2644     for (iFace = 0; iFace < 3; ++iFace) {
2645       const PetscInt ii = iFace * faceSizeTri;
2646       PetscInt       fVertex, cVertex;
2647 
2648       if ((sortedIndices[0] == faceVerticesTriSorted[ii + 0]) && (sortedIndices[1] == faceVerticesTriSorted[ii + 1])) {
2649         for (fVertex = 0; fVertex < faceSizeTri; ++fVertex) {
2650           for (cVertex = 0; cVertex < faceSizeTri; ++cVertex) {
2651             if (indices[cVertex] == faceVerticesTri[ii + fVertex]) {
2652               faceVertices[fVertex] = origVertices[cVertex];
2653               break;
2654             }
2655           }
2656         }
2657         found = PETSC_TRUE;
2658         break;
2659       }
2660     }
2661     PetscCheck(found, comm, PETSC_ERR_ARG_WRONG, "Invalid tri crossface");
2662     if (posOriented) *posOriented = PETSC_TRUE;
2663     PetscFunctionReturn(0);
2664   } else if (cellDim == 2 && numCorners == 9) {
2665     /* Quadratic quad (I hate this) */
2666     /* Edges are determined by the first 2 vertices (corners of edges) */
2667     const PetscInt faceSizeQuad = 3;
2668     PetscInt       sortedIndices[3], i, iFace;
2669     PetscBool      found                      = PETSC_FALSE;
2670     PetscInt       faceVerticesQuadSorted[12] = {
2671       0, 1, 4, /* bottom */
2672       1, 2, 5, /* right */
2673       2, 3, 6, /* top */
2674       0, 3, 7, /* left */
2675     };
2676     PetscInt faceVerticesQuad[12] = {
2677       0, 1, 4, /* bottom */
2678       1, 2, 5, /* right */
2679       2, 3, 6, /* top */
2680       3, 0, 7, /* left */
2681     };
2682 
2683     for (i = 0; i < faceSizeQuad; ++i) sortedIndices[i] = indices[i];
2684     PetscCall(PetscSortInt(faceSizeQuad, sortedIndices));
2685     for (iFace = 0; iFace < 4; ++iFace) {
2686       const PetscInt ii = iFace * faceSizeQuad;
2687       PetscInt       fVertex, cVertex;
2688 
2689       if ((sortedIndices[0] == faceVerticesQuadSorted[ii + 0]) && (sortedIndices[1] == faceVerticesQuadSorted[ii + 1])) {
2690         for (fVertex = 0; fVertex < faceSizeQuad; ++fVertex) {
2691           for (cVertex = 0; cVertex < faceSizeQuad; ++cVertex) {
2692             if (indices[cVertex] == faceVerticesQuad[ii + fVertex]) {
2693               faceVertices[fVertex] = origVertices[cVertex];
2694               break;
2695             }
2696           }
2697         }
2698         found = PETSC_TRUE;
2699         break;
2700       }
2701     }
2702     PetscCheck(found, comm, PETSC_ERR_ARG_WRONG, "Invalid quad crossface");
2703     if (posOriented) *posOriented = PETSC_TRUE;
2704     PetscFunctionReturn(0);
2705   } else if (cellDim == 3 && numCorners == 8) {
2706     /* Hexes
2707        A hex is two oriented quads with the normal of the first
2708        pointing up at the second.
2709 
2710           7---6
2711          /|  /|
2712         4---5 |
2713         | 1-|-2
2714         |/  |/
2715         0---3
2716 
2717         Faces are determined by the first 4 vertices (corners of faces) */
2718     const PetscInt faceSizeHex = 4;
2719     PetscInt       sortedIndices[4], i, iFace;
2720     PetscBool      found                     = PETSC_FALSE;
2721     PetscInt       faceVerticesHexSorted[24] = {
2722       0, 1, 2, 3, /* bottom */
2723       4, 5, 6, 7, /* top */
2724       0, 3, 4, 5, /* front */
2725       2, 3, 5, 6, /* right */
2726       1, 2, 6, 7, /* back */
2727       0, 1, 4, 7, /* left */
2728     };
2729     PetscInt faceVerticesHex[24] = {
2730       1, 2, 3, 0, /* bottom */
2731       4, 5, 6, 7, /* top */
2732       0, 3, 5, 4, /* front */
2733       3, 2, 6, 5, /* right */
2734       2, 1, 7, 6, /* back */
2735       1, 0, 4, 7, /* left */
2736     };
2737 
2738     for (i = 0; i < faceSizeHex; ++i) sortedIndices[i] = indices[i];
2739     PetscCall(PetscSortInt(faceSizeHex, sortedIndices));
2740     for (iFace = 0; iFace < 6; ++iFace) {
2741       const PetscInt ii = iFace * faceSizeHex;
2742       PetscInt       fVertex, cVertex;
2743 
2744       if ((sortedIndices[0] == faceVerticesHexSorted[ii + 0]) && (sortedIndices[1] == faceVerticesHexSorted[ii + 1]) && (sortedIndices[2] == faceVerticesHexSorted[ii + 2]) && (sortedIndices[3] == faceVerticesHexSorted[ii + 3])) {
2745         for (fVertex = 0; fVertex < faceSizeHex; ++fVertex) {
2746           for (cVertex = 0; cVertex < faceSizeHex; ++cVertex) {
2747             if (indices[cVertex] == faceVerticesHex[ii + fVertex]) {
2748               faceVertices[fVertex] = origVertices[cVertex];
2749               break;
2750             }
2751           }
2752         }
2753         found = PETSC_TRUE;
2754         break;
2755       }
2756     }
2757     PetscCheck(found, comm, PETSC_ERR_ARG_WRONG, "Invalid hex crossface");
2758     if (posOriented) *posOriented = PETSC_TRUE;
2759     PetscFunctionReturn(0);
2760   } else if (cellDim == 3 && numCorners == 10) {
2761     /* Quadratic tet */
2762     /* Faces are determined by the first 3 vertices (corners of faces) */
2763     const PetscInt faceSizeTet = 6;
2764     PetscInt       sortedIndices[6], i, iFace;
2765     PetscBool      found                     = PETSC_FALSE;
2766     PetscInt       faceVerticesTetSorted[24] = {
2767       0, 1, 2, 6, 7, 8, /* bottom */
2768       0, 3, 4, 6, 7, 9, /* front */
2769       1, 4, 5, 7, 8, 9, /* right */
2770       2, 3, 5, 6, 8, 9, /* left */
2771     };
2772     PetscInt faceVerticesTet[24] = {
2773       0, 1, 2, 6, 7, 8, /* bottom */
2774       0, 4, 3, 6, 7, 9, /* front */
2775       1, 5, 4, 7, 8, 9, /* right */
2776       2, 3, 5, 8, 6, 9, /* left */
2777     };
2778 
2779     for (i = 0; i < faceSizeTet; ++i) sortedIndices[i] = indices[i];
2780     PetscCall(PetscSortInt(faceSizeTet, sortedIndices));
2781     for (iFace = 0; iFace < 4; ++iFace) {
2782       const PetscInt ii = iFace * faceSizeTet;
2783       PetscInt       fVertex, cVertex;
2784 
2785       if ((sortedIndices[0] == faceVerticesTetSorted[ii + 0]) && (sortedIndices[1] == faceVerticesTetSorted[ii + 1]) && (sortedIndices[2] == faceVerticesTetSorted[ii + 2]) && (sortedIndices[3] == faceVerticesTetSorted[ii + 3])) {
2786         for (fVertex = 0; fVertex < faceSizeTet; ++fVertex) {
2787           for (cVertex = 0; cVertex < faceSizeTet; ++cVertex) {
2788             if (indices[cVertex] == faceVerticesTet[ii + fVertex]) {
2789               faceVertices[fVertex] = origVertices[cVertex];
2790               break;
2791             }
2792           }
2793         }
2794         found = PETSC_TRUE;
2795         break;
2796       }
2797     }
2798     PetscCheck(found, comm, PETSC_ERR_ARG_WRONG, "Invalid tet crossface");
2799     if (posOriented) *posOriented = PETSC_TRUE;
2800     PetscFunctionReturn(0);
2801   } else if (cellDim == 3 && numCorners == 27) {
2802     /* Quadratic hexes (I hate this)
2803        A hex is two oriented quads with the normal of the first
2804        pointing up at the second.
2805 
2806          7---6
2807         /|  /|
2808        4---5 |
2809        | 3-|-2
2810        |/  |/
2811        0---1
2812 
2813        Faces are determined by the first 4 vertices (corners of faces) */
2814     const PetscInt faceSizeQuadHex = 9;
2815     PetscInt       sortedIndices[9], i, iFace;
2816     PetscBool      found                         = PETSC_FALSE;
2817     PetscInt       faceVerticesQuadHexSorted[54] = {
2818       0, 1, 2, 3, 8,  9,  10, 11, 24, /* bottom */
2819       4, 5, 6, 7, 12, 13, 14, 15, 25, /* top */
2820       0, 1, 4, 5, 8,  12, 16, 17, 22, /* front */
2821       1, 2, 5, 6, 9,  13, 17, 18, 21, /* right */
2822       2, 3, 6, 7, 10, 14, 18, 19, 23, /* back */
2823       0, 3, 4, 7, 11, 15, 16, 19, 20, /* left */
2824     };
2825     PetscInt faceVerticesQuadHex[54] = {
2826       3, 2, 1, 0, 10, 9,  8,  11, 24, /* bottom */
2827       4, 5, 6, 7, 12, 13, 14, 15, 25, /* top */
2828       0, 1, 5, 4, 8,  17, 12, 16, 22, /* front */
2829       1, 2, 6, 5, 9,  18, 13, 17, 21, /* right */
2830       2, 3, 7, 6, 10, 19, 14, 18, 23, /* back */
2831       3, 0, 4, 7, 11, 16, 15, 19, 20  /* left */
2832     };
2833 
2834     for (i = 0; i < faceSizeQuadHex; ++i) sortedIndices[i] = indices[i];
2835     PetscCall(PetscSortInt(faceSizeQuadHex, sortedIndices));
2836     for (iFace = 0; iFace < 6; ++iFace) {
2837       const PetscInt ii = iFace * faceSizeQuadHex;
2838       PetscInt       fVertex, cVertex;
2839 
2840       if ((sortedIndices[0] == faceVerticesQuadHexSorted[ii + 0]) && (sortedIndices[1] == faceVerticesQuadHexSorted[ii + 1]) && (sortedIndices[2] == faceVerticesQuadHexSorted[ii + 2]) && (sortedIndices[3] == faceVerticesQuadHexSorted[ii + 3])) {
2841         for (fVertex = 0; fVertex < faceSizeQuadHex; ++fVertex) {
2842           for (cVertex = 0; cVertex < faceSizeQuadHex; ++cVertex) {
2843             if (indices[cVertex] == faceVerticesQuadHex[ii + fVertex]) {
2844               faceVertices[fVertex] = origVertices[cVertex];
2845               break;
2846             }
2847           }
2848         }
2849         found = PETSC_TRUE;
2850         break;
2851       }
2852     }
2853     PetscCheck(found, comm, PETSC_ERR_ARG_WRONG, "Invalid hex crossface");
2854     if (posOriented) *posOriented = PETSC_TRUE;
2855     PetscFunctionReturn(0);
2856   } else SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Unknown cell type for faceOrientation().");
2857   if (!posOrient) {
2858     if (debug) PetscCall(PetscPrintf(comm, "  Reversing initial face orientation\n"));
2859     for (f = 0; f < faceSize; ++f) faceVertices[f] = origVertices[faceSize - 1 - f];
2860   } else {
2861     if (debug) PetscCall(PetscPrintf(comm, "  Keeping initial face orientation\n"));
2862     for (f = 0; f < faceSize; ++f) faceVertices[f] = origVertices[f];
2863   }
2864   if (posOriented) *posOriented = posOrient;
2865   PetscFunctionReturn(0);
2866 }
2867 
2868 /*@
2869   DMPlexGetOrientedFace - Given a cell and a face, as a set of vertices, return the oriented face, as a set of vertices,
2870   in faceVertices. The orientation is such that the face normal points out of the cell
2871 
2872   Not collective
2873 
2874   Input Parameters:
2875 + dm           - The original mesh
2876 . cell         - The cell mesh point
2877 . faceSize     - The number of vertices on the face
2878 . face         - The face vertices
2879 . numCorners   - The number of vertices on the cell
2880 . indices      - Local numbering of face vertices in cell cone
2881 - origVertices - Original face vertices
2882 
2883   Output Parameters:
2884 + faceVertices - The face vertices properly oriented
2885 - posOriented  - PETSC_TRUE if the face was oriented with outward normal
2886 
2887   Level: developer
2888 
2889 .seealso: `DMPlexGetCone()`
2890 @*/
2891 PetscErrorCode DMPlexGetOrientedFace(DM dm, PetscInt cell, PetscInt faceSize, const PetscInt face[], PetscInt numCorners, PetscInt indices[], PetscInt origVertices[], PetscInt faceVertices[], PetscBool *posOriented)
2892 {
2893   const PetscInt *cone = NULL;
2894   PetscInt        coneSize, v, f, v2;
2895   PetscInt        oppositeVertex = -1;
2896 
2897   PetscFunctionBegin;
2898   PetscCall(DMPlexGetConeSize(dm, cell, &coneSize));
2899   PetscCall(DMPlexGetCone(dm, cell, &cone));
2900   for (v = 0, v2 = 0; v < coneSize; ++v) {
2901     PetscBool found = PETSC_FALSE;
2902 
2903     for (f = 0; f < faceSize; ++f) {
2904       if (face[f] == cone[v]) {
2905         found = PETSC_TRUE;
2906         break;
2907       }
2908     }
2909     if (found) {
2910       indices[v2]      = v;
2911       origVertices[v2] = cone[v];
2912       ++v2;
2913     } else {
2914       oppositeVertex = v;
2915     }
2916   }
2917   PetscCall(DMPlexGetFaceOrientation(dm, cell, numCorners, indices, oppositeVertex, origVertices, faceVertices, posOriented));
2918   PetscFunctionReturn(0);
2919 }
2920 
2921 /*
2922   DMPlexInsertFace_Internal - Puts a face into the mesh
2923 
2924   Not collective
2925 
2926   Input Parameters:
2927   + dm              - The DMPlex
2928   . numFaceVertex   - The number of vertices in the face
2929   . faceVertices    - The vertices in the face for dm
2930   . subfaceVertices - The vertices in the face for subdm
2931   . numCorners      - The number of vertices in the cell
2932   . cell            - A cell in dm containing the face
2933   . subcell         - A cell in subdm containing the face
2934   . firstFace       - First face in the mesh
2935   - newFacePoint    - Next face in the mesh
2936 
2937   Output Parameters:
2938   . newFacePoint - Contains next face point number on input, updated on output
2939 
2940   Level: developer
2941 */
2942 static PetscErrorCode DMPlexInsertFace_Internal(DM dm, DM subdm, PetscInt numFaceVertices, const PetscInt faceVertices[], const PetscInt subfaceVertices[], PetscInt numCorners, PetscInt cell, PetscInt subcell, PetscInt firstFace, PetscInt *newFacePoint)
2943 {
2944   MPI_Comm        comm;
2945   DM_Plex        *submesh = (DM_Plex *)subdm->data;
2946   const PetscInt *faces;
2947   PetscInt        numFaces, coneSize;
2948 
2949   PetscFunctionBegin;
2950   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
2951   PetscCall(DMPlexGetConeSize(subdm, subcell, &coneSize));
2952   PetscCheck(coneSize == 1, comm, PETSC_ERR_ARG_OUTOFRANGE, "Cone size of cell %" PetscInt_FMT " is %" PetscInt_FMT " != 1", cell, coneSize);
2953 #if 0
2954   /* Cannot use this because support() has not been constructed yet */
2955   PetscCall(DMPlexGetJoin(subdm, numFaceVertices, subfaceVertices, &numFaces, &faces));
2956 #else
2957   {
2958     PetscInt f;
2959 
2960     numFaces = 0;
2961     PetscCall(DMGetWorkArray(subdm, 1, MPIU_INT, (void **)&faces));
2962     for (f = firstFace; f < *newFacePoint; ++f) {
2963       PetscInt dof, off, d;
2964 
2965       PetscCall(PetscSectionGetDof(submesh->coneSection, f, &dof));
2966       PetscCall(PetscSectionGetOffset(submesh->coneSection, f, &off));
2967       /* Yes, I know this is quadratic, but I expect the sizes to be <5 */
2968       for (d = 0; d < dof; ++d) {
2969         const PetscInt p = submesh->cones[off + d];
2970         PetscInt       v;
2971 
2972         for (v = 0; v < numFaceVertices; ++v) {
2973           if (subfaceVertices[v] == p) break;
2974         }
2975         if (v == numFaceVertices) break;
2976       }
2977       if (d == dof) {
2978         numFaces               = 1;
2979         ((PetscInt *)faces)[0] = f;
2980       }
2981     }
2982   }
2983 #endif
2984   PetscCheck(numFaces <= 1, comm, PETSC_ERR_ARG_WRONG, "Vertex set had %" PetscInt_FMT " faces, not one", numFaces);
2985   if (numFaces == 1) {
2986     /* Add the other cell neighbor for this face */
2987     PetscCall(DMPlexSetCone(subdm, subcell, faces));
2988   } else {
2989     PetscInt *indices, *origVertices, *orientedVertices, *orientedSubVertices, v, ov;
2990     PetscBool posOriented;
2991 
2992     PetscCall(DMGetWorkArray(subdm, 4 * numFaceVertices * sizeof(PetscInt), MPIU_INT, &orientedVertices));
2993     origVertices        = &orientedVertices[numFaceVertices];
2994     indices             = &orientedVertices[numFaceVertices * 2];
2995     orientedSubVertices = &orientedVertices[numFaceVertices * 3];
2996     PetscCall(DMPlexGetOrientedFace(dm, cell, numFaceVertices, faceVertices, numCorners, indices, origVertices, orientedVertices, &posOriented));
2997     /* TODO: I know that routine should return a permutation, not the indices */
2998     for (v = 0; v < numFaceVertices; ++v) {
2999       const PetscInt vertex = faceVertices[v], subvertex = subfaceVertices[v];
3000       for (ov = 0; ov < numFaceVertices; ++ov) {
3001         if (orientedVertices[ov] == vertex) {
3002           orientedSubVertices[ov] = subvertex;
3003           break;
3004         }
3005       }
3006       PetscCheck(ov != numFaceVertices, comm, PETSC_ERR_PLIB, "Could not find face vertex %" PetscInt_FMT " in orientated set", vertex);
3007     }
3008     PetscCall(DMPlexSetCone(subdm, *newFacePoint, orientedSubVertices));
3009     PetscCall(DMPlexSetCone(subdm, subcell, newFacePoint));
3010     PetscCall(DMRestoreWorkArray(subdm, 4 * numFaceVertices * sizeof(PetscInt), MPIU_INT, &orientedVertices));
3011     ++(*newFacePoint);
3012   }
3013 #if 0
3014   PetscCall(DMPlexRestoreJoin(subdm, numFaceVertices, subfaceVertices, &numFaces, &faces));
3015 #else
3016   PetscCall(DMRestoreWorkArray(subdm, 1, MPIU_INT, (void **)&faces));
3017 #endif
3018   PetscFunctionReturn(0);
3019 }
3020 
3021 static PetscErrorCode DMPlexCreateSubmesh_Uninterpolated(DM dm, DMLabel vertexLabel, PetscInt value, DM subdm)
3022 {
3023   MPI_Comm        comm;
3024   DMLabel         subpointMap;
3025   IS              subvertexIS, subcellIS;
3026   const PetscInt *subVertices, *subCells;
3027   PetscInt        numSubVertices, firstSubVertex, numSubCells;
3028   PetscInt       *subface, maxConeSize, numSubFaces = 0, firstSubFace, newFacePoint, nFV = 0;
3029   PetscInt        vStart, vEnd, c, f;
3030 
3031   PetscFunctionBegin;
3032   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
3033   /* Create subpointMap which marks the submesh */
3034   PetscCall(DMLabelCreate(PETSC_COMM_SELF, "subpoint_map", &subpointMap));
3035   PetscCall(DMPlexSetSubpointMap(subdm, subpointMap));
3036   PetscCall(DMLabelDestroy(&subpointMap));
3037   if (vertexLabel) PetscCall(DMPlexMarkSubmesh_Uninterpolated(dm, vertexLabel, value, subpointMap, &numSubFaces, &nFV, subdm));
3038   /* Setup chart */
3039   PetscCall(DMLabelGetStratumSize(subpointMap, 0, &numSubVertices));
3040   PetscCall(DMLabelGetStratumSize(subpointMap, 2, &numSubCells));
3041   PetscCall(DMPlexSetChart(subdm, 0, numSubCells + numSubFaces + numSubVertices));
3042   PetscCall(DMPlexSetVTKCellHeight(subdm, 1));
3043   /* Set cone sizes */
3044   firstSubVertex = numSubCells;
3045   firstSubFace   = numSubCells + numSubVertices;
3046   newFacePoint   = firstSubFace;
3047   PetscCall(DMLabelGetStratumIS(subpointMap, 0, &subvertexIS));
3048   if (subvertexIS) PetscCall(ISGetIndices(subvertexIS, &subVertices));
3049   PetscCall(DMLabelGetStratumIS(subpointMap, 2, &subcellIS));
3050   if (subcellIS) PetscCall(ISGetIndices(subcellIS, &subCells));
3051   for (c = 0; c < numSubCells; ++c) PetscCall(DMPlexSetConeSize(subdm, c, 1));
3052   for (f = firstSubFace; f < firstSubFace + numSubFaces; ++f) PetscCall(DMPlexSetConeSize(subdm, f, nFV));
3053   PetscCall(DMSetUp(subdm));
3054   /* Create face cones */
3055   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
3056   PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, NULL));
3057   PetscCall(DMGetWorkArray(subdm, maxConeSize, MPIU_INT, (void **)&subface));
3058   for (c = 0; c < numSubCells; ++c) {
3059     const PetscInt cell    = subCells[c];
3060     const PetscInt subcell = c;
3061     PetscInt      *closure = NULL;
3062     PetscInt       closureSize, cl, numCorners = 0, faceSize = 0;
3063 
3064     PetscCall(DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
3065     for (cl = 0; cl < closureSize * 2; cl += 2) {
3066       const PetscInt point = closure[cl];
3067       PetscInt       subVertex;
3068 
3069       if ((point >= vStart) && (point < vEnd)) {
3070         ++numCorners;
3071         PetscCall(PetscFindInt(point, numSubVertices, subVertices, &subVertex));
3072         if (subVertex >= 0) {
3073           closure[faceSize] = point;
3074           subface[faceSize] = firstSubVertex + subVertex;
3075           ++faceSize;
3076         }
3077       }
3078     }
3079     PetscCheck(faceSize <= nFV, comm, PETSC_ERR_ARG_WRONG, "Invalid submesh: Too many vertices %" PetscInt_FMT " of an element on the surface", faceSize);
3080     if (faceSize == nFV) PetscCall(DMPlexInsertFace_Internal(dm, subdm, faceSize, closure, subface, numCorners, cell, subcell, firstSubFace, &newFacePoint));
3081     PetscCall(DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure));
3082   }
3083   PetscCall(DMRestoreWorkArray(subdm, maxConeSize, MPIU_INT, (void **)&subface));
3084   PetscCall(DMPlexSymmetrize(subdm));
3085   PetscCall(DMPlexStratify(subdm));
3086   /* Build coordinates */
3087   {
3088     PetscSection coordSection, subCoordSection;
3089     Vec          coordinates, subCoordinates;
3090     PetscScalar *coords, *subCoords;
3091     PetscInt     numComp, coordSize, v;
3092     const char  *name;
3093 
3094     PetscCall(DMGetCoordinateSection(dm, &coordSection));
3095     PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
3096     PetscCall(DMGetCoordinateSection(subdm, &subCoordSection));
3097     PetscCall(PetscSectionSetNumFields(subCoordSection, 1));
3098     PetscCall(PetscSectionGetFieldComponents(coordSection, 0, &numComp));
3099     PetscCall(PetscSectionSetFieldComponents(subCoordSection, 0, numComp));
3100     PetscCall(PetscSectionSetChart(subCoordSection, firstSubVertex, firstSubVertex + numSubVertices));
3101     for (v = 0; v < numSubVertices; ++v) {
3102       const PetscInt vertex    = subVertices[v];
3103       const PetscInt subvertex = firstSubVertex + v;
3104       PetscInt       dof;
3105 
3106       PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3107       PetscCall(PetscSectionSetDof(subCoordSection, subvertex, dof));
3108       PetscCall(PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof));
3109     }
3110     PetscCall(PetscSectionSetUp(subCoordSection));
3111     PetscCall(PetscSectionGetStorageSize(subCoordSection, &coordSize));
3112     PetscCall(VecCreate(PETSC_COMM_SELF, &subCoordinates));
3113     PetscCall(PetscObjectGetName((PetscObject)coordinates, &name));
3114     PetscCall(PetscObjectSetName((PetscObject)subCoordinates, name));
3115     PetscCall(VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE));
3116     PetscCall(VecSetType(subCoordinates, VECSTANDARD));
3117     if (coordSize) {
3118       PetscCall(VecGetArray(coordinates, &coords));
3119       PetscCall(VecGetArray(subCoordinates, &subCoords));
3120       for (v = 0; v < numSubVertices; ++v) {
3121         const PetscInt vertex    = subVertices[v];
3122         const PetscInt subvertex = firstSubVertex + v;
3123         PetscInt       dof, off, sdof, soff, d;
3124 
3125         PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3126         PetscCall(PetscSectionGetOffset(coordSection, vertex, &off));
3127         PetscCall(PetscSectionGetDof(subCoordSection, subvertex, &sdof));
3128         PetscCall(PetscSectionGetOffset(subCoordSection, subvertex, &soff));
3129         PetscCheck(dof == sdof, comm, PETSC_ERR_PLIB, "Coordinate dimension %" PetscInt_FMT " on subvertex %" PetscInt_FMT ", vertex %" PetscInt_FMT " should be %" PetscInt_FMT, sdof, subvertex, vertex, dof);
3130         for (d = 0; d < dof; ++d) subCoords[soff + d] = coords[off + d];
3131       }
3132       PetscCall(VecRestoreArray(coordinates, &coords));
3133       PetscCall(VecRestoreArray(subCoordinates, &subCoords));
3134     }
3135     PetscCall(DMSetCoordinatesLocal(subdm, subCoordinates));
3136     PetscCall(VecDestroy(&subCoordinates));
3137   }
3138   /* Cleanup */
3139   if (subvertexIS) PetscCall(ISRestoreIndices(subvertexIS, &subVertices));
3140   PetscCall(ISDestroy(&subvertexIS));
3141   if (subcellIS) PetscCall(ISRestoreIndices(subcellIS, &subCells));
3142   PetscCall(ISDestroy(&subcellIS));
3143   PetscFunctionReturn(0);
3144 }
3145 
3146 /* TODO: Fix this to properly propagate up error conditions it may find */
3147 static inline PetscInt DMPlexFilterPoint_Internal(PetscInt point, PetscInt firstSubPoint, PetscInt numSubPoints, const PetscInt subPoints[])
3148 {
3149   PetscInt       subPoint;
3150   PetscErrorCode ierr;
3151 
3152   ierr = PetscFindInt(point, numSubPoints, subPoints, &subPoint);
3153   if (ierr) return -1;
3154   return subPoint < 0 ? subPoint : firstSubPoint + subPoint;
3155 }
3156 
3157 /* TODO: Fix this to properly propagate up error conditions it may find */
3158 static inline PetscInt DMPlexFilterPointPerm_Internal(PetscInt point, PetscInt firstSubPoint, PetscInt numSubPoints, const PetscInt subPoints[], const PetscInt subIndices[])
3159 {
3160   PetscInt       subPoint;
3161   PetscErrorCode ierr;
3162 
3163   ierr = PetscFindInt(point, numSubPoints, subPoints, &subPoint);
3164   if (ierr) return -1;
3165   return subPoint < 0 ? subPoint : firstSubPoint + (subIndices ? subIndices[subPoint] : subPoint);
3166 }
3167 
3168 static PetscErrorCode DMPlexFilterLabels_Internal(DM dm, const PetscInt numSubPoints[], const PetscInt *subpoints[], const PetscInt firstSubPoint[], DM subdm)
3169 {
3170   DMLabel  depthLabel;
3171   PetscInt Nl, l, d;
3172 
3173   PetscFunctionBegin;
3174   // Reset depth label for fast lookup
3175   PetscCall(DMPlexGetDepthLabel(dm, &depthLabel));
3176   PetscCall(DMLabelMakeAllInvalid_Internal(depthLabel));
3177   PetscCall(DMGetNumLabels(dm, &Nl));
3178   for (l = 0; l < Nl; ++l) {
3179     DMLabel         label, newlabel;
3180     const char     *lname;
3181     PetscBool       isDepth, isDim, isCelltype, isVTK;
3182     IS              valueIS;
3183     const PetscInt *values;
3184     PetscInt        Nv, v;
3185 
3186     PetscCall(DMGetLabelName(dm, l, &lname));
3187     PetscCall(PetscStrcmp(lname, "depth", &isDepth));
3188     PetscCall(PetscStrcmp(lname, "dim", &isDim));
3189     PetscCall(PetscStrcmp(lname, "celltype", &isCelltype));
3190     PetscCall(PetscStrcmp(lname, "vtk", &isVTK));
3191     if (isDepth || isDim || isCelltype || isVTK) continue;
3192     PetscCall(DMCreateLabel(subdm, lname));
3193     PetscCall(DMGetLabel(dm, lname, &label));
3194     PetscCall(DMGetLabel(subdm, lname, &newlabel));
3195     PetscCall(DMLabelGetDefaultValue(label, &v));
3196     PetscCall(DMLabelSetDefaultValue(newlabel, v));
3197     PetscCall(DMLabelGetValueIS(label, &valueIS));
3198     PetscCall(ISGetLocalSize(valueIS, &Nv));
3199     PetscCall(ISGetIndices(valueIS, &values));
3200     for (v = 0; v < Nv; ++v) {
3201       IS              pointIS;
3202       const PetscInt *points;
3203       PetscInt        Np, p;
3204 
3205       PetscCall(DMLabelGetStratumIS(label, values[v], &pointIS));
3206       PetscCall(ISGetLocalSize(pointIS, &Np));
3207       PetscCall(ISGetIndices(pointIS, &points));
3208       for (p = 0; p < Np; ++p) {
3209         const PetscInt point = points[p];
3210         PetscInt       subp;
3211 
3212         PetscCall(DMPlexGetPointDepth(dm, point, &d));
3213         subp = DMPlexFilterPoint_Internal(point, firstSubPoint[d], numSubPoints[d], subpoints[d]);
3214         if (subp >= 0) PetscCall(DMLabelSetValue(newlabel, subp, values[v]));
3215       }
3216       PetscCall(ISRestoreIndices(pointIS, &points));
3217       PetscCall(ISDestroy(&pointIS));
3218     }
3219     PetscCall(ISRestoreIndices(valueIS, &values));
3220     PetscCall(ISDestroy(&valueIS));
3221   }
3222   PetscFunctionReturn(0);
3223 }
3224 
3225 static PetscErrorCode DMPlexCreateSubmeshGeneric_Interpolated(DM dm, DMLabel label, PetscInt value, PetscBool markedFaces, PetscBool isCohesive, PetscInt cellHeight, DM subdm)
3226 {
3227   MPI_Comm         comm;
3228   DMLabel          subpointMap;
3229   IS              *subpointIS;
3230   const PetscInt **subpoints;
3231   PetscInt        *numSubPoints, *firstSubPoint, *coneNew, *orntNew;
3232   PetscInt         totSubPoints = 0, maxConeSize, dim, sdim, cdim, p, d, v;
3233   PetscMPIInt      rank;
3234 
3235   PetscFunctionBegin;
3236   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
3237   PetscCallMPI(MPI_Comm_rank(comm, &rank));
3238   /* Create subpointMap which marks the submesh */
3239   PetscCall(DMLabelCreate(PETSC_COMM_SELF, "subpoint_map", &subpointMap));
3240   PetscCall(DMPlexSetSubpointMap(subdm, subpointMap));
3241   if (cellHeight) {
3242     if (isCohesive) PetscCall(DMPlexMarkCohesiveSubmesh_Interpolated(dm, label, value, subpointMap, subdm));
3243     else PetscCall(DMPlexMarkSubmesh_Interpolated(dm, label, value, markedFaces, subpointMap, subdm));
3244   } else {
3245     DMLabel         depth;
3246     IS              pointIS;
3247     const PetscInt *points;
3248     PetscInt        numPoints = 0;
3249 
3250     PetscCall(DMPlexGetDepthLabel(dm, &depth));
3251     PetscCall(DMLabelGetStratumIS(label, value, &pointIS));
3252     if (pointIS) {
3253       PetscCall(ISGetIndices(pointIS, &points));
3254       PetscCall(ISGetLocalSize(pointIS, &numPoints));
3255     }
3256     for (p = 0; p < numPoints; ++p) {
3257       PetscInt *closure = NULL;
3258       PetscInt  closureSize, c, pdim;
3259 
3260       PetscCall(DMPlexGetTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closure));
3261       for (c = 0; c < closureSize * 2; c += 2) {
3262         PetscCall(DMLabelGetValue(depth, closure[c], &pdim));
3263         PetscCall(DMLabelSetValue(subpointMap, closure[c], pdim));
3264       }
3265       PetscCall(DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closure));
3266     }
3267     if (pointIS) PetscCall(ISRestoreIndices(pointIS, &points));
3268     PetscCall(ISDestroy(&pointIS));
3269   }
3270   /* Setup chart */
3271   PetscCall(DMGetDimension(dm, &dim));
3272   PetscCall(DMGetCoordinateDim(dm, &cdim));
3273   PetscCall(PetscMalloc4(dim + 1, &numSubPoints, dim + 1, &firstSubPoint, dim + 1, &subpointIS, dim + 1, &subpoints));
3274   for (d = 0; d <= dim; ++d) {
3275     PetscCall(DMLabelGetStratumSize(subpointMap, d, &numSubPoints[d]));
3276     totSubPoints += numSubPoints[d];
3277   }
3278   // Determine submesh dimension
3279   PetscCall(DMGetDimension(subdm, &sdim));
3280   if (sdim > 0) {
3281     // Calling function knows what dimension to use, and we include neighboring cells as well
3282     sdim = dim;
3283   } else {
3284     // We reset the subdimension based on what is being selected
3285     PetscInt lsdim;
3286     for (lsdim = dim; lsdim >= 0; --lsdim)
3287       if (numSubPoints[lsdim]) break;
3288     PetscCall(MPI_Allreduce(&lsdim, &sdim, 1, MPIU_INT, MPIU_MAX, comm));
3289     PetscCall(DMSetDimension(subdm, sdim));
3290     PetscCall(DMSetCoordinateDim(subdm, cdim));
3291   }
3292   PetscCall(DMPlexSetChart(subdm, 0, totSubPoints));
3293   PetscCall(DMPlexSetVTKCellHeight(subdm, cellHeight));
3294   /* Set cone sizes */
3295   firstSubPoint[sdim] = 0;
3296   firstSubPoint[0]    = firstSubPoint[sdim] + numSubPoints[sdim];
3297   if (sdim > 1) firstSubPoint[sdim - 1] = firstSubPoint[0] + numSubPoints[0];
3298   if (sdim > 2) firstSubPoint[sdim - 2] = firstSubPoint[sdim - 1] + numSubPoints[sdim - 1];
3299   for (d = 0; d <= sdim; ++d) {
3300     PetscCall(DMLabelGetStratumIS(subpointMap, d, &subpointIS[d]));
3301     if (subpointIS[d]) PetscCall(ISGetIndices(subpointIS[d], &subpoints[d]));
3302   }
3303   /* We do not want this label automatically computed, instead we compute it here */
3304   PetscCall(DMCreateLabel(subdm, "celltype"));
3305   for (d = 0; d <= sdim; ++d) {
3306     for (p = 0; p < numSubPoints[d]; ++p) {
3307       const PetscInt  point    = subpoints[d][p];
3308       const PetscInt  subpoint = firstSubPoint[d] + p;
3309       const PetscInt *cone;
3310       PetscInt        coneSize;
3311 
3312       PetscCall(DMPlexGetConeSize(dm, point, &coneSize));
3313       if (cellHeight && (d == sdim)) {
3314         PetscInt coneSizeNew, c, val;
3315 
3316         PetscCall(DMPlexGetCone(dm, point, &cone));
3317         for (c = 0, coneSizeNew = 0; c < coneSize; ++c) {
3318           PetscCall(DMLabelGetValue(subpointMap, cone[c], &val));
3319           if (val >= 0) coneSizeNew++;
3320         }
3321         PetscCall(DMPlexSetConeSize(subdm, subpoint, coneSizeNew));
3322         PetscCall(DMPlexSetCellType(subdm, subpoint, DM_POLYTOPE_FV_GHOST));
3323       } else {
3324         DMPolytopeType ct;
3325 
3326         PetscCall(DMPlexSetConeSize(subdm, subpoint, coneSize));
3327         PetscCall(DMPlexGetCellType(dm, point, &ct));
3328         PetscCall(DMPlexSetCellType(subdm, subpoint, ct));
3329       }
3330     }
3331   }
3332   PetscCall(DMLabelDestroy(&subpointMap));
3333   PetscCall(DMSetUp(subdm));
3334   /* Set cones */
3335   PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, NULL));
3336   PetscCall(PetscMalloc2(maxConeSize, &coneNew, maxConeSize, &orntNew));
3337   for (d = 0; d <= sdim; ++d) {
3338     for (p = 0; p < numSubPoints[d]; ++p) {
3339       const PetscInt  point    = subpoints[d][p];
3340       const PetscInt  subpoint = firstSubPoint[d] + p;
3341       const PetscInt *cone, *ornt;
3342       PetscInt        coneSize, subconeSize, coneSizeNew, c, subc, fornt = 0;
3343 
3344       if (d == sdim - 1) {
3345         const PetscInt *support, *cone, *ornt;
3346         PetscInt        supportSize, coneSize, s, subc;
3347 
3348         PetscCall(DMPlexGetSupport(dm, point, &support));
3349         PetscCall(DMPlexGetSupportSize(dm, point, &supportSize));
3350         for (s = 0; s < supportSize; ++s) {
3351           PetscBool isHybrid = PETSC_FALSE;
3352 
3353           PetscCall(DMPlexCellIsHybrid_Internal(dm, support[s], &isHybrid));
3354           if (!isHybrid) continue;
3355           PetscCall(PetscFindInt(support[s], numSubPoints[d + 1], subpoints[d + 1], &subc));
3356           if (subc >= 0) {
3357             const PetscInt ccell = subpoints[d + 1][subc];
3358 
3359             PetscCall(DMPlexGetCone(dm, ccell, &cone));
3360             PetscCall(DMPlexGetConeSize(dm, ccell, &coneSize));
3361             PetscCall(DMPlexGetConeOrientation(dm, ccell, &ornt));
3362             for (c = 0; c < coneSize; ++c) {
3363               if (cone[c] == point) {
3364                 fornt = ornt[c];
3365                 break;
3366               }
3367             }
3368             break;
3369           }
3370         }
3371       }
3372       PetscCall(DMPlexGetConeSize(dm, point, &coneSize));
3373       PetscCall(DMPlexGetConeSize(subdm, subpoint, &subconeSize));
3374       PetscCall(DMPlexGetCone(dm, point, &cone));
3375       PetscCall(DMPlexGetConeOrientation(dm, point, &ornt));
3376       for (c = 0, coneSizeNew = 0; c < coneSize; ++c) {
3377         PetscCall(PetscFindInt(cone[c], numSubPoints[d - 1], subpoints[d - 1], &subc));
3378         if (subc >= 0) {
3379           coneNew[coneSizeNew] = firstSubPoint[d - 1] + subc;
3380           orntNew[coneSizeNew] = ornt[c];
3381           ++coneSizeNew;
3382         }
3383       }
3384       PetscCheck(coneSizeNew == subconeSize, comm, PETSC_ERR_PLIB, "Number of cone points located %" PetscInt_FMT " does not match subcone size %" PetscInt_FMT, coneSizeNew, subconeSize);
3385       PetscCall(DMPlexSetCone(subdm, subpoint, coneNew));
3386       PetscCall(DMPlexSetConeOrientation(subdm, subpoint, orntNew));
3387       if (fornt < 0) PetscCall(DMPlexOrientPoint(subdm, subpoint, fornt));
3388     }
3389   }
3390   PetscCall(PetscFree2(coneNew, orntNew));
3391   PetscCall(DMPlexSymmetrize(subdm));
3392   PetscCall(DMPlexStratify(subdm));
3393   /* Build coordinates */
3394   {
3395     PetscSection coordSection, subCoordSection;
3396     Vec          coordinates, subCoordinates;
3397     PetscScalar *coords, *subCoords;
3398     PetscInt     cdim, numComp, coordSize;
3399     const char  *name;
3400 
3401     PetscCall(DMGetCoordinateDim(dm, &cdim));
3402     PetscCall(DMGetCoordinateSection(dm, &coordSection));
3403     PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
3404     PetscCall(DMGetCoordinateSection(subdm, &subCoordSection));
3405     PetscCall(PetscSectionSetNumFields(subCoordSection, 1));
3406     PetscCall(PetscSectionGetFieldComponents(coordSection, 0, &numComp));
3407     PetscCall(PetscSectionSetFieldComponents(subCoordSection, 0, numComp));
3408     PetscCall(PetscSectionSetChart(subCoordSection, firstSubPoint[0], firstSubPoint[0] + numSubPoints[0]));
3409     for (v = 0; v < numSubPoints[0]; ++v) {
3410       const PetscInt vertex    = subpoints[0][v];
3411       const PetscInt subvertex = firstSubPoint[0] + v;
3412       PetscInt       dof;
3413 
3414       PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3415       PetscCall(PetscSectionSetDof(subCoordSection, subvertex, dof));
3416       PetscCall(PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof));
3417     }
3418     PetscCall(PetscSectionSetUp(subCoordSection));
3419     PetscCall(PetscSectionGetStorageSize(subCoordSection, &coordSize));
3420     PetscCall(VecCreate(PETSC_COMM_SELF, &subCoordinates));
3421     PetscCall(PetscObjectGetName((PetscObject)coordinates, &name));
3422     PetscCall(PetscObjectSetName((PetscObject)subCoordinates, name));
3423     PetscCall(VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE));
3424     PetscCall(VecSetBlockSize(subCoordinates, cdim));
3425     PetscCall(VecSetType(subCoordinates, VECSTANDARD));
3426     PetscCall(VecGetArray(coordinates, &coords));
3427     PetscCall(VecGetArray(subCoordinates, &subCoords));
3428     for (v = 0; v < numSubPoints[0]; ++v) {
3429       const PetscInt vertex    = subpoints[0][v];
3430       const PetscInt subvertex = firstSubPoint[0] + v;
3431       PetscInt       dof, off, sdof, soff, d;
3432 
3433       PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3434       PetscCall(PetscSectionGetOffset(coordSection, vertex, &off));
3435       PetscCall(PetscSectionGetDof(subCoordSection, subvertex, &sdof));
3436       PetscCall(PetscSectionGetOffset(subCoordSection, subvertex, &soff));
3437       PetscCheck(dof == sdof, comm, PETSC_ERR_PLIB, "Coordinate dimension %" PetscInt_FMT " on subvertex %" PetscInt_FMT ", vertex %" PetscInt_FMT " should be %" PetscInt_FMT, sdof, subvertex, vertex, dof);
3438       for (d = 0; d < dof; ++d) subCoords[soff + d] = coords[off + d];
3439     }
3440     PetscCall(VecRestoreArray(coordinates, &coords));
3441     PetscCall(VecRestoreArray(subCoordinates, &subCoords));
3442     PetscCall(DMSetCoordinatesLocal(subdm, subCoordinates));
3443     PetscCall(VecDestroy(&subCoordinates));
3444   }
3445   /* Build SF: We need this complexity because subpoints might not be selected on the owning process */
3446   {
3447     PetscSF            sfPoint, sfPointSub;
3448     IS                 subpIS;
3449     const PetscSFNode *remotePoints;
3450     PetscSFNode       *sremotePoints = NULL, *newLocalPoints = NULL, *newOwners = NULL;
3451     const PetscInt    *localPoints, *subpoints, *rootdegree;
3452     PetscInt          *slocalPoints = NULL, *sortedPoints = NULL, *sortedIndices = NULL;
3453     PetscInt           numRoots, numLeaves, numSubpoints = 0, numSubroots, numSubleaves = 0, l, sl = 0, ll = 0, pStart, pEnd, p;
3454     PetscMPIInt        rank, size;
3455 
3456     PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
3457     PetscCallMPI(MPI_Comm_size(PetscObjectComm((PetscObject)dm), &size));
3458     PetscCall(DMGetPointSF(dm, &sfPoint));
3459     PetscCall(DMGetPointSF(subdm, &sfPointSub));
3460     PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
3461     PetscCall(DMPlexGetChart(subdm, NULL, &numSubroots));
3462     PetscCall(DMPlexGetSubpointIS(subdm, &subpIS));
3463     if (subpIS) {
3464       PetscBool sorted = PETSC_TRUE;
3465 
3466       PetscCall(ISGetIndices(subpIS, &subpoints));
3467       PetscCall(ISGetLocalSize(subpIS, &numSubpoints));
3468       for (p = 1; p < numSubpoints; ++p) sorted = sorted && (subpoints[p] >= subpoints[p - 1]) ? PETSC_TRUE : PETSC_FALSE;
3469       if (!sorted) {
3470         PetscCall(PetscMalloc2(numSubpoints, &sortedPoints, numSubpoints, &sortedIndices));
3471         for (p = 0; p < numSubpoints; ++p) sortedIndices[p] = p;
3472         PetscCall(PetscArraycpy(sortedPoints, subpoints, numSubpoints));
3473         PetscCall(PetscSortIntWithArray(numSubpoints, sortedPoints, sortedIndices));
3474       }
3475     }
3476     PetscCall(PetscSFGetGraph(sfPoint, &numRoots, &numLeaves, &localPoints, &remotePoints));
3477     if (numRoots >= 0) {
3478       PetscCall(PetscSFComputeDegreeBegin(sfPoint, &rootdegree));
3479       PetscCall(PetscSFComputeDegreeEnd(sfPoint, &rootdegree));
3480       PetscCall(PetscMalloc2(pEnd - pStart, &newLocalPoints, numRoots, &newOwners));
3481       for (p = 0; p < pEnd - pStart; ++p) {
3482         newLocalPoints[p].rank  = -2;
3483         newLocalPoints[p].index = -2;
3484       }
3485       /* Set subleaves */
3486       for (l = 0; l < numLeaves; ++l) {
3487         const PetscInt point    = localPoints[l];
3488         const PetscInt subpoint = DMPlexFilterPointPerm_Internal(point, 0, numSubpoints, sortedPoints ? sortedPoints : subpoints, sortedIndices);
3489 
3490         if (subpoint < 0) continue;
3491         newLocalPoints[point - pStart].rank  = rank;
3492         newLocalPoints[point - pStart].index = subpoint;
3493         ++numSubleaves;
3494       }
3495       /* Must put in owned subpoints */
3496       for (p = pStart; p < pEnd; ++p) {
3497         newOwners[p - pStart].rank  = -3;
3498         newOwners[p - pStart].index = -3;
3499       }
3500       for (p = 0; p < numSubpoints; ++p) {
3501         /* Hold on to currently owned points */
3502         if (rootdegree[subpoints[p] - pStart]) newOwners[subpoints[p] - pStart].rank = rank + size;
3503         else newOwners[subpoints[p] - pStart].rank = rank;
3504         newOwners[subpoints[p] - pStart].index = p;
3505       }
3506       PetscCall(PetscSFReduceBegin(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC));
3507       PetscCall(PetscSFReduceEnd(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC));
3508       for (p = pStart; p < pEnd; ++p)
3509         if (newOwners[p - pStart].rank >= size) newOwners[p - pStart].rank -= size;
3510       PetscCall(PetscSFBcastBegin(sfPoint, MPIU_2INT, newOwners, newLocalPoints, MPI_REPLACE));
3511       PetscCall(PetscSFBcastEnd(sfPoint, MPIU_2INT, newOwners, newLocalPoints, MPI_REPLACE));
3512       PetscCall(PetscMalloc1(numSubleaves, &slocalPoints));
3513       PetscCall(PetscMalloc1(numSubleaves, &sremotePoints));
3514       for (l = 0; l < numLeaves; ++l) {
3515         const PetscInt point    = localPoints[l];
3516         const PetscInt subpoint = DMPlexFilterPointPerm_Internal(point, 0, numSubpoints, sortedPoints ? sortedPoints : subpoints, sortedIndices);
3517 
3518         if (subpoint < 0) continue;
3519         if (newLocalPoints[point].rank == rank) {
3520           ++ll;
3521           continue;
3522         }
3523         slocalPoints[sl]        = subpoint;
3524         sremotePoints[sl].rank  = newLocalPoints[point].rank;
3525         sremotePoints[sl].index = newLocalPoints[point].index;
3526         PetscCheck(sremotePoints[sl].rank >= 0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote rank for local point %" PetscInt_FMT, point);
3527         PetscCheck(sremotePoints[sl].index >= 0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote subpoint for local point %" PetscInt_FMT, point);
3528         ++sl;
3529       }
3530       PetscCheck(sl + ll == numSubleaves, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatch in number of subleaves %" PetscInt_FMT " + %" PetscInt_FMT " != %" PetscInt_FMT, sl, ll, numSubleaves);
3531       PetscCall(PetscFree2(newLocalPoints, newOwners));
3532       PetscCall(PetscSFSetGraph(sfPointSub, numSubroots, sl, slocalPoints, PETSC_OWN_POINTER, sremotePoints, PETSC_OWN_POINTER));
3533     }
3534     if (subpIS) PetscCall(ISRestoreIndices(subpIS, &subpoints));
3535     PetscCall(PetscFree2(sortedPoints, sortedIndices));
3536   }
3537   /* Filter labels */
3538   PetscCall(DMPlexFilterLabels_Internal(dm, numSubPoints, subpoints, firstSubPoint, subdm));
3539   /* Cleanup */
3540   for (d = 0; d <= sdim; ++d) {
3541     if (subpointIS[d]) PetscCall(ISRestoreIndices(subpointIS[d], &subpoints[d]));
3542     PetscCall(ISDestroy(&subpointIS[d]));
3543   }
3544   PetscCall(PetscFree4(numSubPoints, firstSubPoint, subpointIS, subpoints));
3545   PetscFunctionReturn(0);
3546 }
3547 
3548 static PetscErrorCode DMPlexCreateSubmesh_Interpolated(DM dm, DMLabel vertexLabel, PetscInt value, PetscBool markedFaces, DM subdm)
3549 {
3550   PetscFunctionBegin;
3551   PetscCall(DMPlexCreateSubmeshGeneric_Interpolated(dm, vertexLabel, value, markedFaces, PETSC_FALSE, 1, subdm));
3552   PetscFunctionReturn(0);
3553 }
3554 
3555 /*@
3556   DMPlexCreateSubmesh - Extract a hypersurface from the mesh using vertices defined by a label
3557 
3558   Input Parameters:
3559 + dm           - The original mesh
3560 . vertexLabel  - The DMLabel marking points contained in the surface
3561 . value        - The label value to use
3562 - markedFaces  - PETSC_TRUE if surface faces are marked in addition to vertices, PETSC_FALSE if only vertices are marked
3563 
3564   Output Parameter:
3565 . subdm - The surface mesh
3566 
3567   Note: This function produces a DMLabel mapping original points in the submesh to their depth. This can be obtained using DMPlexGetSubpointMap().
3568 
3569   Level: developer
3570 
3571 .seealso: `DMPlexGetSubpointMap()`, `DMGetLabel()`, `DMLabelSetValue()`
3572 @*/
3573 PetscErrorCode DMPlexCreateSubmesh(DM dm, DMLabel vertexLabel, PetscInt value, PetscBool markedFaces, DM *subdm)
3574 {
3575   DMPlexInterpolatedFlag interpolated;
3576   PetscInt               dim, cdim;
3577 
3578   PetscFunctionBegin;
3579   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3580   PetscValidPointer(subdm, 5);
3581   PetscCall(DMGetDimension(dm, &dim));
3582   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), subdm));
3583   PetscCall(DMSetType(*subdm, DMPLEX));
3584   PetscCall(DMSetDimension(*subdm, dim - 1));
3585   PetscCall(DMGetCoordinateDim(dm, &cdim));
3586   PetscCall(DMSetCoordinateDim(*subdm, cdim));
3587   PetscCall(DMPlexIsInterpolated(dm, &interpolated));
3588   PetscCheck(interpolated != DMPLEX_INTERPOLATED_PARTIAL, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Not for partially interpolated meshes");
3589   if (interpolated) {
3590     PetscCall(DMPlexCreateSubmesh_Interpolated(dm, vertexLabel, value, markedFaces, *subdm));
3591   } else {
3592     PetscCall(DMPlexCreateSubmesh_Uninterpolated(dm, vertexLabel, value, *subdm));
3593   }
3594   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_TRUE, *subdm));
3595   PetscFunctionReturn(0);
3596 }
3597 
3598 static PetscErrorCode DMPlexCreateCohesiveSubmesh_Uninterpolated(DM dm, PetscBool hasLagrange, const char label[], PetscInt value, DM subdm)
3599 {
3600   MPI_Comm        comm;
3601   DMLabel         subpointMap;
3602   IS              subvertexIS;
3603   const PetscInt *subVertices;
3604   PetscInt        numSubVertices, firstSubVertex, numSubCells, *subCells = NULL;
3605   PetscInt       *subface, maxConeSize, numSubFaces, firstSubFace, newFacePoint, nFV;
3606   PetscInt        c, f;
3607 
3608   PetscFunctionBegin;
3609   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
3610   /* Create subpointMap which marks the submesh */
3611   PetscCall(DMLabelCreate(PETSC_COMM_SELF, "subpoint_map", &subpointMap));
3612   PetscCall(DMPlexSetSubpointMap(subdm, subpointMap));
3613   PetscCall(DMLabelDestroy(&subpointMap));
3614   PetscCall(DMPlexMarkCohesiveSubmesh_Uninterpolated(dm, hasLagrange, label, value, subpointMap, &numSubFaces, &nFV, &subCells, subdm));
3615   /* Setup chart */
3616   PetscCall(DMLabelGetStratumSize(subpointMap, 0, &numSubVertices));
3617   PetscCall(DMLabelGetStratumSize(subpointMap, 2, &numSubCells));
3618   PetscCall(DMPlexSetChart(subdm, 0, numSubCells + numSubFaces + numSubVertices));
3619   PetscCall(DMPlexSetVTKCellHeight(subdm, 1));
3620   /* Set cone sizes */
3621   firstSubVertex = numSubCells;
3622   firstSubFace   = numSubCells + numSubVertices;
3623   newFacePoint   = firstSubFace;
3624   PetscCall(DMLabelGetStratumIS(subpointMap, 0, &subvertexIS));
3625   if (subvertexIS) PetscCall(ISGetIndices(subvertexIS, &subVertices));
3626   for (c = 0; c < numSubCells; ++c) PetscCall(DMPlexSetConeSize(subdm, c, 1));
3627   for (f = firstSubFace; f < firstSubFace + numSubFaces; ++f) PetscCall(DMPlexSetConeSize(subdm, f, nFV));
3628   PetscCall(DMSetUp(subdm));
3629   /* Create face cones */
3630   PetscCall(DMPlexGetMaxSizes(dm, &maxConeSize, NULL));
3631   PetscCall(DMGetWorkArray(subdm, maxConeSize, MPIU_INT, (void **)&subface));
3632   for (c = 0; c < numSubCells; ++c) {
3633     const PetscInt  cell    = subCells[c];
3634     const PetscInt  subcell = c;
3635     const PetscInt *cone, *cells;
3636     PetscBool       isHybrid = PETSC_FALSE;
3637     PetscInt        numCells, subVertex, p, v;
3638 
3639     PetscCall(DMPlexCellIsHybrid_Internal(dm, cell, &isHybrid));
3640     if (!isHybrid) continue;
3641     PetscCall(DMPlexGetCone(dm, cell, &cone));
3642     for (v = 0; v < nFV; ++v) {
3643       PetscCall(PetscFindInt(cone[v], numSubVertices, subVertices, &subVertex));
3644       subface[v] = firstSubVertex + subVertex;
3645     }
3646     PetscCall(DMPlexSetCone(subdm, newFacePoint, subface));
3647     PetscCall(DMPlexSetCone(subdm, subcell, &newFacePoint));
3648     PetscCall(DMPlexGetJoin(dm, nFV, cone, &numCells, &cells));
3649     /* Not true in parallel
3650     PetscCheck(numCells == 2,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive cells should separate two cells"); */
3651     for (p = 0; p < numCells; ++p) {
3652       PetscInt  negsubcell;
3653       PetscBool isHybrid = PETSC_FALSE;
3654 
3655       PetscCall(DMPlexCellIsHybrid_Internal(dm, cells[p], &isHybrid));
3656       if (isHybrid) continue;
3657       /* I know this is a crap search */
3658       for (negsubcell = 0; negsubcell < numSubCells; ++negsubcell) {
3659         if (subCells[negsubcell] == cells[p]) break;
3660       }
3661       PetscCheck(negsubcell != numSubCells, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not find negative face neighbor for cohesive cell %" PetscInt_FMT, cell);
3662       PetscCall(DMPlexSetCone(subdm, negsubcell, &newFacePoint));
3663     }
3664     PetscCall(DMPlexRestoreJoin(dm, nFV, cone, &numCells, &cells));
3665     ++newFacePoint;
3666   }
3667   PetscCall(DMRestoreWorkArray(subdm, maxConeSize, MPIU_INT, (void **)&subface));
3668   PetscCall(DMPlexSymmetrize(subdm));
3669   PetscCall(DMPlexStratify(subdm));
3670   /* Build coordinates */
3671   {
3672     PetscSection coordSection, subCoordSection;
3673     Vec          coordinates, subCoordinates;
3674     PetscScalar *coords, *subCoords;
3675     PetscInt     cdim, numComp, coordSize, v;
3676     const char  *name;
3677 
3678     PetscCall(DMGetCoordinateDim(dm, &cdim));
3679     PetscCall(DMGetCoordinateSection(dm, &coordSection));
3680     PetscCall(DMGetCoordinatesLocal(dm, &coordinates));
3681     PetscCall(DMGetCoordinateSection(subdm, &subCoordSection));
3682     PetscCall(PetscSectionSetNumFields(subCoordSection, 1));
3683     PetscCall(PetscSectionGetFieldComponents(coordSection, 0, &numComp));
3684     PetscCall(PetscSectionSetFieldComponents(subCoordSection, 0, numComp));
3685     PetscCall(PetscSectionSetChart(subCoordSection, firstSubVertex, firstSubVertex + numSubVertices));
3686     for (v = 0; v < numSubVertices; ++v) {
3687       const PetscInt vertex    = subVertices[v];
3688       const PetscInt subvertex = firstSubVertex + v;
3689       PetscInt       dof;
3690 
3691       PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3692       PetscCall(PetscSectionSetDof(subCoordSection, subvertex, dof));
3693       PetscCall(PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof));
3694     }
3695     PetscCall(PetscSectionSetUp(subCoordSection));
3696     PetscCall(PetscSectionGetStorageSize(subCoordSection, &coordSize));
3697     PetscCall(VecCreate(PETSC_COMM_SELF, &subCoordinates));
3698     PetscCall(PetscObjectGetName((PetscObject)coordinates, &name));
3699     PetscCall(PetscObjectSetName((PetscObject)subCoordinates, name));
3700     PetscCall(VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE));
3701     PetscCall(VecSetBlockSize(subCoordinates, cdim));
3702     PetscCall(VecSetType(subCoordinates, VECSTANDARD));
3703     PetscCall(VecGetArray(coordinates, &coords));
3704     PetscCall(VecGetArray(subCoordinates, &subCoords));
3705     for (v = 0; v < numSubVertices; ++v) {
3706       const PetscInt vertex    = subVertices[v];
3707       const PetscInt subvertex = firstSubVertex + v;
3708       PetscInt       dof, off, sdof, soff, d;
3709 
3710       PetscCall(PetscSectionGetDof(coordSection, vertex, &dof));
3711       PetscCall(PetscSectionGetOffset(coordSection, vertex, &off));
3712       PetscCall(PetscSectionGetDof(subCoordSection, subvertex, &sdof));
3713       PetscCall(PetscSectionGetOffset(subCoordSection, subvertex, &soff));
3714       PetscCheck(dof == sdof, comm, PETSC_ERR_PLIB, "Coordinate dimension %" PetscInt_FMT " on subvertex %" PetscInt_FMT ", vertex %" PetscInt_FMT " should be %" PetscInt_FMT, sdof, subvertex, vertex, dof);
3715       for (d = 0; d < dof; ++d) subCoords[soff + d] = coords[off + d];
3716     }
3717     PetscCall(VecRestoreArray(coordinates, &coords));
3718     PetscCall(VecRestoreArray(subCoordinates, &subCoords));
3719     PetscCall(DMSetCoordinatesLocal(subdm, subCoordinates));
3720     PetscCall(VecDestroy(&subCoordinates));
3721   }
3722   /* Build SF */
3723   CHKMEMQ;
3724   {
3725     PetscSF            sfPoint, sfPointSub;
3726     const PetscSFNode *remotePoints;
3727     PetscSFNode       *sremotePoints, *newLocalPoints, *newOwners;
3728     const PetscInt    *localPoints;
3729     PetscInt          *slocalPoints;
3730     PetscInt           numRoots, numLeaves, numSubRoots = numSubCells + numSubFaces + numSubVertices, numSubLeaves = 0, l, sl, ll, pStart, pEnd, p, vStart, vEnd;
3731     PetscMPIInt        rank;
3732 
3733     PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank));
3734     PetscCall(DMGetPointSF(dm, &sfPoint));
3735     PetscCall(DMGetPointSF(subdm, &sfPointSub));
3736     PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
3737     PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
3738     PetscCall(PetscSFGetGraph(sfPoint, &numRoots, &numLeaves, &localPoints, &remotePoints));
3739     if (numRoots >= 0) {
3740       /* Only vertices should be shared */
3741       PetscCall(PetscMalloc2(pEnd - pStart, &newLocalPoints, numRoots, &newOwners));
3742       for (p = 0; p < pEnd - pStart; ++p) {
3743         newLocalPoints[p].rank  = -2;
3744         newLocalPoints[p].index = -2;
3745       }
3746       /* Set subleaves */
3747       for (l = 0; l < numLeaves; ++l) {
3748         const PetscInt point    = localPoints[l];
3749         const PetscInt subPoint = DMPlexFilterPoint_Internal(point, firstSubVertex, numSubVertices, subVertices);
3750 
3751         PetscCheck(!(point < vStart) || !(point >= vEnd), PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Should not be mapping anything but vertices, %" PetscInt_FMT, point);
3752         if (subPoint < 0) continue;
3753         newLocalPoints[point - pStart].rank  = rank;
3754         newLocalPoints[point - pStart].index = subPoint;
3755         ++numSubLeaves;
3756       }
3757       /* Must put in owned subpoints */
3758       for (p = pStart; p < pEnd; ++p) {
3759         const PetscInt subPoint = DMPlexFilterPoint_Internal(p, firstSubVertex, numSubVertices, subVertices);
3760 
3761         if (subPoint < 0) {
3762           newOwners[p - pStart].rank  = -3;
3763           newOwners[p - pStart].index = -3;
3764         } else {
3765           newOwners[p - pStart].rank  = rank;
3766           newOwners[p - pStart].index = subPoint;
3767         }
3768       }
3769       PetscCall(PetscSFReduceBegin(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC));
3770       PetscCall(PetscSFReduceEnd(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC));
3771       PetscCall(PetscSFBcastBegin(sfPoint, MPIU_2INT, newOwners, newLocalPoints, MPI_REPLACE));
3772       PetscCall(PetscSFBcastEnd(sfPoint, MPIU_2INT, newOwners, newLocalPoints, MPI_REPLACE));
3773       PetscCall(PetscMalloc1(numSubLeaves, &slocalPoints));
3774       PetscCall(PetscMalloc1(numSubLeaves, &sremotePoints));
3775       for (l = 0, sl = 0, ll = 0; l < numLeaves; ++l) {
3776         const PetscInt point    = localPoints[l];
3777         const PetscInt subPoint = DMPlexFilterPoint_Internal(point, firstSubVertex, numSubVertices, subVertices);
3778 
3779         if (subPoint < 0) continue;
3780         if (newLocalPoints[point].rank == rank) {
3781           ++ll;
3782           continue;
3783         }
3784         slocalPoints[sl]        = subPoint;
3785         sremotePoints[sl].rank  = newLocalPoints[point].rank;
3786         sremotePoints[sl].index = newLocalPoints[point].index;
3787         PetscCheck(sremotePoints[sl].rank >= 0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote rank for local point %" PetscInt_FMT, point);
3788         PetscCheck(sremotePoints[sl].index >= 0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote subpoint for local point %" PetscInt_FMT, point);
3789         ++sl;
3790       }
3791       PetscCall(PetscFree2(newLocalPoints, newOwners));
3792       PetscCheck(sl + ll == numSubLeaves, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatch in number of subleaves %" PetscInt_FMT " + %" PetscInt_FMT " != %" PetscInt_FMT, sl, ll, numSubLeaves);
3793       PetscCall(PetscSFSetGraph(sfPointSub, numSubRoots, sl, slocalPoints, PETSC_OWN_POINTER, sremotePoints, PETSC_OWN_POINTER));
3794     }
3795   }
3796   CHKMEMQ;
3797   /* Cleanup */
3798   if (subvertexIS) PetscCall(ISRestoreIndices(subvertexIS, &subVertices));
3799   PetscCall(ISDestroy(&subvertexIS));
3800   PetscCall(PetscFree(subCells));
3801   PetscFunctionReturn(0);
3802 }
3803 
3804 static PetscErrorCode DMPlexCreateCohesiveSubmesh_Interpolated(DM dm, const char labelname[], PetscInt value, DM subdm)
3805 {
3806   DMLabel label = NULL;
3807 
3808   PetscFunctionBegin;
3809   if (labelname) PetscCall(DMGetLabel(dm, labelname, &label));
3810   PetscCall(DMPlexCreateSubmeshGeneric_Interpolated(dm, label, value, PETSC_FALSE, PETSC_TRUE, 1, subdm));
3811   PetscFunctionReturn(0);
3812 }
3813 
3814 /*@C
3815   DMPlexCreateCohesiveSubmesh - Extract from a mesh with cohesive cells the hypersurface defined by one face of the cells. Optionally, a Label can be given to restrict the cells.
3816 
3817   Input Parameters:
3818 + dm          - The original mesh
3819 . hasLagrange - The mesh has Lagrange unknowns in the cohesive cells
3820 . label       - A label name, or NULL
3821 - value  - A label value
3822 
3823   Output Parameter:
3824 . subdm - The surface mesh
3825 
3826   Note: This function produces a DMLabel mapping original points in the submesh to their depth. This can be obtained using DMPlexGetSubpointMap().
3827 
3828   Level: developer
3829 
3830 .seealso: `DMPlexGetSubpointMap()`, `DMPlexCreateSubmesh()`
3831 @*/
3832 PetscErrorCode DMPlexCreateCohesiveSubmesh(DM dm, PetscBool hasLagrange, const char label[], PetscInt value, DM *subdm)
3833 {
3834   PetscInt dim, cdim, depth;
3835 
3836   PetscFunctionBegin;
3837   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3838   PetscValidPointer(subdm, 5);
3839   PetscCall(DMGetDimension(dm, &dim));
3840   PetscCall(DMPlexGetDepth(dm, &depth));
3841   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), subdm));
3842   PetscCall(DMSetType(*subdm, DMPLEX));
3843   PetscCall(DMSetDimension(*subdm, dim - 1));
3844   PetscCall(DMGetCoordinateDim(dm, &cdim));
3845   PetscCall(DMSetCoordinateDim(*subdm, cdim));
3846   if (depth == dim) {
3847     PetscCall(DMPlexCreateCohesiveSubmesh_Interpolated(dm, label, value, *subdm));
3848   } else {
3849     PetscCall(DMPlexCreateCohesiveSubmesh_Uninterpolated(dm, hasLagrange, label, value, *subdm));
3850   }
3851   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_TRUE, *subdm));
3852   PetscFunctionReturn(0);
3853 }
3854 
3855 /*@
3856   DMPlexFilter - Extract a subset of mesh cells defined by a label as a separate mesh
3857 
3858   Input Parameters:
3859 + dm        - The original mesh
3860 . cellLabel - The DMLabel marking cells contained in the new mesh
3861 - value     - The label value to use
3862 
3863   Output Parameter:
3864 . subdm - The new mesh
3865 
3866   Notes:
3867   This function produces a `DMLabel` mapping original points in the submesh to their depth. This can be obtained using `DMPlexGetSubpointMap()`.
3868 
3869   Level: developer
3870 
3871 .seealso: `DMPlexGetSubpointMap()`, `DMGetLabel()`, `DMLabelSetValue()`, `DMPlexCreateSubmesh()`
3872 @*/
3873 PetscErrorCode DMPlexFilter(DM dm, DMLabel cellLabel, PetscInt value, DM *subdm)
3874 {
3875   PetscInt dim, overlap;
3876 
3877   PetscFunctionBegin;
3878   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3879   PetscValidPointer(subdm, 4);
3880   PetscCall(DMGetDimension(dm, &dim));
3881   PetscCall(DMCreate(PetscObjectComm((PetscObject)dm), subdm));
3882   PetscCall(DMSetType(*subdm, DMPLEX));
3883   /* Extract submesh in place, could be empty on some procs, could have inconsistency if procs do not both extract a shared cell */
3884   PetscCall(DMPlexCreateSubmeshGeneric_Interpolated(dm, cellLabel, value, PETSC_FALSE, PETSC_FALSE, 0, *subdm));
3885   PetscCall(DMPlexCopy_Internal(dm, PETSC_TRUE, PETSC_TRUE, *subdm));
3886   // It is possible to obtain a surface mesh where some faces are in SF
3887   //   We should either mark the mesh as having an overlap, or delete these from the SF
3888   PetscCall(DMPlexGetOverlap(dm, &overlap));
3889   if (!overlap) {
3890     PetscSF         sf;
3891     const PetscInt *leaves;
3892     PetscInt        cStart, cEnd, Nl;
3893     PetscBool       hasSubcell = PETSC_FALSE, ghasSubcell;
3894 
3895     PetscCall(DMPlexGetHeightStratum(*subdm, 0, &cStart, &cEnd));
3896     PetscCall(DMGetPointSF(*subdm, &sf));
3897     PetscCall(PetscSFGetGraph(sf, NULL, &Nl, &leaves, NULL));
3898     for (PetscInt l = 0; l < Nl; ++l) {
3899       const PetscInt point = leaves ? leaves[l] : l;
3900 
3901       if (point >= cStart && point < cEnd) {
3902         hasSubcell = PETSC_TRUE;
3903         break;
3904       }
3905     }
3906     PetscCall(MPIU_Allreduce(&hasSubcell, &ghasSubcell, 1, MPIU_BOOL, MPI_LOR, PetscObjectComm((PetscObject)dm)));
3907     if (ghasSubcell) PetscCall(DMPlexSetOverlap(*subdm, NULL, 1));
3908   }
3909   PetscFunctionReturn(0);
3910 }
3911 
3912 /*@
3913   DMPlexGetSubpointMap - Returns a DMLabel with point dimension as values
3914 
3915   Input Parameter:
3916 . dm - The submesh DM
3917 
3918   Output Parameter:
3919 . subpointMap - The DMLabel of all the points from the original mesh in this submesh, or NULL if this is not a submesh
3920 
3921   Level: developer
3922 
3923 .seealso: `DMPlexCreateSubmesh()`, `DMPlexGetSubpointIS()`
3924 @*/
3925 PetscErrorCode DMPlexGetSubpointMap(DM dm, DMLabel *subpointMap)
3926 {
3927   PetscFunctionBegin;
3928   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3929   PetscValidPointer(subpointMap, 2);
3930   *subpointMap = ((DM_Plex *)dm->data)->subpointMap;
3931   PetscFunctionReturn(0);
3932 }
3933 
3934 /*@
3935   DMPlexSetSubpointMap - Sets the DMLabel with point dimension as values
3936 
3937   Input Parameters:
3938 + dm - The submesh DM
3939 - subpointMap - The DMLabel of all the points from the original mesh in this submesh
3940 
3941   Note: Should normally not be called by the user, since it is set in DMPlexCreateSubmesh()
3942 
3943   Level: developer
3944 
3945 .seealso: `DMPlexCreateSubmesh()`, `DMPlexGetSubpointIS()`
3946 @*/
3947 PetscErrorCode DMPlexSetSubpointMap(DM dm, DMLabel subpointMap)
3948 {
3949   DM_Plex *mesh = (DM_Plex *)dm->data;
3950   DMLabel  tmp;
3951 
3952   PetscFunctionBegin;
3953   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3954   tmp               = mesh->subpointMap;
3955   mesh->subpointMap = subpointMap;
3956   PetscCall(PetscObjectReference((PetscObject)mesh->subpointMap));
3957   PetscCall(DMLabelDestroy(&tmp));
3958   PetscFunctionReturn(0);
3959 }
3960 
3961 static PetscErrorCode DMPlexCreateSubpointIS_Internal(DM dm, IS *subpointIS)
3962 {
3963   DMLabel  spmap;
3964   PetscInt depth, d;
3965 
3966   PetscFunctionBegin;
3967   PetscCall(DMPlexGetSubpointMap(dm, &spmap));
3968   PetscCall(DMPlexGetDepth(dm, &depth));
3969   if (spmap && depth >= 0) {
3970     DM_Plex  *mesh = (DM_Plex *)dm->data;
3971     PetscInt *points, *depths;
3972     PetscInt  pStart, pEnd, p, off;
3973 
3974     PetscCall(DMPlexGetChart(dm, &pStart, &pEnd));
3975     PetscCheck(!pStart, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Submeshes must start the point numbering at 0, not %" PetscInt_FMT, pStart);
3976     PetscCall(PetscMalloc1(pEnd, &points));
3977     PetscCall(DMGetWorkArray(dm, depth + 1, MPIU_INT, &depths));
3978     depths[0] = depth;
3979     depths[1] = 0;
3980     for (d = 2; d <= depth; ++d) depths[d] = depth + 1 - d;
3981     for (d = 0, off = 0; d <= depth; ++d) {
3982       const PetscInt dep = depths[d];
3983       PetscInt       depStart, depEnd, n;
3984 
3985       PetscCall(DMPlexGetDepthStratum(dm, dep, &depStart, &depEnd));
3986       PetscCall(DMLabelGetStratumSize(spmap, dep, &n));
3987       if (((d < 2) && (depth > 1)) || (d == 1)) { /* Only check vertices and cells for now since the map is broken for others */
3988         PetscCheck(n == depEnd - depStart, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "The number of mapped submesh points %" PetscInt_FMT " at depth %" PetscInt_FMT " should be %" PetscInt_FMT, n, dep, depEnd - depStart);
3989       } else {
3990         if (!n) {
3991           if (d == 0) {
3992             /* Missing cells */
3993             for (p = 0; p < depEnd - depStart; ++p, ++off) points[off] = -1;
3994           } else {
3995             /* Missing faces */
3996             for (p = 0; p < depEnd - depStart; ++p, ++off) points[off] = PETSC_MAX_INT;
3997           }
3998         }
3999       }
4000       if (n) {
4001         IS              is;
4002         const PetscInt *opoints;
4003 
4004         PetscCall(DMLabelGetStratumIS(spmap, dep, &is));
4005         PetscCall(ISGetIndices(is, &opoints));
4006         for (p = 0; p < n; ++p, ++off) points[off] = opoints[p];
4007         PetscCall(ISRestoreIndices(is, &opoints));
4008         PetscCall(ISDestroy(&is));
4009       }
4010     }
4011     PetscCall(DMRestoreWorkArray(dm, depth + 1, MPIU_INT, &depths));
4012     PetscCheck(off == pEnd, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "The number of mapped submesh points %" PetscInt_FMT " should be %" PetscInt_FMT, off, pEnd);
4013     PetscCall(ISCreateGeneral(PETSC_COMM_SELF, pEnd, points, PETSC_OWN_POINTER, subpointIS));
4014     PetscCall(PetscObjectStateGet((PetscObject)spmap, &mesh->subpointState));
4015   }
4016   PetscFunctionReturn(0);
4017 }
4018 
4019 /*@
4020   DMPlexGetSubpointIS - Returns an IS covering the entire subdm chart with the original points as data
4021 
4022   Input Parameter:
4023 . dm - The submesh DM
4024 
4025   Output Parameter:
4026 . subpointIS - The IS of all the points from the original mesh in this submesh, or NULL if this is not a submesh
4027 
4028   Note: This IS is guaranteed to be sorted by the construction of the submesh
4029 
4030   Level: developer
4031 
4032 .seealso: `DMPlexCreateSubmesh()`, `DMPlexGetSubpointMap()`
4033 @*/
4034 PetscErrorCode DMPlexGetSubpointIS(DM dm, IS *subpointIS)
4035 {
4036   DM_Plex         *mesh = (DM_Plex *)dm->data;
4037   DMLabel          spmap;
4038   PetscObjectState state;
4039 
4040   PetscFunctionBegin;
4041   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
4042   PetscValidPointer(subpointIS, 2);
4043   PetscCall(DMPlexGetSubpointMap(dm, &spmap));
4044   PetscCall(PetscObjectStateGet((PetscObject)spmap, &state));
4045   if (state != mesh->subpointState || !mesh->subpointIS) PetscCall(DMPlexCreateSubpointIS_Internal(dm, &mesh->subpointIS));
4046   *subpointIS = mesh->subpointIS;
4047   PetscFunctionReturn(0);
4048 }
4049 
4050 /*@
4051   DMGetEnclosureRelation - Get the relationship between dmA and dmB
4052 
4053   Input Parameters:
4054 + dmA - The first DM
4055 - dmB - The second DM
4056 
4057   Output Parameter:
4058 . rel - The relation of dmA to dmB
4059 
4060   Level: intermediate
4061 
4062 .seealso: `DMGetEnclosurePoint()`
4063 @*/
4064 PetscErrorCode DMGetEnclosureRelation(DM dmA, DM dmB, DMEnclosureType *rel)
4065 {
4066   DM       plexA, plexB, sdm;
4067   DMLabel  spmap;
4068   PetscInt pStartA, pEndA, pStartB, pEndB, NpA, NpB;
4069 
4070   PetscFunctionBegin;
4071   PetscValidPointer(rel, 3);
4072   *rel = DM_ENC_NONE;
4073   if (!dmA || !dmB) PetscFunctionReturn(0);
4074   PetscValidHeaderSpecific(dmA, DM_CLASSID, 1);
4075   PetscValidHeaderSpecific(dmB, DM_CLASSID, 2);
4076   if (dmA == dmB) {
4077     *rel = DM_ENC_EQUALITY;
4078     PetscFunctionReturn(0);
4079   }
4080   PetscCall(DMConvert(dmA, DMPLEX, &plexA));
4081   PetscCall(DMConvert(dmB, DMPLEX, &plexB));
4082   PetscCall(DMPlexGetChart(plexA, &pStartA, &pEndA));
4083   PetscCall(DMPlexGetChart(plexB, &pStartB, &pEndB));
4084   /* Assumption 1: subDMs have smaller charts than the DMs that they originate from
4085     - The degenerate case of a subdomain which includes all of the domain on some process can be treated as equality */
4086   if ((pStartA == pStartB) && (pEndA == pEndB)) {
4087     *rel = DM_ENC_EQUALITY;
4088     goto end;
4089   }
4090   NpA = pEndA - pStartA;
4091   NpB = pEndB - pStartB;
4092   if (NpA == NpB) goto end;
4093   sdm = NpA > NpB ? plexB : plexA; /* The other is the original, enclosing dm */
4094   PetscCall(DMPlexGetSubpointMap(sdm, &spmap));
4095   if (!spmap) goto end;
4096   /* TODO Check the space mapped to by subpointMap is same size as dm */
4097   if (NpA > NpB) {
4098     *rel = DM_ENC_SUPERMESH;
4099   } else {
4100     *rel = DM_ENC_SUBMESH;
4101   }
4102 end:
4103   PetscCall(DMDestroy(&plexA));
4104   PetscCall(DMDestroy(&plexB));
4105   PetscFunctionReturn(0);
4106 }
4107 
4108 /*@
4109   DMGetEnclosurePoint - Get the point pA in dmA which corresponds to the point pB in dmB
4110 
4111   Input Parameters:
4112 + dmA   - The first DM
4113 . dmB   - The second DM
4114 . etype - The type of enclosure relation that dmA has to dmB
4115 - pB    - A point of dmB
4116 
4117   Output Parameter:
4118 . pA    - The corresponding point of dmA
4119 
4120   Level: intermediate
4121 
4122 .seealso: `DMGetEnclosureRelation()`
4123 @*/
4124 PetscErrorCode DMGetEnclosurePoint(DM dmA, DM dmB, DMEnclosureType etype, PetscInt pB, PetscInt *pA)
4125 {
4126   DM              sdm;
4127   IS              subpointIS;
4128   const PetscInt *subpoints;
4129   PetscInt        numSubpoints;
4130 
4131   PetscFunctionBegin;
4132   /* TODO Cache the IS, making it look like an index */
4133   switch (etype) {
4134   case DM_ENC_SUPERMESH:
4135     sdm = dmB;
4136     PetscCall(DMPlexGetSubpointIS(sdm, &subpointIS));
4137     PetscCall(ISGetIndices(subpointIS, &subpoints));
4138     *pA = subpoints[pB];
4139     PetscCall(ISRestoreIndices(subpointIS, &subpoints));
4140     break;
4141   case DM_ENC_SUBMESH:
4142     sdm = dmA;
4143     PetscCall(DMPlexGetSubpointIS(sdm, &subpointIS));
4144     PetscCall(ISGetLocalSize(subpointIS, &numSubpoints));
4145     PetscCall(ISGetIndices(subpointIS, &subpoints));
4146     PetscCall(PetscFindInt(pB, numSubpoints, subpoints, pA));
4147     if (*pA < 0) {
4148       PetscCall(DMViewFromOptions(dmA, NULL, "-dm_enc_A_view"));
4149       PetscCall(DMViewFromOptions(dmB, NULL, "-dm_enc_B_view"));
4150       SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Point %" PetscInt_FMT " not found in submesh", pB);
4151     }
4152     PetscCall(ISRestoreIndices(subpointIS, &subpoints));
4153     break;
4154   case DM_ENC_EQUALITY:
4155   case DM_ENC_NONE:
4156     *pA = pB;
4157     break;
4158   case DM_ENC_UNKNOWN: {
4159     DMEnclosureType enc;
4160 
4161     PetscCall(DMGetEnclosureRelation(dmA, dmB, &enc));
4162     PetscCall(DMGetEnclosurePoint(dmA, dmB, enc, pB, pA));
4163   } break;
4164   default:
4165     SETERRQ(PetscObjectComm((PetscObject)dmA), PETSC_ERR_ARG_OUTOFRANGE, "Invalid enclosure type %d", (int)etype);
4166   }
4167   PetscFunctionReturn(0);
4168 }
4169