xref: /petsc/src/dm/impls/plex/plexsubmesh.c (revision 030f984af8d8bb4c203755d35bded3c05b3d83ce)
1 #include <petsc/private/dmpleximpl.h>    /*I      "petscdmplex.h"    I*/
2 #include <petsc/private/dmlabelimpl.h>   /*I      "petscdmlabel.h"   I*/
3 #include <petscsf.h>
4 
5 static PetscErrorCode DMPlexCellIsHybrid_Internal(DM dm, PetscInt p, PetscBool *isHybrid)
6 {
7   DMPolytopeType ct;
8   PetscErrorCode ierr;
9 
10   PetscFunctionBegin;
11   ierr = DMPlexGetCellType(dm, p, &ct);CHKERRQ(ierr);
12   switch (ct) {
13     case DM_POLYTOPE_POINT_PRISM_TENSOR:
14     case DM_POLYTOPE_SEG_PRISM_TENSOR:
15     case DM_POLYTOPE_TRI_PRISM_TENSOR:
16     case DM_POLYTOPE_QUAD_PRISM_TENSOR:
17       *isHybrid = PETSC_TRUE;
18     default: *isHybrid = PETSC_FALSE;
19   }
20   PetscFunctionReturn(0);
21 }
22 
23 static PetscErrorCode DMPlexGetTensorPrismBounds_Internal(DM dm, PetscInt dim, PetscInt *cStart, PetscInt *cEnd)
24 {
25   DMLabel        ctLabel;
26   PetscErrorCode ierr;
27 
28   PetscFunctionBegin;
29   if (cStart) *cStart = -1;
30   if (cEnd)   *cEnd   = -1;
31   ierr = DMPlexGetCellTypeLabel(dm, &ctLabel);CHKERRQ(ierr);
32   switch (dim) {
33     case 1: ierr = DMLabelGetStratumBounds(ctLabel, DM_POLYTOPE_POINT_PRISM_TENSOR, cStart, cEnd);CHKERRQ(ierr);break;
34     case 2: ierr = DMLabelGetStratumBounds(ctLabel, DM_POLYTOPE_SEG_PRISM_TENSOR, cStart, cEnd);CHKERRQ(ierr);break;
35     case 3:
36       ierr = DMLabelGetStratumBounds(ctLabel, DM_POLYTOPE_TRI_PRISM_TENSOR, cStart, cEnd);CHKERRQ(ierr);
37       if (*cStart < 0) {ierr = DMLabelGetStratumBounds(ctLabel, DM_POLYTOPE_QUAD_PRISM_TENSOR, cStart, cEnd);CHKERRQ(ierr);}
38       break;
39     default: PetscFunctionReturn(0);
40   }
41   PetscFunctionReturn(0);
42 }
43 
44 static PetscErrorCode DMPlexMarkBoundaryFaces_Internal(DM dm, PetscInt val, PetscInt cellHeight, DMLabel label)
45 {
46   PetscInt       fStart, fEnd, f;
47   PetscErrorCode ierr;
48 
49   PetscFunctionBegin;
50   ierr = DMPlexGetHeightStratum(dm, cellHeight+1, &fStart, &fEnd);CHKERRQ(ierr);
51   for (f = fStart; f < fEnd; ++f) {
52     PetscInt supportSize;
53 
54     ierr = DMPlexGetSupportSize(dm, f, &supportSize);CHKERRQ(ierr);
55     if (supportSize == 1) {
56       if (val < 0) {
57         PetscInt *closure = NULL;
58         PetscInt  clSize, cl, cval;
59 
60         ierr = DMPlexGetTransitiveClosure(dm, f, PETSC_TRUE, &clSize, &closure);CHKERRQ(ierr);
61         for (cl = 0; cl < clSize*2; cl += 2) {
62           ierr = DMLabelGetValue(label, closure[cl], &cval);CHKERRQ(ierr);
63           if (cval < 0) continue;
64           ierr = DMLabelSetValue(label, f, cval);CHKERRQ(ierr);
65           break;
66         }
67         if (cl == clSize*2) {ierr = DMLabelSetValue(label, f, 1);CHKERRQ(ierr);}
68         ierr = DMPlexRestoreTransitiveClosure(dm, f, PETSC_TRUE, &clSize, &closure);CHKERRQ(ierr);
69       } else {
70         ierr = DMLabelSetValue(label, f, val);CHKERRQ(ierr);
71       }
72     }
73   }
74   PetscFunctionReturn(0);
75 }
76 
77 /*@
78   DMPlexMarkBoundaryFaces - Mark all faces on the boundary
79 
80   Not Collective
81 
82   Input Parameter:
83 + dm - The original DM
84 - val - The marker value, or PETSC_DETERMINE to use some value in the closure (or 1 if none are found)
85 
86   Output Parameter:
87 . label - The DMLabel marking boundary faces with the given value
88 
89   Level: developer
90 
91 .seealso: DMLabelCreate(), DMCreateLabel()
92 @*/
93 PetscErrorCode DMPlexMarkBoundaryFaces(DM dm, PetscInt val, DMLabel label)
94 {
95   DMPlexInterpolatedFlag  flg;
96   PetscErrorCode          ierr;
97 
98   PetscFunctionBegin;
99   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
100   ierr = DMPlexIsInterpolated(dm, &flg);CHKERRQ(ierr);
101   if (flg != DMPLEX_INTERPOLATED_FULL) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM is not fully interpolated on this rank");
102   ierr = DMPlexMarkBoundaryFaces_Internal(dm, val, 0, label);CHKERRQ(ierr);
103   PetscFunctionReturn(0);
104 }
105 
106 static PetscErrorCode DMPlexLabelComplete_Internal(DM dm, DMLabel label, PetscBool completeCells)
107 {
108   IS              valueIS;
109   PetscSF         sfPoint;
110   const PetscInt *values;
111   PetscInt        numValues, v, cStart, cEnd, nroots;
112   PetscErrorCode  ierr;
113 
114   PetscFunctionBegin;
115   ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
116   ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
117   ierr = DMPlexGetHeightStratum(dm,0,&cStart,&cEnd);CHKERRQ(ierr);
118   ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
119   for (v = 0; v < numValues; ++v) {
120     IS              pointIS;
121     const PetscInt *points;
122     PetscInt        numPoints, p;
123 
124     ierr = DMLabelGetStratumSize(label, values[v], &numPoints);CHKERRQ(ierr);
125     ierr = DMLabelGetStratumIS(label, values[v], &pointIS);CHKERRQ(ierr);
126     ierr = ISGetIndices(pointIS, &points);CHKERRQ(ierr);
127     for (p = 0; p < numPoints; ++p) {
128       PetscInt  q = points[p];
129       PetscInt *closure = NULL;
130       PetscInt  closureSize, c;
131 
132       if (cStart <= q && q < cEnd && !completeCells) { /* skip cells */
133         continue;
134       }
135       ierr = DMPlexGetTransitiveClosure(dm, q, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
136       for (c = 0; c < closureSize*2; c += 2) {
137         ierr = DMLabelSetValue(label, closure[c], values[v]);CHKERRQ(ierr);
138       }
139       ierr = DMPlexRestoreTransitiveClosure(dm, q, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
140     }
141     ierr = ISRestoreIndices(pointIS, &points);CHKERRQ(ierr);
142     ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
143   }
144   ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
145   ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
146   ierr = DMGetPointSF(dm, &sfPoint);CHKERRQ(ierr);
147   ierr = PetscSFGetGraph(sfPoint, &nroots, NULL, NULL, NULL);CHKERRQ(ierr);
148   if (nroots >= 0) {
149     DMLabel         lblRoots, lblLeaves;
150     IS              valueIS, pointIS;
151     const PetscInt *values;
152     PetscInt        numValues, v;
153     PetscErrorCode  ierr;
154 
155     /* Pull point contributions from remote leaves into local roots */
156     ierr = DMLabelGather(label, sfPoint, &lblLeaves);CHKERRQ(ierr);
157     ierr = DMLabelGetValueIS(lblLeaves, &valueIS);CHKERRQ(ierr);
158     ierr = ISGetLocalSize(valueIS, &numValues);CHKERRQ(ierr);
159     ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
160     for (v = 0; v < numValues; ++v) {
161       const PetscInt value = values[v];
162 
163       ierr = DMLabelGetStratumIS(lblLeaves, value, &pointIS);CHKERRQ(ierr);
164       ierr = DMLabelInsertIS(label, pointIS, value);CHKERRQ(ierr);
165       ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
166     }
167     ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
168     ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
169     ierr = DMLabelDestroy(&lblLeaves);CHKERRQ(ierr);
170     /* Push point contributions from roots into remote leaves */
171     ierr = DMLabelDistribute(label, sfPoint, &lblRoots);CHKERRQ(ierr);
172     ierr = DMLabelGetValueIS(lblRoots, &valueIS);CHKERRQ(ierr);
173     ierr = ISGetLocalSize(valueIS, &numValues);CHKERRQ(ierr);
174     ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
175     for (v = 0; v < numValues; ++v) {
176       const PetscInt value = values[v];
177 
178       ierr = DMLabelGetStratumIS(lblRoots, value, &pointIS);CHKERRQ(ierr);
179       ierr = DMLabelInsertIS(label, pointIS, value);CHKERRQ(ierr);
180       ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
181     }
182     ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
183     ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
184     ierr = DMLabelDestroy(&lblRoots);CHKERRQ(ierr);
185   }
186   PetscFunctionReturn(0);
187 }
188 
189 /*@
190   DMPlexLabelComplete - Starting with a label marking points on a surface, we add the transitive closure to the surface
191 
192   Input Parameters:
193 + dm - The DM
194 - label - A DMLabel marking the surface points
195 
196   Output Parameter:
197 . label - A DMLabel marking all surface points in the transitive closure
198 
199   Level: developer
200 
201 .seealso: DMPlexLabelCohesiveComplete()
202 @*/
203 PetscErrorCode DMPlexLabelComplete(DM dm, DMLabel label)
204 {
205   PetscErrorCode ierr;
206 
207   PetscFunctionBegin;
208   ierr = DMPlexLabelComplete_Internal(dm, label, PETSC_TRUE);CHKERRQ(ierr);
209   PetscFunctionReturn(0);
210 }
211 
212 /*@
213   DMPlexLabelAddCells - Starting with a label marking points on a surface, we add a cell for each point
214 
215   Input Parameters:
216 + dm - The DM
217 - label - A DMLabel marking the surface points
218 
219   Output Parameter:
220 . label - A DMLabel incorporating cells
221 
222   Level: developer
223 
224   Note: The cells allow FEM boundary conditions to be applied using the cell geometry
225 
226 .seealso: DMPlexLabelAddFaceCells(), DMPlexLabelComplete(), DMPlexLabelCohesiveComplete()
227 @*/
228 PetscErrorCode DMPlexLabelAddCells(DM dm, DMLabel label)
229 {
230   IS              valueIS;
231   const PetscInt *values;
232   PetscInt        numValues, v, cStart, cEnd;
233   PetscErrorCode  ierr;
234 
235   PetscFunctionBegin;
236   ierr = DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
237   ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
238   ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
239   ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
240   for (v = 0; v < numValues; ++v) {
241     IS              pointIS;
242     const PetscInt *points;
243     PetscInt        numPoints, p;
244 
245     ierr = DMLabelGetStratumSize(label, values[v], &numPoints);CHKERRQ(ierr);
246     ierr = DMLabelGetStratumIS(label, values[v], &pointIS);CHKERRQ(ierr);
247     ierr = ISGetIndices(pointIS, &points);CHKERRQ(ierr);
248     for (p = 0; p < numPoints; ++p) {
249       PetscInt *closure = NULL;
250       PetscInt  closureSize, cl;
251 
252       ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_FALSE, &closureSize, &closure);CHKERRQ(ierr);
253       for (cl = closureSize-1; cl > 0; --cl) {
254         const PetscInt cell = closure[cl*2];
255         if ((cell >= cStart) && (cell < cEnd)) {ierr = DMLabelSetValue(label, cell, values[v]);CHKERRQ(ierr); break;}
256       }
257       ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_FALSE, &closureSize, &closure);CHKERRQ(ierr);
258     }
259     ierr = ISRestoreIndices(pointIS, &points);CHKERRQ(ierr);
260     ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
261   }
262   ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
263   ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
264   PetscFunctionReturn(0);
265 }
266 
267 /*@
268   DMPlexLabelAddFaceCells - Starting with a label marking faces on a surface, we add a cell for each face
269 
270   Input Parameters:
271 + dm - The DM
272 - label - A DMLabel marking the surface points
273 
274   Output Parameter:
275 . label - A DMLabel incorporating cells
276 
277   Level: developer
278 
279   Note: The cells allow FEM boundary conditions to be applied using the cell geometry
280 
281 .seealso: DMPlexLabelAddCells(), DMPlexLabelComplete(), DMPlexLabelCohesiveComplete()
282 @*/
283 PetscErrorCode DMPlexLabelAddFaceCells(DM dm, DMLabel label)
284 {
285   IS              valueIS;
286   const PetscInt *values;
287   PetscInt        numValues, v, cStart, cEnd, fStart, fEnd;
288   PetscErrorCode  ierr;
289 
290   PetscFunctionBegin;
291   ierr = DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
292   ierr = DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd);CHKERRQ(ierr);
293   ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
294   ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
295   ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
296   for (v = 0; v < numValues; ++v) {
297     IS              pointIS;
298     const PetscInt *points;
299     PetscInt        numPoints, p;
300 
301     ierr = DMLabelGetStratumSize(label, values[v], &numPoints);CHKERRQ(ierr);
302     ierr = DMLabelGetStratumIS(label, values[v], &pointIS);CHKERRQ(ierr);
303     ierr = ISGetIndices(pointIS, &points);CHKERRQ(ierr);
304     for (p = 0; p < numPoints; ++p) {
305       const PetscInt face = points[p];
306       PetscInt      *closure = NULL;
307       PetscInt       closureSize, cl;
308 
309       if ((face < fStart) || (face >= fEnd)) continue;
310       ierr = DMPlexGetTransitiveClosure(dm, face, PETSC_FALSE, &closureSize, &closure);CHKERRQ(ierr);
311       for (cl = closureSize-1; cl > 0; --cl) {
312         const PetscInt cell = closure[cl*2];
313         if ((cell >= cStart) && (cell < cEnd)) {ierr = DMLabelSetValue(label, cell, values[v]);CHKERRQ(ierr); break;}
314       }
315       ierr = DMPlexRestoreTransitiveClosure(dm, face, PETSC_FALSE, &closureSize, &closure);CHKERRQ(ierr);
316     }
317     ierr = ISRestoreIndices(pointIS, &points);CHKERRQ(ierr);
318     ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
319   }
320   ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
321   ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
322   PetscFunctionReturn(0);
323 }
324 
325 /*@
326   DMPlexLabelClearCells - Remove cells from a label
327 
328   Input Parameters:
329 + dm - The DM
330 - label - A DMLabel marking surface points and their adjacent cells
331 
332   Output Parameter:
333 . label - A DMLabel without cells
334 
335   Level: developer
336 
337   Note: This undoes DMPlexLabelAddCells() or DMPlexLabelAddFaceCells()
338 
339 .seealso: DMPlexLabelComplete(), DMPlexLabelCohesiveComplete(), DMPlexLabelAddCells()
340 @*/
341 PetscErrorCode DMPlexLabelClearCells(DM dm, DMLabel label)
342 {
343   IS              valueIS;
344   const PetscInt *values;
345   PetscInt        numValues, v, cStart, cEnd;
346   PetscErrorCode  ierr;
347 
348   PetscFunctionBegin;
349   ierr = DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
350   ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
351   ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
352   ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
353   for (v = 0; v < numValues; ++v) {
354     IS              pointIS;
355     const PetscInt *points;
356     PetscInt        numPoints, p;
357 
358     ierr = DMLabelGetStratumSize(label, values[v], &numPoints);CHKERRQ(ierr);
359     ierr = DMLabelGetStratumIS(label, values[v], &pointIS);CHKERRQ(ierr);
360     ierr = ISGetIndices(pointIS, &points);CHKERRQ(ierr);
361     for (p = 0; p < numPoints; ++p) {
362       PetscInt point = points[p];
363 
364       if (point >= cStart && point < cEnd) {
365         ierr = DMLabelClearValue(label,point,values[v]);CHKERRQ(ierr);
366       }
367     }
368     ierr = ISRestoreIndices(pointIS, &points);CHKERRQ(ierr);
369     ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
370   }
371   ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
372   ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
373   PetscFunctionReturn(0);
374 }
375 
376 /* take (oldEnd, added) pairs, ordered by height and convert them to (oldstart, newstart) pairs, ordered by ascending
377  * index (skipping first, which is (0,0)) */
378 PETSC_STATIC_INLINE PetscErrorCode DMPlexShiftPointSetUp_Internal(PetscInt depth, PetscInt depthShift[])
379 {
380   PetscInt d, off = 0;
381 
382   PetscFunctionBegin;
383   /* sort by (oldend): yes this is an O(n^2) sort, we expect depth <= 3 */
384   for (d = 0; d < depth; d++) {
385     PetscInt firstd = d;
386     PetscInt firstStart = depthShift[2*d];
387     PetscInt e;
388 
389     for (e = d+1; e <= depth; e++) {
390       if (depthShift[2*e] < firstStart) {
391         firstd = e;
392         firstStart = depthShift[2*d];
393       }
394     }
395     if (firstd != d) {
396       PetscInt swap[2];
397 
398       e = firstd;
399       swap[0] = depthShift[2*d];
400       swap[1] = depthShift[2*d+1];
401       depthShift[2*d]   = depthShift[2*e];
402       depthShift[2*d+1] = depthShift[2*e+1];
403       depthShift[2*e]   = swap[0];
404       depthShift[2*e+1] = swap[1];
405     }
406   }
407   /* convert (oldstart, added) to (oldstart, newstart) */
408   for (d = 0; d <= depth; d++) {
409     off += depthShift[2*d+1];
410     depthShift[2*d+1] = depthShift[2*d] + off;
411   }
412   PetscFunctionReturn(0);
413 }
414 
415 /* depthShift is a list of (old, new) pairs */
416 PETSC_STATIC_INLINE PetscInt DMPlexShiftPoint_Internal(PetscInt p, PetscInt depth, PetscInt depthShift[])
417 {
418   PetscInt d;
419   PetscInt newOff = 0;
420 
421   for (d = 0; d <= depth; d++) {
422     if (p < depthShift[2*d]) return p + newOff;
423     else newOff = depthShift[2*d+1] - depthShift[2*d];
424   }
425   return p + newOff;
426 }
427 
428 /* depthShift is a list of (old, new) pairs */
429 PETSC_STATIC_INLINE PetscInt DMPlexShiftPointInverse_Internal(PetscInt p, PetscInt depth, PetscInt depthShift[])
430 {
431   PetscInt d;
432   PetscInt newOff = 0;
433 
434   for (d = 0; d <= depth; d++) {
435     if (p < depthShift[2*d+1]) return p + newOff;
436     else newOff = depthShift[2*d] - depthShift[2*d+1];
437   }
438   return p + newOff;
439 }
440 
441 static PetscErrorCode DMPlexShiftSizes_Internal(DM dm, PetscInt depthShift[], DM dmNew)
442 {
443   PetscInt       depth = 0, d, pStart, pEnd, p;
444   DMLabel        depthLabel;
445   PetscErrorCode ierr;
446 
447   PetscFunctionBegin;
448   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
449   if (depth < 0) PetscFunctionReturn(0);
450   /* Step 1: Expand chart */
451   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
452   pEnd = DMPlexShiftPoint_Internal(pEnd,depth,depthShift);
453   ierr = DMPlexSetChart(dmNew, pStart, pEnd);CHKERRQ(ierr);
454   ierr = DMCreateLabel(dmNew,"depth");CHKERRQ(ierr);
455   ierr = DMPlexGetDepthLabel(dmNew,&depthLabel);CHKERRQ(ierr);
456   ierr = DMCreateLabel(dmNew, "celltype");CHKERRQ(ierr);
457   /* Step 2: Set cone and support sizes */
458   for (d = 0; d <= depth; ++d) {
459     PetscInt pStartNew, pEndNew;
460     IS pIS;
461 
462     ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
463     pStartNew = DMPlexShiftPoint_Internal(pStart, depth, depthShift);
464     pEndNew = DMPlexShiftPoint_Internal(pEnd, depth, depthShift);
465     ierr = ISCreateStride(PETSC_COMM_SELF, pEndNew - pStartNew, pStartNew, 1, &pIS);CHKERRQ(ierr);
466     ierr = DMLabelSetStratumIS(depthLabel, d, pIS);CHKERRQ(ierr);
467     ierr = ISDestroy(&pIS);CHKERRQ(ierr);
468     for (p = pStart; p < pEnd; ++p) {
469       PetscInt       newp = DMPlexShiftPoint_Internal(p, depth, depthShift);
470       PetscInt       size;
471       DMPolytopeType ct;
472 
473       ierr = DMPlexGetConeSize(dm, p, &size);CHKERRQ(ierr);
474       ierr = DMPlexSetConeSize(dmNew, newp, size);CHKERRQ(ierr);
475       ierr = DMPlexGetSupportSize(dm, p, &size);CHKERRQ(ierr);
476       ierr = DMPlexSetSupportSize(dmNew, newp, size);CHKERRQ(ierr);
477       ierr = DMPlexGetCellType(dm, p, &ct);CHKERRQ(ierr);
478       ierr = DMPlexSetCellType(dmNew, newp, ct);CHKERRQ(ierr);
479     }
480   }
481   PetscFunctionReturn(0);
482 }
483 
484 static PetscErrorCode DMPlexShiftPoints_Internal(DM dm, PetscInt depthShift[], DM dmNew)
485 {
486   PetscInt      *newpoints;
487   PetscInt       depth = 0, maxConeSize, maxSupportSize, maxConeSizeNew, maxSupportSizeNew, pStart, pEnd, p;
488   PetscErrorCode ierr;
489 
490   PetscFunctionBegin;
491   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
492   if (depth < 0) PetscFunctionReturn(0);
493   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);CHKERRQ(ierr);
494   ierr = DMPlexGetMaxSizes(dmNew, &maxConeSizeNew, &maxSupportSizeNew);CHKERRQ(ierr);
495   ierr = PetscMalloc1(PetscMax(PetscMax(maxConeSize, maxSupportSize), PetscMax(maxConeSizeNew, maxSupportSizeNew)),&newpoints);CHKERRQ(ierr);
496   /* Step 5: Set cones and supports */
497   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
498   for (p = pStart; p < pEnd; ++p) {
499     const PetscInt *points = NULL, *orientations = NULL;
500     PetscInt        size,sizeNew, i, newp = DMPlexShiftPoint_Internal(p, depth, depthShift);
501 
502     ierr = DMPlexGetConeSize(dm, p, &size);CHKERRQ(ierr);
503     ierr = DMPlexGetCone(dm, p, &points);CHKERRQ(ierr);
504     ierr = DMPlexGetConeOrientation(dm, p, &orientations);CHKERRQ(ierr);
505     for (i = 0; i < size; ++i) {
506       newpoints[i] = DMPlexShiftPoint_Internal(points[i], depth, depthShift);
507     }
508     ierr = DMPlexSetCone(dmNew, newp, newpoints);CHKERRQ(ierr);
509     ierr = DMPlexSetConeOrientation(dmNew, newp, orientations);CHKERRQ(ierr);
510     ierr = DMPlexGetSupportSize(dm, p, &size);CHKERRQ(ierr);
511     ierr = DMPlexGetSupportSize(dmNew, newp, &sizeNew);CHKERRQ(ierr);
512     ierr = DMPlexGetSupport(dm, p, &points);CHKERRQ(ierr);
513     for (i = 0; i < size; ++i) {
514       newpoints[i] = DMPlexShiftPoint_Internal(points[i], depth, depthShift);
515     }
516     for (i = size; i < sizeNew; ++i) newpoints[i] = 0;
517     ierr = DMPlexSetSupport(dmNew, newp, newpoints);CHKERRQ(ierr);
518   }
519   ierr = PetscFree(newpoints);CHKERRQ(ierr);
520   PetscFunctionReturn(0);
521 }
522 
523 static PetscErrorCode DMPlexShiftCoordinates_Internal(DM dm, PetscInt depthShift[], DM dmNew)
524 {
525   PetscSection   coordSection, newCoordSection;
526   Vec            coordinates, newCoordinates;
527   PetscScalar   *coords, *newCoords;
528   PetscInt       coordSize, sStart, sEnd;
529   PetscInt       dim, depth = 0, cStart, cEnd, cStartNew, cEndNew, c, vStart, vEnd, vStartNew, vEndNew, v;
530   PetscBool      hasCells;
531   PetscErrorCode ierr;
532 
533   PetscFunctionBegin;
534   ierr = DMGetCoordinateDim(dm, &dim);CHKERRQ(ierr);
535   ierr = DMSetCoordinateDim(dmNew, dim);CHKERRQ(ierr);
536   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
537   /* Step 8: Convert coordinates */
538   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
539   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
540   ierr = DMPlexGetDepthStratum(dmNew, 0, &vStartNew, &vEndNew);CHKERRQ(ierr);
541   ierr = DMPlexGetHeightStratum(dmNew, 0, &cStartNew, &cEndNew);CHKERRQ(ierr);
542   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
543   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &newCoordSection);CHKERRQ(ierr);
544   ierr = PetscSectionSetNumFields(newCoordSection, 1);CHKERRQ(ierr);
545   ierr = PetscSectionSetFieldComponents(newCoordSection, 0, dim);CHKERRQ(ierr);
546   ierr = PetscSectionGetChart(coordSection, &sStart, &sEnd);CHKERRQ(ierr);
547   hasCells = sStart == cStart ? PETSC_TRUE : PETSC_FALSE;
548   ierr = PetscSectionSetChart(newCoordSection, hasCells ? cStartNew : vStartNew, vEndNew);CHKERRQ(ierr);
549   if (hasCells) {
550     for (c = cStart; c < cEnd; ++c) {
551       PetscInt cNew = DMPlexShiftPoint_Internal(c, depth, depthShift), dof;
552 
553       ierr = PetscSectionGetDof(coordSection, c, &dof);CHKERRQ(ierr);
554       ierr = PetscSectionSetDof(newCoordSection, cNew, dof);CHKERRQ(ierr);
555       ierr = PetscSectionSetFieldDof(newCoordSection, cNew, 0, dof);CHKERRQ(ierr);
556     }
557   }
558   for (v = vStartNew; v < vEndNew; ++v) {
559     ierr = PetscSectionSetDof(newCoordSection, v, dim);CHKERRQ(ierr);
560     ierr = PetscSectionSetFieldDof(newCoordSection, v, 0, dim);CHKERRQ(ierr);
561   }
562   ierr = PetscSectionSetUp(newCoordSection);CHKERRQ(ierr);
563   ierr = DMSetCoordinateSection(dmNew, PETSC_DETERMINE, newCoordSection);CHKERRQ(ierr);
564   ierr = PetscSectionGetStorageSize(newCoordSection, &coordSize);CHKERRQ(ierr);
565   ierr = VecCreate(PETSC_COMM_SELF, &newCoordinates);CHKERRQ(ierr);
566   ierr = PetscObjectSetName((PetscObject) newCoordinates, "coordinates");CHKERRQ(ierr);
567   ierr = VecSetSizes(newCoordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
568   ierr = VecSetBlockSize(newCoordinates, dim);CHKERRQ(ierr);
569   ierr = VecSetType(newCoordinates,VECSTANDARD);CHKERRQ(ierr);
570   ierr = DMSetCoordinatesLocal(dmNew, newCoordinates);CHKERRQ(ierr);
571   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
572   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
573   ierr = VecGetArray(newCoordinates, &newCoords);CHKERRQ(ierr);
574   if (hasCells) {
575     for (c = cStart; c < cEnd; ++c) {
576       PetscInt cNew = DMPlexShiftPoint_Internal(c, depth, depthShift), dof, off, noff, d;
577 
578       ierr = PetscSectionGetDof(coordSection, c, &dof);CHKERRQ(ierr);
579       ierr = PetscSectionGetOffset(coordSection, c, &off);CHKERRQ(ierr);
580       ierr = PetscSectionGetOffset(newCoordSection, cNew, &noff);CHKERRQ(ierr);
581       for (d = 0; d < dof; ++d) newCoords[noff+d] = coords[off+d];
582     }
583   }
584   for (v = vStart; v < vEnd; ++v) {
585     PetscInt dof, off, noff, d;
586 
587     ierr = PetscSectionGetDof(coordSection, v, &dof);CHKERRQ(ierr);
588     ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
589     ierr = PetscSectionGetOffset(newCoordSection, DMPlexShiftPoint_Internal(v, depth, depthShift), &noff);CHKERRQ(ierr);
590     for (d = 0; d < dof; ++d) newCoords[noff+d] = coords[off+d];
591   }
592   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
593   ierr = VecRestoreArray(newCoordinates, &newCoords);CHKERRQ(ierr);
594   ierr = VecDestroy(&newCoordinates);CHKERRQ(ierr);
595   ierr = PetscSectionDestroy(&newCoordSection);CHKERRQ(ierr);
596   PetscFunctionReturn(0);
597 }
598 
599 static PetscErrorCode DMPlexShiftSF_Internal(DM dm, PetscInt depthShift[], DM dmNew)
600 {
601   PetscInt           depth = 0;
602   PetscSF            sfPoint, sfPointNew;
603   const PetscSFNode *remotePoints;
604   PetscSFNode       *gremotePoints;
605   const PetscInt    *localPoints;
606   PetscInt          *glocalPoints, *newLocation, *newRemoteLocation;
607   PetscInt           numRoots, numLeaves, l, pStart, pEnd, totShift = 0;
608   PetscErrorCode     ierr;
609 
610   PetscFunctionBegin;
611   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
612   /* Step 9: Convert pointSF */
613   ierr = DMGetPointSF(dm, &sfPoint);CHKERRQ(ierr);
614   ierr = DMGetPointSF(dmNew, &sfPointNew);CHKERRQ(ierr);
615   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
616   ierr = PetscSFGetGraph(sfPoint, &numRoots, &numLeaves, &localPoints, &remotePoints);CHKERRQ(ierr);
617   totShift = DMPlexShiftPoint_Internal(pEnd,depth,depthShift) - pEnd;
618   if (numRoots >= 0) {
619     ierr = PetscMalloc2(numRoots,&newLocation,pEnd-pStart,&newRemoteLocation);CHKERRQ(ierr);
620     for (l=0; l<numRoots; l++) newLocation[l] = DMPlexShiftPoint_Internal(l, depth, depthShift);
621     ierr = PetscSFBcastBegin(sfPoint, MPIU_INT, newLocation, newRemoteLocation,MPI_REPLACE);CHKERRQ(ierr);
622     ierr = PetscSFBcastEnd(sfPoint, MPIU_INT, newLocation, newRemoteLocation,MPI_REPLACE);CHKERRQ(ierr);
623     ierr = PetscMalloc1(numLeaves,    &glocalPoints);CHKERRQ(ierr);
624     ierr = PetscMalloc1(numLeaves, &gremotePoints);CHKERRQ(ierr);
625     for (l = 0; l < numLeaves; ++l) {
626       glocalPoints[l]        = DMPlexShiftPoint_Internal(localPoints[l], depth, depthShift);
627       gremotePoints[l].rank  = remotePoints[l].rank;
628       gremotePoints[l].index = newRemoteLocation[localPoints[l]];
629     }
630     ierr = PetscFree2(newLocation,newRemoteLocation);CHKERRQ(ierr);
631     ierr = PetscSFSetGraph(sfPointNew, numRoots + totShift, numLeaves, glocalPoints, PETSC_OWN_POINTER, gremotePoints, PETSC_OWN_POINTER);CHKERRQ(ierr);
632   }
633   PetscFunctionReturn(0);
634 }
635 
636 static PetscErrorCode DMPlexShiftLabels_Internal(DM dm, PetscInt depthShift[], DM dmNew)
637 {
638   PetscSF            sfPoint;
639   DMLabel            vtkLabel, ghostLabel;
640   const PetscSFNode *leafRemote;
641   const PetscInt    *leafLocal;
642   PetscInt           depth = 0, numLeaves, numLabels, l, cStart, cEnd, c, fStart, fEnd, f;
643   PetscMPIInt        rank;
644   PetscErrorCode     ierr;
645 
646   PetscFunctionBegin;
647   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
648   /* Step 10: Convert labels */
649   ierr = DMGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
650   for (l = 0; l < numLabels; ++l) {
651     DMLabel         label, newlabel;
652     const char     *lname;
653     PetscBool       isDepth, isDim;
654     IS              valueIS;
655     const PetscInt *values;
656     PetscInt        numValues, val;
657 
658     ierr = DMGetLabelName(dm, l, &lname);CHKERRQ(ierr);
659     ierr = PetscStrcmp(lname, "depth", &isDepth);CHKERRQ(ierr);
660     if (isDepth) continue;
661     ierr = PetscStrcmp(lname, "dim", &isDim);CHKERRQ(ierr);
662     if (isDim) continue;
663     ierr = DMCreateLabel(dmNew, lname);CHKERRQ(ierr);
664     ierr = DMGetLabel(dm, lname, &label);CHKERRQ(ierr);
665     ierr = DMGetLabel(dmNew, lname, &newlabel);CHKERRQ(ierr);
666     ierr = DMLabelGetDefaultValue(label,&val);CHKERRQ(ierr);
667     ierr = DMLabelSetDefaultValue(newlabel,val);CHKERRQ(ierr);
668     ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
669     ierr = ISGetLocalSize(valueIS, &numValues);CHKERRQ(ierr);
670     ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
671     for (val = 0; val < numValues; ++val) {
672       IS              pointIS;
673       const PetscInt *points;
674       PetscInt        numPoints, p;
675 
676       ierr = DMLabelGetStratumIS(label, values[val], &pointIS);CHKERRQ(ierr);
677       ierr = ISGetLocalSize(pointIS, &numPoints);CHKERRQ(ierr);
678       ierr = ISGetIndices(pointIS, &points);CHKERRQ(ierr);
679       for (p = 0; p < numPoints; ++p) {
680         const PetscInt newpoint = DMPlexShiftPoint_Internal(points[p], depth, depthShift);
681 
682         ierr = DMLabelSetValue(newlabel, newpoint, values[val]);CHKERRQ(ierr);
683       }
684       ierr = ISRestoreIndices(pointIS, &points);CHKERRQ(ierr);
685       ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
686     }
687     ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
688     ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
689   }
690   /* Step 11: Make label for output (vtk) and to mark ghost points (ghost) */
691   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRMPI(ierr);
692   ierr = DMGetPointSF(dm, &sfPoint);CHKERRQ(ierr);
693   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
694   ierr = PetscSFGetGraph(sfPoint, NULL, &numLeaves, &leafLocal, &leafRemote);CHKERRQ(ierr);
695   ierr = DMCreateLabel(dmNew, "vtk");CHKERRQ(ierr);
696   ierr = DMCreateLabel(dmNew, "ghost");CHKERRQ(ierr);
697   ierr = DMGetLabel(dmNew, "vtk", &vtkLabel);CHKERRQ(ierr);
698   ierr = DMGetLabel(dmNew, "ghost", &ghostLabel);CHKERRQ(ierr);
699   for (l = 0, c = cStart; l < numLeaves && c < cEnd; ++l, ++c) {
700     for (; c < leafLocal[l] && c < cEnd; ++c) {
701       ierr = DMLabelSetValue(vtkLabel, c, 1);CHKERRQ(ierr);
702     }
703     if (leafLocal[l] >= cEnd) break;
704     if (leafRemote[l].rank == rank) {
705       ierr = DMLabelSetValue(vtkLabel, c, 1);CHKERRQ(ierr);
706     } else {
707       ierr = DMLabelSetValue(ghostLabel, c, 2);CHKERRQ(ierr);
708     }
709   }
710   for (; c < cEnd; ++c) {
711     ierr = DMLabelSetValue(vtkLabel, c, 1);CHKERRQ(ierr);
712   }
713   ierr = DMPlexGetHeightStratum(dmNew, 1, &fStart, &fEnd);CHKERRQ(ierr);
714   for (f = fStart; f < fEnd; ++f) {
715     PetscInt numCells;
716 
717     ierr = DMPlexGetSupportSize(dmNew, f, &numCells);CHKERRQ(ierr);
718     if (numCells < 2) {
719       ierr = DMLabelSetValue(ghostLabel, f, 1);CHKERRQ(ierr);
720     } else {
721       const PetscInt *cells = NULL;
722       PetscInt        vA, vB;
723 
724       ierr = DMPlexGetSupport(dmNew, f, &cells);CHKERRQ(ierr);
725       ierr = DMLabelGetValue(vtkLabel, cells[0], &vA);CHKERRQ(ierr);
726       ierr = DMLabelGetValue(vtkLabel, cells[1], &vB);CHKERRQ(ierr);
727       if (vA != 1 && vB != 1) {ierr = DMLabelSetValue(ghostLabel, f, 1);CHKERRQ(ierr);}
728     }
729   }
730   PetscFunctionReturn(0);
731 }
732 
733 static PetscErrorCode DMPlexShiftTree_Internal(DM dm, PetscInt depthShift[], DM dmNew)
734 {
735   DM             refTree;
736   PetscSection   pSec;
737   PetscInt       *parents, *childIDs;
738   PetscErrorCode ierr;
739 
740   PetscFunctionBegin;
741   ierr = DMPlexGetReferenceTree(dm,&refTree);CHKERRQ(ierr);
742   ierr = DMPlexSetReferenceTree(dmNew,refTree);CHKERRQ(ierr);
743   ierr = DMPlexGetTree(dm,&pSec,&parents,&childIDs,NULL,NULL);CHKERRQ(ierr);
744   if (pSec) {
745     PetscInt p, pStart, pEnd, *parentsShifted, pStartShifted, pEndShifted, depth;
746     PetscInt *childIDsShifted;
747     PetscSection pSecShifted;
748 
749     ierr = PetscSectionGetChart(pSec,&pStart,&pEnd);CHKERRQ(ierr);
750     ierr = DMPlexGetDepth(dm,&depth);CHKERRQ(ierr);
751     pStartShifted = DMPlexShiftPoint_Internal(pStart,depth,depthShift);
752     pEndShifted   = DMPlexShiftPoint_Internal(pEnd,depth,depthShift);
753     ierr = PetscMalloc2(pEndShifted - pStartShifted,&parentsShifted,pEndShifted-pStartShifted,&childIDsShifted);CHKERRQ(ierr);
754     ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dmNew),&pSecShifted);CHKERRQ(ierr);
755     ierr = PetscSectionSetChart(pSecShifted,pStartShifted,pEndShifted);CHKERRQ(ierr);
756     for (p = pStartShifted; p < pEndShifted; p++) {
757       /* start off assuming no children */
758       ierr = PetscSectionSetDof(pSecShifted,p,0);CHKERRQ(ierr);
759     }
760     for (p = pStart; p < pEnd; p++) {
761       PetscInt dof;
762       PetscInt pNew = DMPlexShiftPoint_Internal(p,depth,depthShift);
763 
764       ierr = PetscSectionGetDof(pSec,p,&dof);CHKERRQ(ierr);
765       ierr = PetscSectionSetDof(pSecShifted,pNew,dof);CHKERRQ(ierr);
766     }
767     ierr = PetscSectionSetUp(pSecShifted);CHKERRQ(ierr);
768     for (p = pStart; p < pEnd; p++) {
769       PetscInt dof;
770       PetscInt pNew = DMPlexShiftPoint_Internal(p,depth,depthShift);
771 
772       ierr = PetscSectionGetDof(pSec,p,&dof);CHKERRQ(ierr);
773       if (dof) {
774         PetscInt off, offNew;
775 
776         ierr = PetscSectionGetOffset(pSec,p,&off);CHKERRQ(ierr);
777         ierr = PetscSectionGetOffset(pSecShifted,pNew,&offNew);CHKERRQ(ierr);
778         parentsShifted[offNew] = DMPlexShiftPoint_Internal(parents[off],depth,depthShift);
779         childIDsShifted[offNew] = childIDs[off];
780       }
781     }
782     ierr = DMPlexSetTree(dmNew,pSecShifted,parentsShifted,childIDsShifted);CHKERRQ(ierr);
783     ierr = PetscFree2(parentsShifted,childIDsShifted);CHKERRQ(ierr);
784     ierr = PetscSectionDestroy(&pSecShifted);CHKERRQ(ierr);
785   }
786   PetscFunctionReturn(0);
787 }
788 
789 static PetscErrorCode DMPlexConstructGhostCells_Internal(DM dm, DMLabel label, PetscInt *numGhostCells, DM gdm)
790 {
791   PetscSF               sf;
792   IS                    valueIS;
793   const PetscInt       *values, *leaves;
794   PetscInt             *depthShift;
795   PetscInt              d, depth = 0, nleaves, loc, Ng, numFS, fs, fStart, fEnd, ghostCell, cEnd, c;
796   PetscBool             isper;
797   const PetscReal      *maxCell, *L;
798   const DMBoundaryType *bd;
799   PetscErrorCode        ierr;
800 
801   PetscFunctionBegin;
802   ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
803   ierr = PetscSFGetGraph(sf, NULL, &nleaves, &leaves, NULL);CHKERRQ(ierr);
804   nleaves = PetscMax(0, nleaves);
805   ierr = DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd);CHKERRQ(ierr);
806   /* Count ghost cells */
807   ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
808   ierr = ISGetLocalSize(valueIS, &numFS);CHKERRQ(ierr);
809   ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
810   Ng   = 0;
811   for (fs = 0; fs < numFS; ++fs) {
812     IS              faceIS;
813     const PetscInt *faces;
814     PetscInt        numFaces, f, numBdFaces = 0;
815 
816     ierr = DMLabelGetStratumIS(label, values[fs], &faceIS);CHKERRQ(ierr);
817     ierr = ISGetLocalSize(faceIS, &numFaces);CHKERRQ(ierr);
818     ierr = ISGetIndices(faceIS, &faces);CHKERRQ(ierr);
819     for (f = 0; f < numFaces; ++f) {
820       PetscInt numChildren;
821 
822       ierr = PetscFindInt(faces[f], nleaves, leaves, &loc);CHKERRQ(ierr);
823       ierr = DMPlexGetTreeChildren(dm,faces[f],&numChildren,NULL);CHKERRQ(ierr);
824       /* non-local and ancestors points don't get to register ghosts */
825       if (loc >= 0 || numChildren) continue;
826       if ((faces[f] >= fStart) && (faces[f] < fEnd)) ++numBdFaces;
827     }
828     Ng += numBdFaces;
829     ierr = ISRestoreIndices(faceIS, &faces);CHKERRQ(ierr);
830     ierr = ISDestroy(&faceIS);CHKERRQ(ierr);
831   }
832   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
833   ierr = PetscMalloc1(2*(depth+1), &depthShift);CHKERRQ(ierr);
834   for (d = 0; d <= depth; d++) {
835     PetscInt dEnd;
836 
837     ierr = DMPlexGetDepthStratum(dm,d,NULL,&dEnd);CHKERRQ(ierr);
838     depthShift[2*d]   = dEnd;
839     depthShift[2*d+1] = 0;
840   }
841   if (depth >= 0) depthShift[2*depth+1] = Ng;
842   ierr = DMPlexShiftPointSetUp_Internal(depth,depthShift);CHKERRQ(ierr);
843   ierr = DMPlexShiftSizes_Internal(dm, depthShift, gdm);CHKERRQ(ierr);
844   /* Step 3: Set cone/support sizes for new points */
845   ierr = DMPlexGetHeightStratum(dm, 0, NULL, &cEnd);CHKERRQ(ierr);
846   for (c = cEnd; c < cEnd + Ng; ++c) {
847     ierr = DMPlexSetConeSize(gdm, c, 1);CHKERRQ(ierr);
848   }
849   for (fs = 0; fs < numFS; ++fs) {
850     IS              faceIS;
851     const PetscInt *faces;
852     PetscInt        numFaces, f;
853 
854     ierr = DMLabelGetStratumIS(label, values[fs], &faceIS);CHKERRQ(ierr);
855     ierr = ISGetLocalSize(faceIS, &numFaces);CHKERRQ(ierr);
856     ierr = ISGetIndices(faceIS, &faces);CHKERRQ(ierr);
857     for (f = 0; f < numFaces; ++f) {
858       PetscInt size, numChildren;
859 
860       ierr = PetscFindInt(faces[f], nleaves, leaves, &loc);CHKERRQ(ierr);
861       ierr = DMPlexGetTreeChildren(dm,faces[f],&numChildren,NULL);CHKERRQ(ierr);
862       if (loc >= 0 || numChildren) continue;
863       if ((faces[f] < fStart) || (faces[f] >= fEnd)) continue;
864       ierr = DMPlexGetSupportSize(dm, faces[f], &size);CHKERRQ(ierr);
865       if (size != 1) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM has boundary face %d with %d support cells", faces[f], size);
866       ierr = DMPlexSetSupportSize(gdm, faces[f] + Ng, 2);CHKERRQ(ierr);
867     }
868     ierr = ISRestoreIndices(faceIS, &faces);CHKERRQ(ierr);
869     ierr = ISDestroy(&faceIS);CHKERRQ(ierr);
870   }
871   /* Step 4: Setup ghosted DM */
872   ierr = DMSetUp(gdm);CHKERRQ(ierr);
873   ierr = DMPlexShiftPoints_Internal(dm, depthShift, gdm);CHKERRQ(ierr);
874   /* Step 6: Set cones and supports for new points */
875   ghostCell = cEnd;
876   for (fs = 0; fs < numFS; ++fs) {
877     IS              faceIS;
878     const PetscInt *faces;
879     PetscInt        numFaces, f;
880 
881     ierr = DMLabelGetStratumIS(label, values[fs], &faceIS);CHKERRQ(ierr);
882     ierr = ISGetLocalSize(faceIS, &numFaces);CHKERRQ(ierr);
883     ierr = ISGetIndices(faceIS, &faces);CHKERRQ(ierr);
884     for (f = 0; f < numFaces; ++f) {
885       PetscInt newFace = faces[f] + Ng, numChildren;
886 
887       ierr = PetscFindInt(faces[f], nleaves, leaves, &loc);CHKERRQ(ierr);
888       ierr = DMPlexGetTreeChildren(dm,faces[f],&numChildren,NULL);CHKERRQ(ierr);
889       if (loc >= 0 || numChildren) continue;
890       if ((faces[f] < fStart) || (faces[f] >= fEnd)) continue;
891       ierr = DMPlexSetCone(gdm, ghostCell, &newFace);CHKERRQ(ierr);
892       ierr = DMPlexInsertSupport(gdm, newFace, 1, ghostCell);CHKERRQ(ierr);
893       ++ghostCell;
894     }
895     ierr = ISRestoreIndices(faceIS, &faces);CHKERRQ(ierr);
896     ierr = ISDestroy(&faceIS);CHKERRQ(ierr);
897   }
898   ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
899   ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
900   ierr = DMPlexShiftCoordinates_Internal(dm, depthShift, gdm);CHKERRQ(ierr);
901   ierr = DMPlexShiftSF_Internal(dm, depthShift, gdm);CHKERRQ(ierr);
902   ierr = DMPlexShiftLabels_Internal(dm, depthShift, gdm);CHKERRQ(ierr);
903   ierr = DMPlexShiftTree_Internal(dm, depthShift, gdm);CHKERRQ(ierr);
904   ierr = PetscFree(depthShift);CHKERRQ(ierr);
905   for (c = cEnd; c < cEnd + Ng; ++c) {
906     ierr = DMPlexSetCellType(gdm, c, DM_POLYTOPE_FV_GHOST);CHKERRQ(ierr);
907   }
908   /* Step 7: Periodicity */
909   ierr = DMGetPeriodicity(dm, &isper, &maxCell, &L, &bd);CHKERRQ(ierr);
910   ierr = DMSetPeriodicity(gdm, isper, maxCell,  L,  bd);CHKERRQ(ierr);
911   if (numGhostCells) *numGhostCells = Ng;
912   PetscFunctionReturn(0);
913 }
914 
915 /*@C
916   DMPlexConstructGhostCells - Construct ghost cells which connect to every boundary face
917 
918   Collective on dm
919 
920   Input Parameters:
921 + dm - The original DM
922 - labelName - The label specifying the boundary faces, or "Face Sets" if this is NULL
923 
924   Output Parameters:
925 + numGhostCells - The number of ghost cells added to the DM
926 - dmGhosted - The new DM
927 
928   Note: If no label exists of that name, one will be created marking all boundary faces
929 
930   Level: developer
931 
932 .seealso: DMCreate()
933 @*/
934 PetscErrorCode DMPlexConstructGhostCells(DM dm, const char labelName[], PetscInt *numGhostCells, DM *dmGhosted)
935 {
936   DM             gdm;
937   DMLabel        label;
938   const char    *name = labelName ? labelName : "Face Sets";
939   PetscInt       dim, Ng = 0;
940   PetscBool      useCone, useClosure;
941   PetscErrorCode ierr;
942 
943   PetscFunctionBegin;
944   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
945   if (numGhostCells) PetscValidPointer(numGhostCells, 3);
946   PetscValidPointer(dmGhosted, 4);
947   ierr = DMCreate(PetscObjectComm((PetscObject)dm), &gdm);CHKERRQ(ierr);
948   ierr = DMSetType(gdm, DMPLEX);CHKERRQ(ierr);
949   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
950   ierr = DMSetDimension(gdm, dim);CHKERRQ(ierr);
951   ierr = DMGetBasicAdjacency(dm, &useCone, &useClosure);CHKERRQ(ierr);
952   ierr = DMSetBasicAdjacency(gdm, useCone,  useClosure);CHKERRQ(ierr);
953   ierr = DMGetLabel(dm, name, &label);CHKERRQ(ierr);
954   if (!label) {
955     /* Get label for boundary faces */
956     ierr = DMCreateLabel(dm, name);CHKERRQ(ierr);
957     ierr = DMGetLabel(dm, name, &label);CHKERRQ(ierr);
958     ierr = DMPlexMarkBoundaryFaces(dm, 1, label);CHKERRQ(ierr);
959   }
960   ierr = DMPlexConstructGhostCells_Internal(dm, label, &Ng, gdm);CHKERRQ(ierr);
961   ierr = DMCopyDisc(dm, gdm);CHKERRQ(ierr);
962   gdm->setfromoptionscalled = dm->setfromoptionscalled;
963   if (numGhostCells) *numGhostCells = Ng;
964   *dmGhosted = gdm;
965   PetscFunctionReturn(0);
966 }
967 
968 /*
969   We are adding three kinds of points here:
970     Replicated:     Copies of points which exist in the mesh, such as vertices identified across a fault
971     Non-replicated: Points which exist on the fault, but are not replicated
972     Hybrid:         Entirely new points, such as cohesive cells
973 
974   When creating subsequent cohesive cells, we shift the old hybrid cells to the end of the numbering at
975   each depth so that the new split/hybrid points can be inserted as a block.
976 */
977 static PetscErrorCode DMPlexConstructCohesiveCells_Internal(DM dm, DMLabel label, DMLabel splitLabel, DM sdm)
978 {
979   MPI_Comm         comm;
980   IS               valueIS;
981   PetscInt         numSP = 0;          /* The number of depths for which we have replicated points */
982   const PetscInt  *values;             /* List of depths for which we have replicated points */
983   IS              *splitIS;
984   IS              *unsplitIS;
985   PetscInt        *numSplitPoints;     /* The number of replicated points at each depth */
986   PetscInt        *numUnsplitPoints;   /* The number of non-replicated points at each depth which still give rise to hybrid points */
987   PetscInt        *numHybridPoints;    /* The number of new hybrid points at each depth */
988   PetscInt        *numHybridPointsOld; /* The number of existing hybrid points at each depth */
989   const PetscInt **splitPoints;        /* Replicated points for each depth */
990   const PetscInt **unsplitPoints;      /* Non-replicated points for each depth */
991   PetscSection     coordSection;
992   Vec              coordinates;
993   PetscScalar     *coords;
994   PetscInt        *depthMax;           /* The first hybrid point at each depth in the original mesh */
995   PetscInt        *depthEnd;           /* The point limit at each depth in the original mesh */
996   PetscInt        *depthShift;         /* Number of replicated+hybrid points at each depth */
997   PetscInt        *pMaxNew;            /* The first replicated point at each depth in the new mesh, hybrids come after this */
998   PetscInt        *coneNew, *coneONew, *supportNew;
999   PetscInt         shift = 100, shift2 = 200, depth = 0, dep, dim, d, sp, maxConeSize, maxSupportSize, maxConeSizeNew, maxSupportSizeNew, numLabels, vStart, vEnd, pEnd, p, v;
1000   PetscErrorCode   ierr;
1001 
1002   PetscFunctionBegin;
1003   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
1004   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
1005   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
1006   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
1007   /* We do not want this label automatically computed, instead we compute it here */
1008   ierr = DMCreateLabel(sdm, "celltype");CHKERRQ(ierr);
1009   /* Count split points and add cohesive cells */
1010   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);CHKERRQ(ierr);
1011   ierr = PetscMalloc5(depth+1,&depthMax,depth+1,&depthEnd,2*(depth+1),&depthShift,depth+1,&pMaxNew,depth+1,&numHybridPointsOld);CHKERRQ(ierr);
1012   ierr = PetscMalloc7(depth+1,&splitIS,depth+1,&unsplitIS,depth+1,&numSplitPoints,depth+1,&numUnsplitPoints,depth+1,&numHybridPoints,depth+1,&splitPoints,depth+1,&unsplitPoints);CHKERRQ(ierr);
1013   for (d = 0; d <= depth; ++d) {
1014     ierr = DMPlexGetDepthStratum(dm, d, NULL, &pMaxNew[d]);CHKERRQ(ierr);
1015     ierr = DMPlexGetTensorPrismBounds_Internal(dm, d, &depthMax[d], NULL);CHKERRQ(ierr);
1016     depthEnd[d]           = pMaxNew[d];
1017     depthMax[d]           = depthMax[d] < 0 ? depthEnd[d] : depthMax[d];
1018     numSplitPoints[d]     = 0;
1019     numUnsplitPoints[d]   = 0;
1020     numHybridPoints[d]    = 0;
1021     numHybridPointsOld[d] = depthMax[d] < 0 ? 0 : depthEnd[d] - depthMax[d];
1022     splitPoints[d]        = NULL;
1023     unsplitPoints[d]      = NULL;
1024     splitIS[d]            = NULL;
1025     unsplitIS[d]          = NULL;
1026     /* we are shifting the existing hybrid points with the stratum behind them, so
1027      * the split comes at the end of the normal points, i.e., at depthMax[d] */
1028     depthShift[2*d]       = depthMax[d];
1029     depthShift[2*d+1]     = 0;
1030   }
1031   if (label) {
1032     ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
1033     ierr = ISGetLocalSize(valueIS, &numSP);CHKERRQ(ierr);
1034     ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
1035   }
1036   for (sp = 0; sp < numSP; ++sp) {
1037     const PetscInt dep = values[sp];
1038 
1039     if ((dep < 0) || (dep > depth)) continue;
1040     ierr = DMLabelGetStratumIS(label, dep, &splitIS[dep]);CHKERRQ(ierr);
1041     if (splitIS[dep]) {
1042       ierr = ISGetLocalSize(splitIS[dep], &numSplitPoints[dep]);CHKERRQ(ierr);
1043       ierr = ISGetIndices(splitIS[dep], &splitPoints[dep]);CHKERRQ(ierr);
1044     }
1045     ierr = DMLabelGetStratumIS(label, shift2+dep, &unsplitIS[dep]);CHKERRQ(ierr);
1046     if (unsplitIS[dep]) {
1047       ierr = ISGetLocalSize(unsplitIS[dep], &numUnsplitPoints[dep]);CHKERRQ(ierr);
1048       ierr = ISGetIndices(unsplitIS[dep], &unsplitPoints[dep]);CHKERRQ(ierr);
1049     }
1050   }
1051   /* Calculate number of hybrid points */
1052   for (d = 1; d <= depth; ++d) numHybridPoints[d]     = numSplitPoints[d-1] + numUnsplitPoints[d-1]; /* There is a hybrid cell/face/edge for every split face/edge/vertex   */
1053   for (d = 0; d <= depth; ++d) depthShift[2*d+1]      = numSplitPoints[d] + numHybridPoints[d];
1054   ierr = DMPlexShiftPointSetUp_Internal(depth,depthShift);CHKERRQ(ierr);
1055   /* the end of the points in this stratum that come before the new points:
1056    * shifting pMaxNew[d] gets the new start of the next stratum, then count back the old hybrid points and the newly
1057    * added points */
1058   for (d = 0; d <= depth; ++d) pMaxNew[d]             = DMPlexShiftPoint_Internal(pMaxNew[d],depth,depthShift) - (numHybridPointsOld[d] + numSplitPoints[d] + numHybridPoints[d]);
1059   ierr = DMPlexShiftSizes_Internal(dm, depthShift, sdm);CHKERRQ(ierr);
1060   /* Step 3: Set cone/support sizes for new points */
1061   for (dep = 0; dep <= depth; ++dep) {
1062     for (p = 0; p < numSplitPoints[dep]; ++p) {
1063       const PetscInt  oldp   = splitPoints[dep][p];
1064       const PetscInt  newp   = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*oldp + depthOffset[dep]*/;
1065       const PetscInt  splitp = p    + pMaxNew[dep];
1066       const PetscInt *support;
1067       DMPolytopeType  ct;
1068       PetscInt        coneSize, supportSize, qf, qn, qp, e;
1069 
1070       ierr = DMPlexGetConeSize(dm, oldp, &coneSize);CHKERRQ(ierr);
1071       ierr = DMPlexSetConeSize(sdm, splitp, coneSize);CHKERRQ(ierr);
1072       ierr = DMPlexGetSupportSize(dm, oldp, &supportSize);CHKERRQ(ierr);
1073       ierr = DMPlexSetSupportSize(sdm, splitp, supportSize);CHKERRQ(ierr);
1074       ierr = DMPlexGetCellType(dm, oldp, &ct);CHKERRQ(ierr);
1075       ierr = DMPlexSetCellType(sdm, splitp, ct);CHKERRQ(ierr);
1076       if (dep == depth-1) {
1077         const PetscInt hybcell = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1078 
1079         /* Add cohesive cells, they are prisms */
1080         ierr = DMPlexSetConeSize(sdm, hybcell, 2 + coneSize);CHKERRQ(ierr);
1081         switch (coneSize) {
1082           case 2: ierr = DMPlexSetCellType(sdm, hybcell, DM_POLYTOPE_SEG_PRISM_TENSOR);CHKERRQ(ierr);break;
1083           case 3: ierr = DMPlexSetCellType(sdm, hybcell, DM_POLYTOPE_TRI_PRISM_TENSOR);CHKERRQ(ierr);break;
1084           case 4: ierr = DMPlexSetCellType(sdm, hybcell, DM_POLYTOPE_QUAD_PRISM_TENSOR);CHKERRQ(ierr);break;
1085         }
1086       } else if (dep == 0) {
1087         const PetscInt hybedge = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1088 
1089         ierr = DMPlexGetSupport(dm, oldp, &support);CHKERRQ(ierr);
1090         for (e = 0, qn = 0, qp = 0, qf = 0; e < supportSize; ++e) {
1091           PetscInt val;
1092 
1093           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
1094           if (val == 1) ++qf;
1095           if ((val == 1) || (val ==  (shift + 1))) ++qn;
1096           if ((val == 1) || (val == -(shift + 1))) ++qp;
1097         }
1098         /* Split old vertex: Edges into original vertex and new cohesive edge */
1099         ierr = DMPlexSetSupportSize(sdm, newp, qn+1);CHKERRQ(ierr);
1100         /* Split new vertex: Edges into split vertex and new cohesive edge */
1101         ierr = DMPlexSetSupportSize(sdm, splitp, qp+1);CHKERRQ(ierr);
1102         /* Add hybrid edge */
1103         ierr = DMPlexSetConeSize(sdm, hybedge, 2);CHKERRQ(ierr);
1104         ierr = DMPlexSetSupportSize(sdm, hybedge, qf);CHKERRQ(ierr);
1105         ierr = DMPlexSetCellType(sdm, hybedge, DM_POLYTOPE_POINT_PRISM_TENSOR);CHKERRQ(ierr);
1106       } else if (dep == dim-2) {
1107         const PetscInt hybface = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1108 
1109         ierr = DMPlexGetSupport(dm, oldp, &support);CHKERRQ(ierr);
1110         for (e = 0, qn = 0, qp = 0, qf = 0; e < supportSize; ++e) {
1111           PetscInt val;
1112 
1113           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
1114           if (val == dim-1) ++qf;
1115           if ((val == dim-1) || (val ==  (shift + dim-1))) ++qn;
1116           if ((val == dim-1) || (val == -(shift + dim-1))) ++qp;
1117         }
1118         /* Split old edge: Faces into original edge and cohesive face (positive side?) */
1119         ierr = DMPlexSetSupportSize(sdm, newp, qn+1);CHKERRQ(ierr);
1120         /* Split new edge: Faces into split edge and cohesive face (negative side?) */
1121         ierr = DMPlexSetSupportSize(sdm, splitp, qp+1);CHKERRQ(ierr);
1122         /* Add hybrid face */
1123         ierr = DMPlexSetConeSize(sdm, hybface, 4);CHKERRQ(ierr);
1124         ierr = DMPlexSetSupportSize(sdm, hybface, qf);CHKERRQ(ierr);
1125         ierr = DMPlexSetCellType(sdm, hybface, DM_POLYTOPE_SEG_PRISM_TENSOR);CHKERRQ(ierr);
1126       }
1127     }
1128   }
1129   for (dep = 0; dep <= depth; ++dep) {
1130     for (p = 0; p < numUnsplitPoints[dep]; ++p) {
1131       const PetscInt  oldp   = unsplitPoints[dep][p];
1132       const PetscInt  newp   = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*oldp + depthOffset[dep]*/;
1133       const PetscInt *support;
1134       PetscInt        coneSize, supportSize, qf, e, s;
1135 
1136       ierr = DMPlexGetConeSize(dm, oldp, &coneSize);CHKERRQ(ierr);
1137       ierr = DMPlexGetSupportSize(dm, oldp, &supportSize);CHKERRQ(ierr);
1138       ierr = DMPlexGetSupport(dm, oldp, &support);CHKERRQ(ierr);
1139       if (dep == 0) {
1140         const PetscInt hybedge = p + pMaxNew[dep+1] + numSplitPoints[dep+1] + numSplitPoints[dep];
1141 
1142         /* Unsplit vertex: Edges into original vertex, split edges, and new cohesive edge twice */
1143         for (s = 0, qf = 0; s < supportSize; ++s, ++qf) {
1144           ierr = PetscFindInt(support[s], numSplitPoints[dep+1], splitPoints[dep+1], &e);CHKERRQ(ierr);
1145           if (e >= 0) ++qf;
1146         }
1147         ierr = DMPlexSetSupportSize(sdm, newp, qf+2);CHKERRQ(ierr);
1148         /* Add hybrid edge */
1149         ierr = DMPlexSetConeSize(sdm, hybedge, 2);CHKERRQ(ierr);
1150         for (e = 0, qf = 0; e < supportSize; ++e) {
1151           PetscInt val;
1152 
1153           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
1154           /* Split and unsplit edges produce hybrid faces */
1155           if (val == 1) ++qf;
1156           if (val == (shift2 + 1)) ++qf;
1157         }
1158         ierr = DMPlexSetSupportSize(sdm, hybedge, qf);CHKERRQ(ierr);
1159         ierr = DMPlexSetCellType(sdm, hybedge, DM_POLYTOPE_POINT_PRISM_TENSOR);CHKERRQ(ierr);
1160       } else if (dep == dim-2) {
1161         const PetscInt hybface = p + pMaxNew[dep+1] + numSplitPoints[dep+1] + numSplitPoints[dep];
1162         PetscInt       val;
1163 
1164         for (e = 0, qf = 0; e < supportSize; ++e) {
1165           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
1166           if (val == dim-1) qf += 2;
1167           else              ++qf;
1168         }
1169         /* Unsplit edge: Faces into original edge, split face, and cohesive face twice */
1170         ierr = DMPlexSetSupportSize(sdm, newp, qf+2);CHKERRQ(ierr);
1171         /* Add hybrid face */
1172         for (e = 0, qf = 0; e < supportSize; ++e) {
1173           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
1174           if (val == dim-1) ++qf;
1175         }
1176         ierr = DMPlexSetConeSize(sdm, hybface, 4);CHKERRQ(ierr);
1177         ierr = DMPlexSetSupportSize(sdm, hybface, qf);CHKERRQ(ierr);
1178         ierr = DMPlexSetCellType(sdm, hybface, DM_POLYTOPE_SEG_PRISM_TENSOR);CHKERRQ(ierr);
1179       }
1180     }
1181   }
1182   /* Step 4: Setup split DM */
1183   ierr = DMSetUp(sdm);CHKERRQ(ierr);
1184   ierr = DMPlexShiftPoints_Internal(dm, depthShift, sdm);CHKERRQ(ierr);
1185   ierr = DMPlexGetMaxSizes(sdm, &maxConeSizeNew, &maxSupportSizeNew);CHKERRQ(ierr);
1186   ierr = PetscMalloc3(PetscMax(maxConeSize, maxConeSizeNew)*3,&coneNew,PetscMax(maxConeSize, maxConeSizeNew)*3,&coneONew,PetscMax(maxSupportSize, maxSupportSizeNew),&supportNew);CHKERRQ(ierr);
1187   /* Step 6: Set cones and supports for new points */
1188   for (dep = 0; dep <= depth; ++dep) {
1189     for (p = 0; p < numSplitPoints[dep]; ++p) {
1190       const PetscInt  oldp   = splitPoints[dep][p];
1191       const PetscInt  newp   = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*oldp + depthOffset[dep]*/;
1192       const PetscInt  splitp = p    + pMaxNew[dep];
1193       const PetscInt *cone, *support, *ornt;
1194       DMPolytopeType  ct;
1195       PetscInt        coneSize, supportSize, q, qf, qn, qp, v, e, s;
1196 
1197       ierr = DMPlexGetCellType(dm, oldp, &ct);CHKERRQ(ierr);
1198       ierr = DMPlexGetConeSize(dm, oldp, &coneSize);CHKERRQ(ierr);
1199       ierr = DMPlexGetCone(dm, oldp, &cone);CHKERRQ(ierr);
1200       ierr = DMPlexGetConeOrientation(dm, oldp, &ornt);CHKERRQ(ierr);
1201       ierr = DMPlexGetSupportSize(dm, oldp, &supportSize);CHKERRQ(ierr);
1202       ierr = DMPlexGetSupport(dm, oldp, &support);CHKERRQ(ierr);
1203       if (dep == depth-1) {
1204         PetscBool       hasUnsplit = PETSC_FALSE;
1205         const PetscInt  hybcell    = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1206         const PetscInt *supportF;
1207 
1208         /* Split face:       copy in old face to new face to start */
1209         ierr = DMPlexGetSupport(sdm, newp,  &supportF);CHKERRQ(ierr);
1210         ierr = DMPlexSetSupport(sdm, splitp, supportF);CHKERRQ(ierr);
1211         /* Split old face:   old vertices/edges in cone so no change */
1212         /* Split new face:   new vertices/edges in cone */
1213         for (q = 0; q < coneSize; ++q) {
1214           ierr = PetscFindInt(cone[q], numSplitPoints[dep-1], splitPoints[dep-1], &v);CHKERRQ(ierr);
1215           if (v < 0) {
1216             ierr = PetscFindInt(cone[q], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &v);CHKERRQ(ierr);
1217             if (v < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %d in split or unsplit points of depth %d", cone[q], dep-1);
1218             coneNew[2+q] = DMPlexShiftPoint_Internal(cone[q], depth, depthShift) /*cone[q] + depthOffset[dep-1]*/;
1219             hasUnsplit   = PETSC_TRUE;
1220           } else {
1221             coneNew[2+q] = v + pMaxNew[dep-1];
1222             if (dep > 1) {
1223               const PetscInt *econe;
1224               PetscInt        econeSize, r, vs, vu;
1225 
1226               ierr = DMPlexGetConeSize(dm, cone[q], &econeSize);CHKERRQ(ierr);
1227               ierr = DMPlexGetCone(dm, cone[q], &econe);CHKERRQ(ierr);
1228               for (r = 0; r < econeSize; ++r) {
1229                 ierr = PetscFindInt(econe[r], numSplitPoints[dep-2],   splitPoints[dep-2],   &vs);CHKERRQ(ierr);
1230                 ierr = PetscFindInt(econe[r], numUnsplitPoints[dep-2], unsplitPoints[dep-2], &vu);CHKERRQ(ierr);
1231                 if (vs >= 0) continue;
1232                 if (vu < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %d in split or unsplit points of depth %d", econe[r], dep-2);
1233                 hasUnsplit   = PETSC_TRUE;
1234               }
1235             }
1236           }
1237         }
1238         ierr = DMPlexSetCone(sdm, splitp, &coneNew[2]);CHKERRQ(ierr);
1239         ierr = DMPlexSetConeOrientation(sdm, splitp, ornt);CHKERRQ(ierr);
1240         /* Face support */
1241         for (s = 0; s < supportSize; ++s) {
1242           PetscInt val;
1243 
1244           ierr = DMLabelGetValue(label, support[s], &val);CHKERRQ(ierr);
1245           if (val < 0) {
1246             /* Split old face:   Replace negative side cell with cohesive cell */
1247              ierr = DMPlexInsertSupport(sdm, newp, s, hybcell);CHKERRQ(ierr);
1248           } else {
1249             /* Split new face:   Replace positive side cell with cohesive cell */
1250             ierr = DMPlexInsertSupport(sdm, splitp, s, hybcell);CHKERRQ(ierr);
1251             /* Get orientation for cohesive face */
1252             {
1253               const PetscInt *ncone, *nconeO;
1254               PetscInt        nconeSize, nc;
1255 
1256               ierr = DMPlexGetConeSize(dm, support[s], &nconeSize);CHKERRQ(ierr);
1257               ierr = DMPlexGetCone(dm, support[s], &ncone);CHKERRQ(ierr);
1258               ierr = DMPlexGetConeOrientation(dm, support[s], &nconeO);CHKERRQ(ierr);
1259               for (nc = 0; nc < nconeSize; ++nc) {
1260                 if (ncone[nc] == oldp) {
1261                   coneONew[0] = nconeO[nc];
1262                   break;
1263                 }
1264               }
1265               if (nc >= nconeSize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate face %d in neighboring cell %d", oldp, support[s]);
1266             }
1267           }
1268         }
1269         /* Cohesive cell:    Old and new split face, then new cohesive faces */
1270         const PetscInt *arr = DMPolytopeTypeGetArrangment(ct, coneONew[0]);
1271 
1272         coneNew[0]  = newp;   /* Extracted negative side orientation above */
1273         coneNew[1]  = splitp;
1274         coneONew[1] = coneONew[0];
1275         for (q = 0; q < coneSize; ++q) {
1276           /* Hybrid faces must follow order from oriented end face */
1277           const PetscInt qa = arr[q*2+0];
1278           const PetscInt qo = arr[q*2+1];
1279           DMPolytopeType ft = dep == 2 ? DM_POLYTOPE_SEGMENT : DM_POLYTOPE_POINT;
1280 
1281           ierr = PetscFindInt(cone[qa], numSplitPoints[dep-1], splitPoints[dep-1], &v);CHKERRQ(ierr);
1282           if (v < 0) {
1283             ierr = PetscFindInt(cone[qa], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &v);CHKERRQ(ierr);
1284             coneNew[2+q]  = v + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep-1];
1285           } else {
1286             coneNew[2+q]  = v + pMaxNew[dep] + numSplitPoints[dep];
1287           }
1288           coneONew[2+q] = DMPolytopeTypeComposeOrientation(ft, qo, ornt[qa]);
1289         }
1290         ierr = DMPlexSetCone(sdm, hybcell, coneNew);CHKERRQ(ierr);
1291         ierr = DMPlexSetConeOrientation(sdm, hybcell, coneONew);CHKERRQ(ierr);
1292         /* Label the hybrid cells on the boundary of the split */
1293         if (hasUnsplit) {ierr = DMLabelSetValue(label, -hybcell, dim);CHKERRQ(ierr);}
1294       } else if (dep == 0) {
1295         const PetscInt hybedge = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1296 
1297         /* Split old vertex: Edges in old split faces and new cohesive edge */
1298         for (e = 0, qn = 0; e < supportSize; ++e) {
1299           PetscInt val;
1300 
1301           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
1302           if ((val == 1) || (val == (shift + 1))) {
1303             supportNew[qn++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1304           }
1305         }
1306         supportNew[qn] = hybedge;
1307         ierr = DMPlexSetSupport(sdm, newp, supportNew);CHKERRQ(ierr);
1308         /* Split new vertex: Edges in new split faces and new cohesive edge */
1309         for (e = 0, qp = 0; e < supportSize; ++e) {
1310           PetscInt val, edge;
1311 
1312           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
1313           if (val == 1) {
1314             ierr = PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &edge);CHKERRQ(ierr);
1315             if (edge < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Edge %d is not a split edge", support[e]);
1316             supportNew[qp++] = edge + pMaxNew[dep+1];
1317           } else if (val == -(shift + 1)) {
1318             supportNew[qp++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1319           }
1320         }
1321         supportNew[qp] = hybedge;
1322         ierr = DMPlexSetSupport(sdm, splitp, supportNew);CHKERRQ(ierr);
1323         /* Hybrid edge:    Old and new split vertex */
1324         coneNew[0] = newp;
1325         coneNew[1] = splitp;
1326         ierr = DMPlexSetCone(sdm, hybedge, coneNew);CHKERRQ(ierr);
1327         for (e = 0, qf = 0; e < supportSize; ++e) {
1328           PetscInt val, edge;
1329 
1330           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
1331           if (val == 1) {
1332             ierr = PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &edge);CHKERRQ(ierr);
1333             if (edge < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Edge %d is not a split edge", support[e]);
1334             supportNew[qf++] = edge + pMaxNew[dep+2] + numSplitPoints[dep+2];
1335           }
1336         }
1337         ierr = DMPlexSetSupport(sdm, hybedge, supportNew);CHKERRQ(ierr);
1338       } else if (dep == dim-2) {
1339         const PetscInt hybface = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1340 
1341         /* Split old edge:   old vertices in cone so no change */
1342         /* Split new edge:   new vertices in cone */
1343         for (q = 0; q < coneSize; ++q) {
1344           ierr = PetscFindInt(cone[q], numSplitPoints[dep-1], splitPoints[dep-1], &v);CHKERRQ(ierr);
1345           if (v < 0) {
1346             ierr = PetscFindInt(cone[q], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &v);CHKERRQ(ierr);
1347             if (v < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %d in split or unsplit points of depth %d", cone[q], dep-1);
1348             coneNew[q] = DMPlexShiftPoint_Internal(cone[q], depth, depthShift) /*cone[q] + depthOffset[dep-1]*/;
1349           } else {
1350             coneNew[q] = v + pMaxNew[dep-1];
1351           }
1352         }
1353         ierr = DMPlexSetCone(sdm, splitp, coneNew);CHKERRQ(ierr);
1354         /* Split old edge: Faces in positive side cells and old split faces */
1355         for (e = 0, q = 0; e < supportSize; ++e) {
1356           PetscInt val;
1357 
1358           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
1359           if (val == dim-1) {
1360             supportNew[q++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1361           } else if (val == (shift + dim-1)) {
1362             supportNew[q++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1363           }
1364         }
1365         supportNew[q++] = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1366         ierr = DMPlexSetSupport(sdm, newp, supportNew);CHKERRQ(ierr);
1367         /* Split new edge: Faces in negative side cells and new split faces */
1368         for (e = 0, q = 0; e < supportSize; ++e) {
1369           PetscInt val, face;
1370 
1371           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
1372           if (val == dim-1) {
1373             ierr = PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &face);CHKERRQ(ierr);
1374             if (face < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Face %d is not a split face", support[e]);
1375             supportNew[q++] = face + pMaxNew[dep+1];
1376           } else if (val == -(shift + dim-1)) {
1377             supportNew[q++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1378           }
1379         }
1380         supportNew[q++] = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1381         ierr = DMPlexSetSupport(sdm, splitp, supportNew);CHKERRQ(ierr);
1382         /* Hybrid face */
1383         coneNew[0] = newp;
1384         coneNew[1] = splitp;
1385         for (v = 0; v < coneSize; ++v) {
1386           PetscInt vertex;
1387           ierr = PetscFindInt(cone[v], numSplitPoints[dep-1], splitPoints[dep-1], &vertex);CHKERRQ(ierr);
1388           if (vertex < 0) {
1389             ierr = PetscFindInt(cone[v], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &vertex);CHKERRQ(ierr);
1390             if (vertex < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %d in split or unsplit points of depth %d", cone[v], dep-1);
1391             coneNew[2+v] = vertex + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep-1];
1392           } else {
1393             coneNew[2+v] = vertex + pMaxNew[dep] + numSplitPoints[dep];
1394           }
1395         }
1396         ierr = DMPlexSetCone(sdm, hybface, coneNew);CHKERRQ(ierr);
1397         for (e = 0, qf = 0; e < supportSize; ++e) {
1398           PetscInt val, face;
1399 
1400           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
1401           if (val == dim-1) {
1402             ierr = PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &face);CHKERRQ(ierr);
1403             if (face < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Face %d is not a split face", support[e]);
1404             supportNew[qf++] = face + pMaxNew[dep+2] + numSplitPoints[dep+2];
1405           }
1406         }
1407         ierr = DMPlexSetSupport(sdm, hybface, supportNew);CHKERRQ(ierr);
1408       }
1409     }
1410   }
1411   for (dep = 0; dep <= depth; ++dep) {
1412     for (p = 0; p < numUnsplitPoints[dep]; ++p) {
1413       const PetscInt  oldp   = unsplitPoints[dep][p];
1414       const PetscInt  newp   = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*oldp + depthOffset[dep]*/;
1415       const PetscInt *cone, *support;
1416       PetscInt        coneSize, supportSize, supportSizeNew, q, qf, e, f, s;
1417 
1418       ierr = DMPlexGetConeSize(dm, oldp, &coneSize);CHKERRQ(ierr);
1419       ierr = DMPlexGetCone(dm, oldp, &cone);CHKERRQ(ierr);
1420       ierr = DMPlexGetSupportSize(dm, oldp, &supportSize);CHKERRQ(ierr);
1421       ierr = DMPlexGetSupport(dm, oldp, &support);CHKERRQ(ierr);
1422       if (dep == 0) {
1423         const PetscInt hybedge = p + pMaxNew[dep+1] + numSplitPoints[dep+1] + numSplitPoints[dep];
1424 
1425         /* Unsplit vertex */
1426         ierr = DMPlexGetSupportSize(sdm, newp, &supportSizeNew);CHKERRQ(ierr);
1427         for (s = 0, q = 0; s < supportSize; ++s) {
1428           supportNew[q++] = DMPlexShiftPoint_Internal(support[s], depth, depthShift) /*support[s] + depthOffset[dep+1]*/;
1429           ierr = PetscFindInt(support[s], numSplitPoints[dep+1], splitPoints[dep+1], &e);CHKERRQ(ierr);
1430           if (e >= 0) {
1431             supportNew[q++] = e + pMaxNew[dep+1];
1432           }
1433         }
1434         supportNew[q++] = hybedge;
1435         supportNew[q++] = hybedge;
1436         if (q != supportSizeNew) SETERRQ3(comm, PETSC_ERR_ARG_WRONG, "Support size %d != %d for vertex %d", q, supportSizeNew, newp);
1437         ierr = DMPlexSetSupport(sdm, newp, supportNew);CHKERRQ(ierr);
1438         /* Hybrid edge */
1439         coneNew[0] = newp;
1440         coneNew[1] = newp;
1441         ierr = DMPlexSetCone(sdm, hybedge, coneNew);CHKERRQ(ierr);
1442         for (e = 0, qf = 0; e < supportSize; ++e) {
1443           PetscInt val, edge;
1444 
1445           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
1446           if (val == 1) {
1447             ierr = PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &edge);CHKERRQ(ierr);
1448             if (edge < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Edge %d is not a split edge", support[e]);
1449             supportNew[qf++] = edge + pMaxNew[dep+2] + numSplitPoints[dep+2];
1450           } else if  (val ==  (shift2 + 1)) {
1451             ierr = PetscFindInt(support[e], numUnsplitPoints[dep+1], unsplitPoints[dep+1], &edge);CHKERRQ(ierr);
1452             if (edge < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Edge %d is not a unsplit edge", support[e]);
1453             supportNew[qf++] = edge + pMaxNew[dep+2] + numSplitPoints[dep+2] + numSplitPoints[dep+1];
1454           }
1455         }
1456         ierr = DMPlexSetSupport(sdm, hybedge, supportNew);CHKERRQ(ierr);
1457       } else if (dep == dim-2) {
1458         const PetscInt hybface = p + pMaxNew[dep+1] + numSplitPoints[dep+1] + numSplitPoints[dep];
1459 
1460         /* Unsplit edge: Faces into original edge, split face, and hybrid face twice */
1461         for (f = 0, qf = 0; f < supportSize; ++f) {
1462           PetscInt val, face;
1463 
1464           ierr = DMLabelGetValue(label, support[f], &val);CHKERRQ(ierr);
1465           if (val == dim-1) {
1466             ierr = PetscFindInt(support[f], numSplitPoints[dep+1], splitPoints[dep+1], &face);CHKERRQ(ierr);
1467             if (face < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Face %d is not a split face", support[f]);
1468             supportNew[qf++] = DMPlexShiftPoint_Internal(support[f], depth, depthShift) /*support[f] + depthOffset[dep+1]*/;
1469             supportNew[qf++] = face + pMaxNew[dep+1];
1470           } else {
1471             supportNew[qf++] = DMPlexShiftPoint_Internal(support[f], depth, depthShift) /*support[f] + depthOffset[dep+1]*/;
1472           }
1473         }
1474         supportNew[qf++] = hybface;
1475         supportNew[qf++] = hybface;
1476         ierr = DMPlexGetSupportSize(sdm, newp, &supportSizeNew);CHKERRQ(ierr);
1477         if (qf != supportSizeNew) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Support size for unsplit edge %d is %d != %d\n", newp, qf, supportSizeNew);
1478         ierr = DMPlexSetSupport(sdm, newp, supportNew);CHKERRQ(ierr);
1479         /* Add hybrid face */
1480         coneNew[0] = newp;
1481         coneNew[1] = newp;
1482         ierr = PetscFindInt(cone[0], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &v);CHKERRQ(ierr);
1483         if (v < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Vertex %d is not an unsplit vertex", cone[0]);
1484         coneNew[2] = v + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep-1];
1485         ierr = PetscFindInt(cone[1], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &v);CHKERRQ(ierr);
1486         if (v < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Vertex %d is not an unsplit vertex", cone[1]);
1487         coneNew[3] = v + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep-1];
1488         ierr = DMPlexSetCone(sdm, hybface, coneNew);CHKERRQ(ierr);
1489         for (f = 0, qf = 0; f < supportSize; ++f) {
1490           PetscInt val, face;
1491 
1492           ierr = DMLabelGetValue(label, support[f], &val);CHKERRQ(ierr);
1493           if (val == dim-1) {
1494             ierr = PetscFindInt(support[f], numSplitPoints[dep+1], splitPoints[dep+1], &face);CHKERRQ(ierr);
1495             supportNew[qf++] = face + pMaxNew[dep+2] + numSplitPoints[dep+2];
1496           }
1497         }
1498         ierr = DMPlexGetSupportSize(sdm, hybface, &supportSizeNew);CHKERRQ(ierr);
1499         if (qf != supportSizeNew) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Support size for hybrid face %d is %d != %d\n", hybface, qf, supportSizeNew);
1500         ierr = DMPlexSetSupport(sdm, hybface, supportNew);CHKERRQ(ierr);
1501       }
1502     }
1503   }
1504   /* Step 6b: Replace split points in negative side cones */
1505   for (sp = 0; sp < numSP; ++sp) {
1506     PetscInt        dep = values[sp];
1507     IS              pIS;
1508     PetscInt        numPoints;
1509     const PetscInt *points;
1510 
1511     if (dep >= 0) continue;
1512     ierr = DMLabelGetStratumIS(label, dep, &pIS);CHKERRQ(ierr);
1513     if (!pIS) continue;
1514     dep  = -dep - shift;
1515     ierr = ISGetLocalSize(pIS, &numPoints);CHKERRQ(ierr);
1516     ierr = ISGetIndices(pIS, &points);CHKERRQ(ierr);
1517     for (p = 0; p < numPoints; ++p) {
1518       const PetscInt  oldp = points[p];
1519       const PetscInt  newp = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*depthOffset[dep] + oldp*/;
1520       const PetscInt *cone;
1521       PetscInt        coneSize, c;
1522       /* PetscBool       replaced = PETSC_FALSE; */
1523 
1524       /* Negative edge: replace split vertex */
1525       /* Negative cell: replace split face */
1526       ierr = DMPlexGetConeSize(sdm, newp, &coneSize);CHKERRQ(ierr);
1527       ierr = DMPlexGetCone(sdm, newp, &cone);CHKERRQ(ierr);
1528       for (c = 0; c < coneSize; ++c) {
1529         const PetscInt coldp = DMPlexShiftPointInverse_Internal(cone[c],depth,depthShift);
1530         PetscInt       csplitp, cp, val;
1531 
1532         ierr = DMLabelGetValue(label, coldp, &val);CHKERRQ(ierr);
1533         if (val == dep-1) {
1534           ierr = PetscFindInt(coldp, numSplitPoints[dep-1], splitPoints[dep-1], &cp);CHKERRQ(ierr);
1535           if (cp < 0) SETERRQ2(comm, PETSC_ERR_ARG_WRONG, "Point %d is not a split point of dimension %d", oldp, dep-1);
1536           csplitp  = pMaxNew[dep-1] + cp;
1537           ierr     = DMPlexInsertCone(sdm, newp, c, csplitp);CHKERRQ(ierr);
1538           /* replaced = PETSC_TRUE; */
1539         }
1540       }
1541       /* Cells with only a vertex or edge on the submesh have no replacement */
1542       /* if (!replaced) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "The cone of point %d does not contain split points", oldp); */
1543     }
1544     ierr = ISRestoreIndices(pIS, &points);CHKERRQ(ierr);
1545     ierr = ISDestroy(&pIS);CHKERRQ(ierr);
1546   }
1547   /* Step 7: Coordinates */
1548   ierr = DMPlexShiftCoordinates_Internal(dm, depthShift, sdm);CHKERRQ(ierr);
1549   ierr = DMGetCoordinateSection(sdm, &coordSection);CHKERRQ(ierr);
1550   ierr = DMGetCoordinatesLocal(sdm, &coordinates);CHKERRQ(ierr);
1551   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
1552   for (v = 0; v < (numSplitPoints ? numSplitPoints[0] : 0); ++v) {
1553     const PetscInt newp   = DMPlexShiftPoint_Internal(splitPoints[0][v], depth, depthShift) /*depthOffset[0] + splitPoints[0][v]*/;
1554     const PetscInt splitp = pMaxNew[0] + v;
1555     PetscInt       dof, off, soff, d;
1556 
1557     ierr = PetscSectionGetDof(coordSection, newp, &dof);CHKERRQ(ierr);
1558     ierr = PetscSectionGetOffset(coordSection, newp, &off);CHKERRQ(ierr);
1559     ierr = PetscSectionGetOffset(coordSection, splitp, &soff);CHKERRQ(ierr);
1560     for (d = 0; d < dof; ++d) coords[soff+d] = coords[off+d];
1561   }
1562   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
1563   /* Step 8: SF, if I can figure this out we can split the mesh in parallel */
1564   ierr = DMPlexShiftSF_Internal(dm, depthShift, sdm);CHKERRQ(ierr);
1565   /* Step 9: Labels */
1566   ierr = DMPlexShiftLabels_Internal(dm, depthShift, sdm);CHKERRQ(ierr);
1567   ierr = DMGetNumLabels(sdm, &numLabels);CHKERRQ(ierr);
1568   for (dep = 0; dep <= depth; ++dep) {
1569     for (p = 0; p < numSplitPoints[dep]; ++p) {
1570       const PetscInt newp   = DMPlexShiftPoint_Internal(splitPoints[dep][p], depth, depthShift) /*depthOffset[dep] + splitPoints[dep][p]*/;
1571       const PetscInt splitp = pMaxNew[dep] + p;
1572       PetscInt       l;
1573 
1574       if (splitLabel) {
1575         const PetscInt val = 100 + dep;
1576 
1577         ierr = DMLabelSetValue(splitLabel, newp,    val);CHKERRQ(ierr);
1578         ierr = DMLabelSetValue(splitLabel, splitp, -val);CHKERRQ(ierr);
1579       }
1580       for (l = 0; l < numLabels; ++l) {
1581         DMLabel     mlabel;
1582         const char *lname;
1583         PetscInt    val;
1584         PetscBool   isDepth;
1585 
1586         ierr = DMGetLabelName(sdm, l, &lname);CHKERRQ(ierr);
1587         ierr = PetscStrcmp(lname, "depth", &isDepth);CHKERRQ(ierr);
1588         if (isDepth) continue;
1589         ierr = DMGetLabel(sdm, lname, &mlabel);CHKERRQ(ierr);
1590         ierr = DMLabelGetValue(mlabel, newp, &val);CHKERRQ(ierr);
1591         if (val >= 0) {
1592           ierr = DMLabelSetValue(mlabel, splitp, val);CHKERRQ(ierr);
1593         }
1594       }
1595     }
1596   }
1597   for (sp = 0; sp < numSP; ++sp) {
1598     const PetscInt dep = values[sp];
1599 
1600     if ((dep < 0) || (dep > depth)) continue;
1601     if (splitIS[dep]) {ierr = ISRestoreIndices(splitIS[dep], &splitPoints[dep]);CHKERRQ(ierr);}
1602     ierr = ISDestroy(&splitIS[dep]);CHKERRQ(ierr);
1603     if (unsplitIS[dep]) {ierr = ISRestoreIndices(unsplitIS[dep], &unsplitPoints[dep]);CHKERRQ(ierr);}
1604     ierr = ISDestroy(&unsplitIS[dep]);CHKERRQ(ierr);
1605   }
1606   if (label) {
1607     ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
1608     ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
1609   }
1610   for (d = 0; d <= depth; ++d) {
1611     ierr = DMPlexGetDepthStratum(sdm, d, NULL, &pEnd);CHKERRQ(ierr);
1612     pMaxNew[d] = pEnd - numHybridPoints[d] - numHybridPointsOld[d];
1613   }
1614   ierr = PetscFree3(coneNew, coneONew, supportNew);CHKERRQ(ierr);
1615   ierr = PetscFree5(depthMax, depthEnd, depthShift, pMaxNew, numHybridPointsOld);CHKERRQ(ierr);
1616   ierr = PetscFree7(splitIS, unsplitIS, numSplitPoints, numUnsplitPoints, numHybridPoints, splitPoints, unsplitPoints);CHKERRQ(ierr);
1617   PetscFunctionReturn(0);
1618 }
1619 
1620 /*@C
1621   DMPlexConstructCohesiveCells - Construct cohesive cells which split the face along an internal interface
1622 
1623   Collective on dm
1624 
1625   Input Parameters:
1626 + dm - The original DM
1627 - label - The label specifying the boundary faces (this could be auto-generated)
1628 
1629   Output Parameters:
1630 + splitLabel - The label containing the split points, or NULL if no output is desired
1631 - dmSplit - The new DM
1632 
1633   Level: developer
1634 
1635 .seealso: DMCreate(), DMPlexLabelCohesiveComplete()
1636 @*/
1637 PetscErrorCode DMPlexConstructCohesiveCells(DM dm, DMLabel label, DMLabel splitLabel, DM *dmSplit)
1638 {
1639   DM             sdm;
1640   PetscInt       dim;
1641   PetscErrorCode ierr;
1642 
1643   PetscFunctionBegin;
1644   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1645   PetscValidPointer(dmSplit, 4);
1646   ierr = DMCreate(PetscObjectComm((PetscObject)dm), &sdm);CHKERRQ(ierr);
1647   ierr = DMSetType(sdm, DMPLEX);CHKERRQ(ierr);
1648   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
1649   ierr = DMSetDimension(sdm, dim);CHKERRQ(ierr);
1650   switch (dim) {
1651   case 2:
1652   case 3:
1653     ierr = DMPlexConstructCohesiveCells_Internal(dm, label, splitLabel, sdm);CHKERRQ(ierr);
1654     break;
1655   default:
1656     SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cannot construct cohesive cells for dimension %d", dim);
1657   }
1658   *dmSplit = sdm;
1659   PetscFunctionReturn(0);
1660 }
1661 
1662 /* Returns the side of the surface for a given cell with a face on the surface */
1663 static PetscErrorCode GetSurfaceSide_Static(DM dm, DM subdm, PetscInt numSubpoints, const PetscInt *subpoints, PetscInt cell, PetscInt face, PetscBool *pos)
1664 {
1665   const PetscInt *cone, *ornt;
1666   PetscInt        dim, coneSize, c;
1667   PetscErrorCode  ierr;
1668 
1669   PetscFunctionBegin;
1670   *pos = PETSC_TRUE;
1671   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
1672   ierr = DMPlexGetConeSize(dm, cell, &coneSize);CHKERRQ(ierr);
1673   ierr = DMPlexGetCone(dm, cell, &cone);CHKERRQ(ierr);
1674   ierr = DMPlexGetConeOrientation(dm, cell, &ornt);CHKERRQ(ierr);
1675   for (c = 0; c < coneSize; ++c) {
1676     if (cone[c] == face) {
1677       PetscInt o = ornt[c];
1678 
1679       if (subdm) {
1680         const PetscInt *subcone, *subornt;
1681         PetscInt        subpoint, subface, subconeSize, sc;
1682 
1683         ierr = PetscFindInt(cell, numSubpoints, subpoints, &subpoint);CHKERRQ(ierr);
1684         ierr = PetscFindInt(face, numSubpoints, subpoints, &subface);CHKERRQ(ierr);
1685         ierr = DMPlexGetConeSize(subdm, subpoint, &subconeSize);CHKERRQ(ierr);
1686         ierr = DMPlexGetCone(subdm, subpoint, &subcone);CHKERRQ(ierr);
1687         ierr = DMPlexGetConeOrientation(subdm, subpoint, &subornt);CHKERRQ(ierr);
1688         for (sc = 0; sc < subconeSize; ++sc) {
1689           if (subcone[sc] == subface) {
1690             o = subornt[0];
1691             break;
1692           }
1693         }
1694         if (sc >= subconeSize) SETERRQ4(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not find subpoint %d (%d) in cone for subpoint %d (%d)", subface, face, subpoint, cell);
1695       }
1696       if (o >= 0) *pos = PETSC_TRUE;
1697       else        *pos = PETSC_FALSE;
1698       break;
1699     }
1700   }
1701   if (c == coneSize) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Cell %d in split face %d support does not have it in the cone", cell, face);
1702   PetscFunctionReturn(0);
1703 }
1704 
1705 /*@
1706   DMPlexLabelCohesiveComplete - Starting with a label marking points on an internal surface, we add all other mesh pieces
1707   to complete the surface
1708 
1709   Input Parameters:
1710 + dm     - The DM
1711 . label  - A DMLabel marking the surface
1712 . blabel - A DMLabel marking the vertices on the boundary which will not be duplicated, or NULL to find them automatically
1713 . flip   - Flag to flip the submesh normal and replace points on the other side
1714 - subdm  - The subDM associated with the label, or NULL
1715 
1716   Output Parameter:
1717 . label - A DMLabel marking all surface points
1718 
1719   Note: The vertices in blabel are called "unsplit" in the terminology from hybrid cell creation.
1720 
1721   Level: developer
1722 
1723 .seealso: DMPlexConstructCohesiveCells(), DMPlexLabelComplete()
1724 @*/
1725 PetscErrorCode DMPlexLabelCohesiveComplete(DM dm, DMLabel label, DMLabel blabel, PetscBool flip, DM subdm)
1726 {
1727   DMLabel         depthLabel;
1728   IS              dimIS, subpointIS = NULL, facePosIS, faceNegIS, crossEdgeIS = NULL;
1729   const PetscInt *points, *subpoints;
1730   const PetscInt  rev   = flip ? -1 : 1;
1731   PetscInt        shift = 100, shift2 = 200, dim, depth, dep, cStart, cEnd, vStart, vEnd, numPoints, numSubpoints, p, val;
1732   PetscErrorCode  ierr;
1733 
1734   PetscFunctionBegin;
1735   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
1736   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
1737   ierr = DMPlexGetDepthLabel(dm, &depthLabel);CHKERRQ(ierr);
1738   if (subdm) {
1739     ierr = DMPlexGetSubpointIS(subdm, &subpointIS);CHKERRQ(ierr);
1740     if (subpointIS) {
1741       ierr = ISGetLocalSize(subpointIS, &numSubpoints);CHKERRQ(ierr);
1742       ierr = ISGetIndices(subpointIS, &subpoints);CHKERRQ(ierr);
1743     }
1744   }
1745   /* Mark cell on the fault, and its faces which touch the fault: cell orientation for face gives the side of the fault */
1746   ierr = DMLabelGetStratumIS(label, dim-1, &dimIS);CHKERRQ(ierr);
1747   if (!dimIS) PetscFunctionReturn(0);
1748   ierr = ISGetLocalSize(dimIS, &numPoints);CHKERRQ(ierr);
1749   ierr = ISGetIndices(dimIS, &points);CHKERRQ(ierr);
1750   for (p = 0; p < numPoints; ++p) { /* Loop over fault faces */
1751     const PetscInt *support;
1752     PetscInt        supportSize, s;
1753 
1754     ierr = DMPlexGetSupportSize(dm, points[p], &supportSize);CHKERRQ(ierr);
1755 #if 0
1756     if (supportSize != 2) {
1757       const PetscInt *lp;
1758       PetscInt        Nlp, pind;
1759 
1760       /* Check that for a cell with a single support face, that face is in the SF */
1761       /*   THis check only works for the remote side. We would need root side information */
1762       ierr = PetscSFGetGraph(dm->sf, NULL, &Nlp, &lp, NULL);CHKERRQ(ierr);
1763       ierr = PetscFindInt(points[p], Nlp, lp, &pind);CHKERRQ(ierr);
1764       if (pind < 0) SETERRQ2(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Split face %d has %d != 2 supports, and the face is not shared with another process", points[p], supportSize);
1765     }
1766 #endif
1767     ierr = DMPlexGetSupport(dm, points[p], &support);CHKERRQ(ierr);
1768     for (s = 0; s < supportSize; ++s) {
1769       const PetscInt *cone;
1770       PetscInt        coneSize, c;
1771       PetscBool       pos;
1772 
1773       ierr = GetSurfaceSide_Static(dm, subdm, numSubpoints, subpoints, support[s], points[p], &pos);CHKERRQ(ierr);
1774       if (pos) {ierr = DMLabelSetValue(label, support[s],  rev*(shift+dim));CHKERRQ(ierr);}
1775       else     {ierr = DMLabelSetValue(label, support[s], -rev*(shift+dim));CHKERRQ(ierr);}
1776       if (rev < 0) pos = !pos ? PETSC_TRUE : PETSC_FALSE;
1777       /* Put faces touching the fault in the label */
1778       ierr = DMPlexGetConeSize(dm, support[s], &coneSize);CHKERRQ(ierr);
1779       ierr = DMPlexGetCone(dm, support[s], &cone);CHKERRQ(ierr);
1780       for (c = 0; c < coneSize; ++c) {
1781         const PetscInt point = cone[c];
1782 
1783         ierr = DMLabelGetValue(label, point, &val);CHKERRQ(ierr);
1784         if (val == -1) {
1785           PetscInt *closure = NULL;
1786           PetscInt  closureSize, cl;
1787 
1788           ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1789           for (cl = 0; cl < closureSize*2; cl += 2) {
1790             const PetscInt clp  = closure[cl];
1791             PetscInt       bval = -1;
1792 
1793             ierr = DMLabelGetValue(label, clp, &val);CHKERRQ(ierr);
1794             if (blabel) {ierr = DMLabelGetValue(blabel, clp, &bval);CHKERRQ(ierr);}
1795             if ((val >= 0) && (val < dim-1) && (bval < 0)) {
1796               ierr = DMLabelSetValue(label, point, pos == PETSC_TRUE ? shift+dim-1 : -(shift+dim-1));CHKERRQ(ierr);
1797               break;
1798             }
1799           }
1800           ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1801         }
1802       }
1803     }
1804   }
1805   ierr = ISRestoreIndices(dimIS, &points);CHKERRQ(ierr);
1806   ierr = ISDestroy(&dimIS);CHKERRQ(ierr);
1807   if (subpointIS) {ierr = ISRestoreIndices(subpointIS, &subpoints);CHKERRQ(ierr);}
1808   /* Mark boundary points as unsplit */
1809   if (blabel) {
1810     ierr = DMLabelGetStratumIS(blabel, 1, &dimIS);CHKERRQ(ierr);
1811     ierr = ISGetLocalSize(dimIS, &numPoints);CHKERRQ(ierr);
1812     ierr = ISGetIndices(dimIS, &points);CHKERRQ(ierr);
1813     for (p = 0; p < numPoints; ++p) {
1814       const PetscInt point = points[p];
1815       PetscInt       val, bval;
1816 
1817       ierr = DMLabelGetValue(blabel, point, &bval);CHKERRQ(ierr);
1818       if (bval >= 0) {
1819         ierr = DMLabelGetValue(label, point, &val);CHKERRQ(ierr);
1820         if ((val < 0) || (val > dim)) {
1821           /* This could be a point added from splitting a vertex on an adjacent fault, otherwise its just wrong */
1822           ierr = DMLabelClearValue(blabel, point, bval);CHKERRQ(ierr);
1823         }
1824       }
1825     }
1826     for (p = 0; p < numPoints; ++p) {
1827       const PetscInt point = points[p];
1828       PetscInt       val, bval;
1829 
1830       ierr = DMLabelGetValue(blabel, point, &bval);CHKERRQ(ierr);
1831       if (bval >= 0) {
1832         const PetscInt *cone,    *support;
1833         PetscInt        coneSize, supportSize, s, valA, valB, valE;
1834 
1835         /* Mark as unsplit */
1836         ierr = DMLabelGetValue(label, point, &val);CHKERRQ(ierr);
1837         if ((val < 0) || (val > dim)) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %d has label value %d, should be part of the fault", point, val);
1838         ierr = DMLabelClearValue(label, point, val);CHKERRQ(ierr);
1839         ierr = DMLabelSetValue(label, point, shift2+val);CHKERRQ(ierr);
1840         /* Check for cross-edge
1841              A cross-edge has endpoints which are both on the boundary of the surface, but the edge itself is not. */
1842         if (val != 0) continue;
1843         ierr = DMPlexGetSupport(dm, point, &support);CHKERRQ(ierr);
1844         ierr = DMPlexGetSupportSize(dm, point, &supportSize);CHKERRQ(ierr);
1845         for (s = 0; s < supportSize; ++s) {
1846           ierr = DMPlexGetCone(dm, support[s], &cone);CHKERRQ(ierr);
1847           ierr = DMPlexGetConeSize(dm, support[s], &coneSize);CHKERRQ(ierr);
1848           if (coneSize != 2) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Edge %D has %D vertices != 2", support[s], coneSize);
1849           ierr = DMLabelGetValue(blabel, cone[0], &valA);CHKERRQ(ierr);
1850           ierr = DMLabelGetValue(blabel, cone[1], &valB);CHKERRQ(ierr);
1851           ierr = DMLabelGetValue(blabel, support[s], &valE);CHKERRQ(ierr);
1852           if ((valE < 0) && (valA >= 0) && (valB >= 0) && (cone[0] != cone[1])) {ierr = DMLabelSetValue(blabel, support[s], 2);CHKERRQ(ierr);}
1853         }
1854       }
1855     }
1856     ierr = ISRestoreIndices(dimIS, &points);CHKERRQ(ierr);
1857     ierr = ISDestroy(&dimIS);CHKERRQ(ierr);
1858   }
1859   /* Search for other cells/faces/edges connected to the fault by a vertex */
1860   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
1861   ierr = DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
1862   ierr = DMLabelGetStratumIS(label, 0, &dimIS);CHKERRQ(ierr);
1863   if (blabel) {ierr = DMLabelGetStratumIS(blabel, 2, &crossEdgeIS);CHKERRQ(ierr);}
1864   if (dimIS && crossEdgeIS) {
1865     IS vertIS = dimIS;
1866 
1867     ierr = ISExpand(vertIS, crossEdgeIS, &dimIS);CHKERRQ(ierr);
1868     ierr = ISDestroy(&crossEdgeIS);CHKERRQ(ierr);
1869     ierr = ISDestroy(&vertIS);CHKERRQ(ierr);
1870   }
1871   if (!dimIS) {
1872     PetscFunctionReturn(0);
1873   }
1874   ierr = ISGetLocalSize(dimIS, &numPoints);CHKERRQ(ierr);
1875   ierr = ISGetIndices(dimIS, &points);CHKERRQ(ierr);
1876   for (p = 0; p < numPoints; ++p) { /* Loop over fault vertices */
1877     PetscInt *star = NULL;
1878     PetscInt  starSize, s;
1879     PetscInt  again = 1;  /* 0: Finished 1: Keep iterating after a change 2: No change */
1880 
1881     /* All points connected to the fault are inside a cell, so at the top level we will only check cells */
1882     ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_FALSE, &starSize, &star);CHKERRQ(ierr);
1883     while (again) {
1884       if (again > 1) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Could not classify all cells connected to the fault");
1885       again = 0;
1886       for (s = 0; s < starSize*2; s += 2) {
1887         const PetscInt  point = star[s];
1888         const PetscInt *cone;
1889         PetscInt        coneSize, c;
1890 
1891         if ((point < cStart) || (point >= cEnd)) continue;
1892         ierr = DMLabelGetValue(label, point, &val);CHKERRQ(ierr);
1893         if (val != -1) continue;
1894         again = again == 1 ? 1 : 2;
1895         ierr  = DMPlexGetConeSize(dm, point, &coneSize);CHKERRQ(ierr);
1896         ierr  = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
1897         for (c = 0; c < coneSize; ++c) {
1898           ierr = DMLabelGetValue(label, cone[c], &val);CHKERRQ(ierr);
1899           if (val != -1) {
1900             const PetscInt *ccone;
1901             PetscInt        cconeSize, cc, side;
1902 
1903             if (PetscAbs(val) < shift) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Face %d on cell %d has an invalid label %d", cone[c], point, val);
1904             if (val > 0) side =  1;
1905             else         side = -1;
1906             ierr = DMLabelSetValue(label, point, side*(shift+dim));CHKERRQ(ierr);
1907             /* Mark cell faces which touch the fault */
1908             ierr = DMPlexGetConeSize(dm, point, &cconeSize);CHKERRQ(ierr);
1909             ierr = DMPlexGetCone(dm, point, &ccone);CHKERRQ(ierr);
1910             for (cc = 0; cc < cconeSize; ++cc) {
1911               PetscInt *closure = NULL;
1912               PetscInt  closureSize, cl;
1913 
1914               ierr = DMLabelGetValue(label, ccone[cc], &val);CHKERRQ(ierr);
1915               if (val != -1) continue;
1916               ierr = DMPlexGetTransitiveClosure(dm, ccone[cc], PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1917               for (cl = 0; cl < closureSize*2; cl += 2) {
1918                 const PetscInt clp = closure[cl];
1919 
1920                 ierr = DMLabelGetValue(label, clp, &val);CHKERRQ(ierr);
1921                 if (val == -1) continue;
1922                 ierr = DMLabelSetValue(label, ccone[cc], side*(shift+dim-1));CHKERRQ(ierr);
1923                 break;
1924               }
1925               ierr = DMPlexRestoreTransitiveClosure(dm, ccone[cc], PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1926             }
1927             again = 1;
1928             break;
1929           }
1930         }
1931       }
1932     }
1933     /* Classify the rest by cell membership */
1934     for (s = 0; s < starSize*2; s += 2) {
1935       const PetscInt point = star[s];
1936 
1937       ierr = DMLabelGetValue(label, point, &val);CHKERRQ(ierr);
1938       if (val == -1) {
1939         PetscInt      *sstar = NULL;
1940         PetscInt       sstarSize, ss;
1941         PetscBool      marked = PETSC_FALSE, isHybrid;
1942 
1943         ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_FALSE, &sstarSize, &sstar);CHKERRQ(ierr);
1944         for (ss = 0; ss < sstarSize*2; ss += 2) {
1945           const PetscInt spoint = sstar[ss];
1946 
1947           if ((spoint < cStart) || (spoint >= cEnd)) continue;
1948           ierr = DMLabelGetValue(label, spoint, &val);CHKERRQ(ierr);
1949           if (val == -1) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Cell %d in star of %d does not have a valid label", spoint, point);
1950           ierr = DMLabelGetValue(depthLabel, point, &dep);CHKERRQ(ierr);
1951           if (val > 0) {
1952             ierr = DMLabelSetValue(label, point,   shift+dep);CHKERRQ(ierr);
1953           } else {
1954             ierr = DMLabelSetValue(label, point, -(shift+dep));CHKERRQ(ierr);
1955           }
1956           marked = PETSC_TRUE;
1957           break;
1958         }
1959         ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_FALSE, &sstarSize, &sstar);CHKERRQ(ierr);
1960         ierr = DMPlexCellIsHybrid_Internal(dm, point, &isHybrid);CHKERRQ(ierr);
1961         if (!isHybrid && !marked) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d could not be classified", point);
1962       }
1963     }
1964     ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_FALSE, &starSize, &star);CHKERRQ(ierr);
1965   }
1966   ierr = ISRestoreIndices(dimIS, &points);CHKERRQ(ierr);
1967   ierr = ISDestroy(&dimIS);CHKERRQ(ierr);
1968   /* If any faces touching the fault divide cells on either side, split them */
1969   ierr = DMLabelGetStratumIS(label,   shift+dim-1,  &facePosIS);CHKERRQ(ierr);
1970   ierr = DMLabelGetStratumIS(label, -(shift+dim-1), &faceNegIS);CHKERRQ(ierr);
1971   ierr = ISExpand(facePosIS, faceNegIS, &dimIS);CHKERRQ(ierr);
1972   ierr = ISDestroy(&facePosIS);CHKERRQ(ierr);
1973   ierr = ISDestroy(&faceNegIS);CHKERRQ(ierr);
1974   ierr = ISGetLocalSize(dimIS, &numPoints);CHKERRQ(ierr);
1975   ierr = ISGetIndices(dimIS, &points);CHKERRQ(ierr);
1976   for (p = 0; p < numPoints; ++p) {
1977     const PetscInt  point = points[p];
1978     const PetscInt *support;
1979     PetscInt        supportSize, valA, valB;
1980 
1981     ierr = DMPlexGetSupportSize(dm, point, &supportSize);CHKERRQ(ierr);
1982     if (supportSize != 2) continue;
1983     ierr = DMPlexGetSupport(dm, point, &support);CHKERRQ(ierr);
1984     ierr = DMLabelGetValue(label, support[0], &valA);CHKERRQ(ierr);
1985     ierr = DMLabelGetValue(label, support[1], &valB);CHKERRQ(ierr);
1986     if ((valA == -1) || (valB == -1)) continue;
1987     if (valA*valB > 0) continue;
1988     /* Split the face */
1989     ierr = DMLabelGetValue(label, point, &valA);CHKERRQ(ierr);
1990     ierr = DMLabelClearValue(label, point, valA);CHKERRQ(ierr);
1991     ierr = DMLabelSetValue(label, point, dim-1);CHKERRQ(ierr);
1992     /* Label its closure:
1993       unmarked: label as unsplit
1994       incident: relabel as split
1995       split:    do nothing
1996     */
1997     {
1998       PetscInt *closure = NULL;
1999       PetscInt  closureSize, cl;
2000 
2001       ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
2002       for (cl = 0; cl < closureSize*2; cl += 2) {
2003         ierr = DMLabelGetValue(label, closure[cl], &valA);CHKERRQ(ierr);
2004         if (valA == -1) { /* Mark as unsplit */
2005           ierr = DMLabelGetValue(depthLabel, closure[cl], &dep);CHKERRQ(ierr);
2006           ierr = DMLabelSetValue(label, closure[cl], shift2+dep);CHKERRQ(ierr);
2007         } else if (((valA >= shift) && (valA < shift2)) || ((valA <= -shift) && (valA > -shift2))) {
2008           ierr = DMLabelGetValue(depthLabel, closure[cl], &dep);CHKERRQ(ierr);
2009           ierr = DMLabelClearValue(label, closure[cl], valA);CHKERRQ(ierr);
2010           ierr = DMLabelSetValue(label, closure[cl], dep);CHKERRQ(ierr);
2011         }
2012       }
2013       ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
2014     }
2015   }
2016   ierr = ISRestoreIndices(dimIS, &points);CHKERRQ(ierr);
2017   ierr = ISDestroy(&dimIS);CHKERRQ(ierr);
2018   PetscFunctionReturn(0);
2019 }
2020 
2021 /* Check that no cell have all vertices on the fault */
2022 PetscErrorCode DMPlexCheckValidSubmesh_Private(DM dm, DMLabel label, DM subdm)
2023 {
2024   IS              subpointIS;
2025   const PetscInt *dmpoints;
2026   PetscInt        defaultValue, cStart, cEnd, c, vStart, vEnd;
2027   PetscErrorCode  ierr;
2028 
2029   PetscFunctionBegin;
2030   if (!label) PetscFunctionReturn(0);
2031   ierr = DMLabelGetDefaultValue(label, &defaultValue);CHKERRQ(ierr);
2032   ierr = DMPlexGetSubpointIS(subdm, &subpointIS);CHKERRQ(ierr);
2033   if (!subpointIS) PetscFunctionReturn(0);
2034   ierr = DMPlexGetHeightStratum(subdm, 0, &cStart, &cEnd);CHKERRQ(ierr);
2035   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
2036   ierr = ISGetIndices(subpointIS, &dmpoints);CHKERRQ(ierr);
2037   for (c = cStart; c < cEnd; ++c) {
2038     PetscBool invalidCell = PETSC_TRUE;
2039     PetscInt *closure     = NULL;
2040     PetscInt  closureSize, cl;
2041 
2042     ierr = DMPlexGetTransitiveClosure(dm, dmpoints[c], PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
2043     for (cl = 0; cl < closureSize*2; cl += 2) {
2044       PetscInt value = 0;
2045 
2046       if ((closure[cl] < vStart) || (closure[cl] >= vEnd)) continue;
2047       ierr = DMLabelGetValue(label, closure[cl], &value);CHKERRQ(ierr);
2048       if (value == defaultValue) {invalidCell = PETSC_FALSE; break;}
2049     }
2050     ierr = DMPlexRestoreTransitiveClosure(dm, dmpoints[c], PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
2051     if (invalidCell) {
2052       ierr = ISRestoreIndices(subpointIS, &dmpoints);CHKERRQ(ierr);
2053       ierr = ISDestroy(&subpointIS);CHKERRQ(ierr);
2054       ierr = DMDestroy(&subdm);CHKERRQ(ierr);
2055       SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Ambiguous submesh. Cell %D has all of its vertices on the submesh.", dmpoints[c]);
2056     }
2057   }
2058   ierr = ISRestoreIndices(subpointIS, &dmpoints);CHKERRQ(ierr);
2059   PetscFunctionReturn(0);
2060 }
2061 
2062 /*@
2063   DMPlexCreateHybridMesh - Create a mesh with hybrid cells along an internal interface
2064 
2065   Collective on dm
2066 
2067   Input Parameters:
2068 + dm - The original DM
2069 . label - The label specifying the interface vertices
2070 - bdlabel - The optional label specifying the interface boundary vertices
2071 
2072   Output Parameters:
2073 + hybridLabel - The label fully marking the interface, or NULL if no output is desired
2074 . splitLabel - The label containing the split points, or NULL if no output is desired
2075 . dmInterface - The new interface DM, or NULL
2076 - dmHybrid - The new DM with cohesive cells
2077 
2078   Note: The hybridLabel indicates what parts of the original mesh impinged on the on division surface. For points
2079   directly on the division surface, they are labeled with their dimension, so an edge 7 on the division surface would be
2080   7 (1) in hybridLabel. For points that impinge from the positive side, they are labeled with 100+dim, so an edge 6 with
2081   one vertex 3 on the surface would be 6 (101) and 3 (0) in hybridLabel. If an edge 9 from the negative side of the
2082   surface also hits vertex 3, it would be 9 (-101) in hybridLabel.
2083 
2084   The splitLabel indicates what points in the new hybrid mesh were the result of splitting points in the original
2085   mesh. The label value is +=100+dim for each point. For example, if two edges 10 and 14 in the hybrid resulting from
2086   splitting an edge in the original mesh, you would have 10 (101) and 14 (-101) in the splitLabel.
2087 
2088   The dmInterface is a DM built from the original division surface. It has a label which can be retrieved using
2089   DMPlexGetSubpointMap() which maps each point back to the point in the surface of the original mesh.
2090 
2091   Level: developer
2092 
2093 .seealso: DMPlexConstructCohesiveCells(), DMPlexLabelCohesiveComplete(), DMPlexGetSubpointMap(), DMCreate()
2094 @*/
2095 PetscErrorCode DMPlexCreateHybridMesh(DM dm, DMLabel label, DMLabel bdlabel, DMLabel *hybridLabel, DMLabel *splitLabel, DM *dmInterface, DM *dmHybrid)
2096 {
2097   DM             idm;
2098   DMLabel        subpointMap, hlabel, slabel = NULL;
2099   PetscInt       dim;
2100   PetscErrorCode ierr;
2101 
2102   PetscFunctionBegin;
2103   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2104   if (bdlabel) PetscValidPointer(bdlabel, 3);
2105   if (hybridLabel) PetscValidPointer(hybridLabel, 4);
2106   if (splitLabel)  PetscValidPointer(splitLabel, 5);
2107   if (dmInterface) PetscValidPointer(dmInterface, 6);
2108   PetscValidPointer(dmHybrid, 7);
2109   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
2110   ierr = DMPlexCreateSubmesh(dm, label, 1, PETSC_FALSE, &idm);CHKERRQ(ierr);
2111   ierr = DMPlexCheckValidSubmesh_Private(dm, label, idm);CHKERRQ(ierr);
2112   ierr = DMPlexOrient(idm);CHKERRQ(ierr);
2113   ierr = DMPlexGetSubpointMap(idm, &subpointMap);CHKERRQ(ierr);
2114   ierr = DMLabelDuplicate(subpointMap, &hlabel);CHKERRQ(ierr);
2115   ierr = DMLabelClearStratum(hlabel, dim);CHKERRQ(ierr);
2116   if (splitLabel) {
2117     const char *name;
2118     char        sname[PETSC_MAX_PATH_LEN];
2119 
2120     ierr = PetscObjectGetName((PetscObject) hlabel, &name);CHKERRQ(ierr);
2121     ierr = PetscStrncpy(sname, name, PETSC_MAX_PATH_LEN);CHKERRQ(ierr);
2122     ierr = PetscStrcat(sname, " split");CHKERRQ(ierr);
2123     ierr = DMLabelCreate(PETSC_COMM_SELF, sname, &slabel);CHKERRQ(ierr);
2124   }
2125   ierr = DMPlexLabelCohesiveComplete(dm, hlabel, bdlabel, PETSC_FALSE, idm);CHKERRQ(ierr);
2126   if (dmInterface) {*dmInterface = idm;}
2127   else             {ierr = DMDestroy(&idm);CHKERRQ(ierr);}
2128   ierr = DMPlexConstructCohesiveCells(dm, hlabel, slabel, dmHybrid);CHKERRQ(ierr);
2129   if (hybridLabel) *hybridLabel = hlabel;
2130   else             {ierr = DMLabelDestroy(&hlabel);CHKERRQ(ierr);}
2131   if (splitLabel)  *splitLabel  = slabel;
2132   {
2133     DM      cdm;
2134     DMLabel ctLabel;
2135 
2136     /* We need to somehow share the celltype label with the coordinate dm */
2137     ierr = DMGetCoordinateDM(*dmHybrid, &cdm);CHKERRQ(ierr);
2138     ierr = DMPlexGetCellTypeLabel(*dmHybrid, &ctLabel);CHKERRQ(ierr);
2139     ierr = DMSetLabel(cdm, ctLabel);CHKERRQ(ierr);
2140   }
2141   PetscFunctionReturn(0);
2142 }
2143 
2144 /* Here we need the explicit assumption that:
2145 
2146      For any marked cell, the marked vertices constitute a single face
2147 */
2148 static PetscErrorCode DMPlexMarkSubmesh_Uninterpolated(DM dm, DMLabel vertexLabel, PetscInt value, DMLabel subpointMap, PetscInt *numFaces, PetscInt *nFV, DM subdm)
2149 {
2150   IS               subvertexIS = NULL;
2151   const PetscInt  *subvertices;
2152   PetscInt        *pStart, *pEnd, pSize;
2153   PetscInt         depth, dim, d, numSubVerticesInitial = 0, v;
2154   PetscErrorCode   ierr;
2155 
2156   PetscFunctionBegin;
2157   *numFaces = 0;
2158   *nFV      = 0;
2159   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
2160   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
2161   pSize = PetscMax(depth, dim) + 1;
2162   ierr = PetscMalloc2(pSize, &pStart, pSize, &pEnd);CHKERRQ(ierr);
2163   for (d = 0; d <= depth; ++d) {
2164     ierr = DMPlexGetSimplexOrBoxCells(dm, depth-d, &pStart[d], &pEnd[d]);CHKERRQ(ierr);
2165   }
2166   /* Loop over initial vertices and mark all faces in the collective star() */
2167   if (vertexLabel) {ierr = DMLabelGetStratumIS(vertexLabel, value, &subvertexIS);CHKERRQ(ierr);}
2168   if (subvertexIS) {
2169     ierr = ISGetSize(subvertexIS, &numSubVerticesInitial);CHKERRQ(ierr);
2170     ierr = ISGetIndices(subvertexIS, &subvertices);CHKERRQ(ierr);
2171   }
2172   for (v = 0; v < numSubVerticesInitial; ++v) {
2173     const PetscInt vertex = subvertices[v];
2174     PetscInt      *star   = NULL;
2175     PetscInt       starSize, s, numCells = 0, c;
2176 
2177     ierr = DMPlexGetTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star);CHKERRQ(ierr);
2178     for (s = 0; s < starSize*2; s += 2) {
2179       const PetscInt point = star[s];
2180       if ((point >= pStart[depth]) && (point < pEnd[depth])) star[numCells++] = point;
2181     }
2182     for (c = 0; c < numCells; ++c) {
2183       const PetscInt cell    = star[c];
2184       PetscInt      *closure = NULL;
2185       PetscInt       closureSize, cl;
2186       PetscInt       cellLoc, numCorners = 0, faceSize = 0;
2187 
2188       ierr = DMLabelGetValue(subpointMap, cell, &cellLoc);CHKERRQ(ierr);
2189       if (cellLoc == 2) continue;
2190       if (cellLoc >= 0) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Cell %d has dimension %d in the surface label", cell, cellLoc);
2191       ierr = DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
2192       for (cl = 0; cl < closureSize*2; cl += 2) {
2193         const PetscInt point = closure[cl];
2194         PetscInt       vertexLoc;
2195 
2196         if ((point >= pStart[0]) && (point < pEnd[0])) {
2197           ++numCorners;
2198           ierr = DMLabelGetValue(vertexLabel, point, &vertexLoc);CHKERRQ(ierr);
2199           if (vertexLoc == value) closure[faceSize++] = point;
2200         }
2201       }
2202       if (!(*nFV)) {ierr = DMPlexGetNumFaceVertices(dm, dim, numCorners, nFV);CHKERRQ(ierr);}
2203       if (faceSize > *nFV) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Invalid submesh: Too many vertices %d of an element on the surface", faceSize);
2204       if (faceSize == *nFV) {
2205         const PetscInt *cells = NULL;
2206         PetscInt        numCells, nc;
2207 
2208         ++(*numFaces);
2209         for (cl = 0; cl < faceSize; ++cl) {
2210           ierr = DMLabelSetValue(subpointMap, closure[cl], 0);CHKERRQ(ierr);
2211         }
2212         ierr = DMPlexGetJoin(dm, faceSize, closure, &numCells, &cells);CHKERRQ(ierr);
2213         for (nc = 0; nc < numCells; ++nc) {
2214           ierr = DMLabelSetValue(subpointMap, cells[nc], 2);CHKERRQ(ierr);
2215         }
2216         ierr = DMPlexRestoreJoin(dm, faceSize, closure, &numCells, &cells);CHKERRQ(ierr);
2217       }
2218       ierr = DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
2219     }
2220     ierr = DMPlexRestoreTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star);CHKERRQ(ierr);
2221   }
2222   if (subvertexIS) {
2223     ierr = ISRestoreIndices(subvertexIS, &subvertices);CHKERRQ(ierr);
2224   }
2225   ierr = ISDestroy(&subvertexIS);CHKERRQ(ierr);
2226   ierr = PetscFree2(pStart, pEnd);CHKERRQ(ierr);
2227   PetscFunctionReturn(0);
2228 }
2229 
2230 static PetscErrorCode DMPlexMarkSubmesh_Interpolated(DM dm, DMLabel vertexLabel, PetscInt value, PetscBool markedFaces, DMLabel subpointMap, DM subdm)
2231 {
2232   IS               subvertexIS = NULL;
2233   const PetscInt  *subvertices;
2234   PetscInt        *pStart, *pEnd;
2235   PetscInt         dim, d, numSubVerticesInitial = 0, v;
2236   PetscErrorCode   ierr;
2237 
2238   PetscFunctionBegin;
2239   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
2240   ierr = PetscMalloc2(dim+1, &pStart, dim+1, &pEnd);CHKERRQ(ierr);
2241   for (d = 0; d <= dim; ++d) {
2242     ierr = DMPlexGetSimplexOrBoxCells(dm, dim-d, &pStart[d], &pEnd[d]);CHKERRQ(ierr);
2243   }
2244   /* Loop over initial vertices and mark all faces in the collective star() */
2245   if (vertexLabel) {
2246     ierr = DMLabelGetStratumIS(vertexLabel, value, &subvertexIS);CHKERRQ(ierr);
2247     if (subvertexIS) {
2248       ierr = ISGetSize(subvertexIS, &numSubVerticesInitial);CHKERRQ(ierr);
2249       ierr = ISGetIndices(subvertexIS, &subvertices);CHKERRQ(ierr);
2250     }
2251   }
2252   for (v = 0; v < numSubVerticesInitial; ++v) {
2253     const PetscInt vertex = subvertices[v];
2254     PetscInt      *star   = NULL;
2255     PetscInt       starSize, s, numFaces = 0, f;
2256 
2257     ierr = DMPlexGetTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star);CHKERRQ(ierr);
2258     for (s = 0; s < starSize*2; s += 2) {
2259       const PetscInt point = star[s];
2260       PetscInt       faceLoc;
2261 
2262       if ((point >= pStart[dim-1]) && (point < pEnd[dim-1])) {
2263         if (markedFaces) {
2264           ierr = DMLabelGetValue(vertexLabel, point, &faceLoc);CHKERRQ(ierr);
2265           if (faceLoc < 0) continue;
2266         }
2267         star[numFaces++] = point;
2268       }
2269     }
2270     for (f = 0; f < numFaces; ++f) {
2271       const PetscInt face    = star[f];
2272       PetscInt      *closure = NULL;
2273       PetscInt       closureSize, c;
2274       PetscInt       faceLoc;
2275 
2276       ierr = DMLabelGetValue(subpointMap, face, &faceLoc);CHKERRQ(ierr);
2277       if (faceLoc == dim-1) continue;
2278       if (faceLoc >= 0) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Face %d has dimension %d in the surface label", face, faceLoc);
2279       ierr = DMPlexGetTransitiveClosure(dm, face, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
2280       for (c = 0; c < closureSize*2; c += 2) {
2281         const PetscInt point = closure[c];
2282         PetscInt       vertexLoc;
2283 
2284         if ((point >= pStart[0]) && (point < pEnd[0])) {
2285           ierr = DMLabelGetValue(vertexLabel, point, &vertexLoc);CHKERRQ(ierr);
2286           if (vertexLoc != value) break;
2287         }
2288       }
2289       if (c == closureSize*2) {
2290         const PetscInt *support;
2291         PetscInt        supportSize, s;
2292 
2293         for (c = 0; c < closureSize*2; c += 2) {
2294           const PetscInt point = closure[c];
2295 
2296           for (d = 0; d < dim; ++d) {
2297             if ((point >= pStart[d]) && (point < pEnd[d])) {
2298               ierr = DMLabelSetValue(subpointMap, point, d);CHKERRQ(ierr);
2299               break;
2300             }
2301           }
2302         }
2303         ierr = DMPlexGetSupportSize(dm, face, &supportSize);CHKERRQ(ierr);
2304         ierr = DMPlexGetSupport(dm, face, &support);CHKERRQ(ierr);
2305         for (s = 0; s < supportSize; ++s) {
2306           ierr = DMLabelSetValue(subpointMap, support[s], dim);CHKERRQ(ierr);
2307         }
2308       }
2309       ierr = DMPlexRestoreTransitiveClosure(dm, face, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
2310     }
2311     ierr = DMPlexRestoreTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star);CHKERRQ(ierr);
2312   }
2313   if (subvertexIS) {ierr = ISRestoreIndices(subvertexIS, &subvertices);CHKERRQ(ierr);}
2314   ierr = ISDestroy(&subvertexIS);CHKERRQ(ierr);
2315   ierr = PetscFree2(pStart, pEnd);CHKERRQ(ierr);
2316   PetscFunctionReturn(0);
2317 }
2318 
2319 static PetscErrorCode DMPlexMarkCohesiveSubmesh_Uninterpolated(DM dm, PetscBool hasLagrange, const char labelname[], PetscInt value, DMLabel subpointMap, PetscInt *numFaces, PetscInt *nFV, PetscInt *subCells[], DM subdm)
2320 {
2321   DMLabel         label = NULL;
2322   const PetscInt *cone;
2323   PetscInt        dim, cMax, cEnd, c, subc = 0, p, coneSize = -1;
2324   PetscErrorCode  ierr;
2325 
2326   PetscFunctionBegin;
2327   *numFaces = 0;
2328   *nFV = 0;
2329   if (labelname) {ierr = DMGetLabel(dm, labelname, &label);CHKERRQ(ierr);}
2330   *subCells = NULL;
2331   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
2332   ierr = DMPlexGetTensorPrismBounds_Internal(dm, dim, &cMax, &cEnd);CHKERRQ(ierr);
2333   if (cMax < 0) PetscFunctionReturn(0);
2334   if (label) {
2335     for (c = cMax; c < cEnd; ++c) {
2336       PetscInt val;
2337 
2338       ierr = DMLabelGetValue(label, c, &val);CHKERRQ(ierr);
2339       if (val == value) {
2340         ++(*numFaces);
2341         ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
2342       }
2343     }
2344   } else {
2345     *numFaces = cEnd - cMax;
2346     ierr = DMPlexGetConeSize(dm, cMax, &coneSize);CHKERRQ(ierr);
2347   }
2348   ierr = PetscMalloc1(*numFaces *2, subCells);CHKERRQ(ierr);
2349   if (!(*numFaces)) PetscFunctionReturn(0);
2350   *nFV = hasLagrange ? coneSize/3 : coneSize/2;
2351   for (c = cMax; c < cEnd; ++c) {
2352     const PetscInt *cells;
2353     PetscInt        numCells;
2354 
2355     if (label) {
2356       PetscInt val;
2357 
2358       ierr = DMLabelGetValue(label, c, &val);CHKERRQ(ierr);
2359       if (val != value) continue;
2360     }
2361     ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
2362     for (p = 0; p < *nFV; ++p) {
2363       ierr = DMLabelSetValue(subpointMap, cone[p], 0);CHKERRQ(ierr);
2364     }
2365     /* Negative face */
2366     ierr = DMPlexGetJoin(dm, *nFV, cone, &numCells, &cells);CHKERRQ(ierr);
2367     /* Not true in parallel
2368     if (numCells != 2) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive cells should separate two cells"); */
2369     for (p = 0; p < numCells; ++p) {
2370       ierr = DMLabelSetValue(subpointMap, cells[p], 2);CHKERRQ(ierr);
2371       (*subCells)[subc++] = cells[p];
2372     }
2373     ierr = DMPlexRestoreJoin(dm, *nFV, cone, &numCells, &cells);CHKERRQ(ierr);
2374     /* Positive face is not included */
2375   }
2376   PetscFunctionReturn(0);
2377 }
2378 
2379 static PetscErrorCode DMPlexMarkCohesiveSubmesh_Interpolated(DM dm, DMLabel label, PetscInt value, DMLabel subpointMap, DM subdm)
2380 {
2381   PetscInt      *pStart, *pEnd;
2382   PetscInt       dim, cMax, cEnd, c, d;
2383   PetscErrorCode ierr;
2384 
2385   PetscFunctionBegin;
2386   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
2387   ierr = DMPlexGetTensorPrismBounds_Internal(dm, dim, &cMax, &cEnd);CHKERRQ(ierr);
2388   if (cMax < 0) PetscFunctionReturn(0);
2389   ierr = PetscMalloc2(dim+1,&pStart,dim+1,&pEnd);CHKERRQ(ierr);
2390   for (d = 0; d <= dim; ++d) {ierr = DMPlexGetDepthStratum(dm, d, &pStart[d], &pEnd[d]);CHKERRQ(ierr);}
2391   for (c = cMax; c < cEnd; ++c) {
2392     const PetscInt *cone;
2393     PetscInt       *closure = NULL;
2394     PetscInt        fconeSize, coneSize, closureSize, cl, val;
2395 
2396     if (label) {
2397       ierr = DMLabelGetValue(label, c, &val);CHKERRQ(ierr);
2398       if (val != value) continue;
2399     }
2400     ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
2401     ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
2402     ierr = DMPlexGetConeSize(dm, cone[0], &fconeSize);CHKERRQ(ierr);
2403     if (coneSize != (fconeSize ? fconeSize : 1) + 2) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive cells should separate two cells");
2404     /* Negative face */
2405     ierr = DMPlexGetTransitiveClosure(dm, cone[0], PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
2406     for (cl = 0; cl < closureSize*2; cl += 2) {
2407       const PetscInt point = closure[cl];
2408 
2409       for (d = 0; d <= dim; ++d) {
2410         if ((point >= pStart[d]) && (point < pEnd[d])) {
2411           ierr = DMLabelSetValue(subpointMap, point, d);CHKERRQ(ierr);
2412           break;
2413         }
2414       }
2415     }
2416     ierr = DMPlexRestoreTransitiveClosure(dm, cone[0], PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
2417     /* Cells -- positive face is not included */
2418     for (cl = 0; cl < 1; ++cl) {
2419       const PetscInt *support;
2420       PetscInt        supportSize, s;
2421 
2422       ierr = DMPlexGetSupportSize(dm, cone[cl], &supportSize);CHKERRQ(ierr);
2423       /* if (supportSize != 2) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive faces should separate two cells"); */
2424       ierr = DMPlexGetSupport(dm, cone[cl], &support);CHKERRQ(ierr);
2425       for (s = 0; s < supportSize; ++s) {
2426         ierr = DMLabelSetValue(subpointMap, support[s], dim);CHKERRQ(ierr);
2427       }
2428     }
2429   }
2430   ierr = PetscFree2(pStart, pEnd);CHKERRQ(ierr);
2431   PetscFunctionReturn(0);
2432 }
2433 
2434 static PetscErrorCode DMPlexGetFaceOrientation(DM dm, PetscInt cell, PetscInt numCorners, PetscInt indices[], PetscInt oppositeVertex, PetscInt origVertices[], PetscInt faceVertices[], PetscBool *posOriented)
2435 {
2436   MPI_Comm       comm;
2437   PetscBool      posOrient = PETSC_FALSE;
2438   const PetscInt debug     = 0;
2439   PetscInt       cellDim, faceSize, f;
2440   PetscErrorCode ierr;
2441 
2442   PetscFunctionBegin;
2443   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
2444   ierr = DMGetDimension(dm, &cellDim);CHKERRQ(ierr);
2445   if (debug) {ierr = PetscPrintf(comm, "cellDim: %d numCorners: %d\n", cellDim, numCorners);CHKERRQ(ierr);}
2446 
2447   if (cellDim == 1 && numCorners == 2) {
2448     /* Triangle */
2449     faceSize  = numCorners-1;
2450     posOrient = !(oppositeVertex%2) ? PETSC_TRUE : PETSC_FALSE;
2451   } else if (cellDim == 2 && numCorners == 3) {
2452     /* Triangle */
2453     faceSize  = numCorners-1;
2454     posOrient = !(oppositeVertex%2) ? PETSC_TRUE : PETSC_FALSE;
2455   } else if (cellDim == 3 && numCorners == 4) {
2456     /* Tetrahedron */
2457     faceSize  = numCorners-1;
2458     posOrient = (oppositeVertex%2) ? PETSC_TRUE : PETSC_FALSE;
2459   } else if (cellDim == 1 && numCorners == 3) {
2460     /* Quadratic line */
2461     faceSize  = 1;
2462     posOrient = PETSC_TRUE;
2463   } else if (cellDim == 2 && numCorners == 4) {
2464     /* Quads */
2465     faceSize = 2;
2466     if ((indices[1] > indices[0]) && (indices[1] - indices[0] == 1)) {
2467       posOrient = PETSC_TRUE;
2468     } else if ((indices[0] == 3) && (indices[1] == 0)) {
2469       posOrient = PETSC_TRUE;
2470     } else {
2471       if (((indices[0] > indices[1]) && (indices[0] - indices[1] == 1)) || ((indices[0] == 0) && (indices[1] == 3))) {
2472         posOrient = PETSC_FALSE;
2473       } else SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid quad crossedge");
2474     }
2475   } else if (cellDim == 2 && numCorners == 6) {
2476     /* Quadratic triangle (I hate this) */
2477     /* Edges are determined by the first 2 vertices (corners of edges) */
2478     const PetscInt faceSizeTri = 3;
2479     PetscInt       sortedIndices[3], i, iFace;
2480     PetscBool      found                    = PETSC_FALSE;
2481     PetscInt       faceVerticesTriSorted[9] = {
2482       0, 3,  4, /* bottom */
2483       1, 4,  5, /* right */
2484       2, 3,  5, /* left */
2485     };
2486     PetscInt       faceVerticesTri[9] = {
2487       0, 3,  4, /* bottom */
2488       1, 4,  5, /* right */
2489       2, 5,  3, /* left */
2490     };
2491 
2492     for (i = 0; i < faceSizeTri; ++i) sortedIndices[i] = indices[i];
2493     ierr = PetscSortInt(faceSizeTri, sortedIndices);CHKERRQ(ierr);
2494     for (iFace = 0; iFace < 3; ++iFace) {
2495       const PetscInt ii = iFace*faceSizeTri;
2496       PetscInt       fVertex, cVertex;
2497 
2498       if ((sortedIndices[0] == faceVerticesTriSorted[ii+0]) &&
2499           (sortedIndices[1] == faceVerticesTriSorted[ii+1])) {
2500         for (fVertex = 0; fVertex < faceSizeTri; ++fVertex) {
2501           for (cVertex = 0; cVertex < faceSizeTri; ++cVertex) {
2502             if (indices[cVertex] == faceVerticesTri[ii+fVertex]) {
2503               faceVertices[fVertex] = origVertices[cVertex];
2504               break;
2505             }
2506           }
2507         }
2508         found = PETSC_TRUE;
2509         break;
2510       }
2511     }
2512     if (!found) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid tri crossface");
2513     if (posOriented) *posOriented = PETSC_TRUE;
2514     PetscFunctionReturn(0);
2515   } else if (cellDim == 2 && numCorners == 9) {
2516     /* Quadratic quad (I hate this) */
2517     /* Edges are determined by the first 2 vertices (corners of edges) */
2518     const PetscInt faceSizeQuad = 3;
2519     PetscInt       sortedIndices[3], i, iFace;
2520     PetscBool      found                      = PETSC_FALSE;
2521     PetscInt       faceVerticesQuadSorted[12] = {
2522       0, 1,  4, /* bottom */
2523       1, 2,  5, /* right */
2524       2, 3,  6, /* top */
2525       0, 3,  7, /* left */
2526     };
2527     PetscInt       faceVerticesQuad[12] = {
2528       0, 1,  4, /* bottom */
2529       1, 2,  5, /* right */
2530       2, 3,  6, /* top */
2531       3, 0,  7, /* left */
2532     };
2533 
2534     for (i = 0; i < faceSizeQuad; ++i) sortedIndices[i] = indices[i];
2535     ierr = PetscSortInt(faceSizeQuad, sortedIndices);CHKERRQ(ierr);
2536     for (iFace = 0; iFace < 4; ++iFace) {
2537       const PetscInt ii = iFace*faceSizeQuad;
2538       PetscInt       fVertex, cVertex;
2539 
2540       if ((sortedIndices[0] == faceVerticesQuadSorted[ii+0]) &&
2541           (sortedIndices[1] == faceVerticesQuadSorted[ii+1])) {
2542         for (fVertex = 0; fVertex < faceSizeQuad; ++fVertex) {
2543           for (cVertex = 0; cVertex < faceSizeQuad; ++cVertex) {
2544             if (indices[cVertex] == faceVerticesQuad[ii+fVertex]) {
2545               faceVertices[fVertex] = origVertices[cVertex];
2546               break;
2547             }
2548           }
2549         }
2550         found = PETSC_TRUE;
2551         break;
2552       }
2553     }
2554     if (!found) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid quad crossface");
2555     if (posOriented) *posOriented = PETSC_TRUE;
2556     PetscFunctionReturn(0);
2557   } else if (cellDim == 3 && numCorners == 8) {
2558     /* Hexes
2559        A hex is two oriented quads with the normal of the first
2560        pointing up at the second.
2561 
2562           7---6
2563          /|  /|
2564         4---5 |
2565         | 1-|-2
2566         |/  |/
2567         0---3
2568 
2569         Faces are determined by the first 4 vertices (corners of faces) */
2570     const PetscInt faceSizeHex = 4;
2571     PetscInt       sortedIndices[4], i, iFace;
2572     PetscBool      found                     = PETSC_FALSE;
2573     PetscInt       faceVerticesHexSorted[24] = {
2574       0, 1, 2, 3,  /* bottom */
2575       4, 5, 6, 7,  /* top */
2576       0, 3, 4, 5,  /* front */
2577       2, 3, 5, 6,  /* right */
2578       1, 2, 6, 7,  /* back */
2579       0, 1, 4, 7,  /* left */
2580     };
2581     PetscInt       faceVerticesHex[24] = {
2582       1, 2, 3, 0,  /* bottom */
2583       4, 5, 6, 7,  /* top */
2584       0, 3, 5, 4,  /* front */
2585       3, 2, 6, 5,  /* right */
2586       2, 1, 7, 6,  /* back */
2587       1, 0, 4, 7,  /* left */
2588     };
2589 
2590     for (i = 0; i < faceSizeHex; ++i) sortedIndices[i] = indices[i];
2591     ierr = PetscSortInt(faceSizeHex, sortedIndices);CHKERRQ(ierr);
2592     for (iFace = 0; iFace < 6; ++iFace) {
2593       const PetscInt ii = iFace*faceSizeHex;
2594       PetscInt       fVertex, cVertex;
2595 
2596       if ((sortedIndices[0] == faceVerticesHexSorted[ii+0]) &&
2597           (sortedIndices[1] == faceVerticesHexSorted[ii+1]) &&
2598           (sortedIndices[2] == faceVerticesHexSorted[ii+2]) &&
2599           (sortedIndices[3] == faceVerticesHexSorted[ii+3])) {
2600         for (fVertex = 0; fVertex < faceSizeHex; ++fVertex) {
2601           for (cVertex = 0; cVertex < faceSizeHex; ++cVertex) {
2602             if (indices[cVertex] == faceVerticesHex[ii+fVertex]) {
2603               faceVertices[fVertex] = origVertices[cVertex];
2604               break;
2605             }
2606           }
2607         }
2608         found = PETSC_TRUE;
2609         break;
2610       }
2611     }
2612     if (!found) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid hex crossface");
2613     if (posOriented) *posOriented = PETSC_TRUE;
2614     PetscFunctionReturn(0);
2615   } else if (cellDim == 3 && numCorners == 10) {
2616     /* Quadratic tet */
2617     /* Faces are determined by the first 3 vertices (corners of faces) */
2618     const PetscInt faceSizeTet = 6;
2619     PetscInt       sortedIndices[6], i, iFace;
2620     PetscBool      found                     = PETSC_FALSE;
2621     PetscInt       faceVerticesTetSorted[24] = {
2622       0, 1, 2,  6, 7, 8, /* bottom */
2623       0, 3, 4,  6, 7, 9,  /* front */
2624       1, 4, 5,  7, 8, 9,  /* right */
2625       2, 3, 5,  6, 8, 9,  /* left */
2626     };
2627     PetscInt       faceVerticesTet[24] = {
2628       0, 1, 2,  6, 7, 8, /* bottom */
2629       0, 4, 3,  6, 7, 9,  /* front */
2630       1, 5, 4,  7, 8, 9,  /* right */
2631       2, 3, 5,  8, 6, 9,  /* left */
2632     };
2633 
2634     for (i = 0; i < faceSizeTet; ++i) sortedIndices[i] = indices[i];
2635     ierr = PetscSortInt(faceSizeTet, sortedIndices);CHKERRQ(ierr);
2636     for (iFace=0; iFace < 4; ++iFace) {
2637       const PetscInt ii = iFace*faceSizeTet;
2638       PetscInt       fVertex, cVertex;
2639 
2640       if ((sortedIndices[0] == faceVerticesTetSorted[ii+0]) &&
2641           (sortedIndices[1] == faceVerticesTetSorted[ii+1]) &&
2642           (sortedIndices[2] == faceVerticesTetSorted[ii+2]) &&
2643           (sortedIndices[3] == faceVerticesTetSorted[ii+3])) {
2644         for (fVertex = 0; fVertex < faceSizeTet; ++fVertex) {
2645           for (cVertex = 0; cVertex < faceSizeTet; ++cVertex) {
2646             if (indices[cVertex] == faceVerticesTet[ii+fVertex]) {
2647               faceVertices[fVertex] = origVertices[cVertex];
2648               break;
2649             }
2650           }
2651         }
2652         found = PETSC_TRUE;
2653         break;
2654       }
2655     }
2656     if (!found) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid tet crossface");
2657     if (posOriented) *posOriented = PETSC_TRUE;
2658     PetscFunctionReturn(0);
2659   } else if (cellDim == 3 && numCorners == 27) {
2660     /* Quadratic hexes (I hate this)
2661        A hex is two oriented quads with the normal of the first
2662        pointing up at the second.
2663 
2664          7---6
2665         /|  /|
2666        4---5 |
2667        | 3-|-2
2668        |/  |/
2669        0---1
2670 
2671        Faces are determined by the first 4 vertices (corners of faces) */
2672     const PetscInt faceSizeQuadHex = 9;
2673     PetscInt       sortedIndices[9], i, iFace;
2674     PetscBool      found                         = PETSC_FALSE;
2675     PetscInt       faceVerticesQuadHexSorted[54] = {
2676       0, 1, 2, 3,  8, 9, 10, 11,  24, /* bottom */
2677       4, 5, 6, 7,  12, 13, 14, 15,  25, /* top */
2678       0, 1, 4, 5,  8, 12, 16, 17,  22, /* front */
2679       1, 2, 5, 6,  9, 13, 17, 18,  21, /* right */
2680       2, 3, 6, 7,  10, 14, 18, 19,  23, /* back */
2681       0, 3, 4, 7,  11, 15, 16, 19,  20, /* left */
2682     };
2683     PetscInt       faceVerticesQuadHex[54] = {
2684       3, 2, 1, 0,  10, 9, 8, 11,  24, /* bottom */
2685       4, 5, 6, 7,  12, 13, 14, 15,  25, /* top */
2686       0, 1, 5, 4,  8, 17, 12, 16,  22, /* front */
2687       1, 2, 6, 5,  9, 18, 13, 17,  21, /* right */
2688       2, 3, 7, 6,  10, 19, 14, 18,  23, /* back */
2689       3, 0, 4, 7,  11, 16, 15, 19,  20 /* left */
2690     };
2691 
2692     for (i = 0; i < faceSizeQuadHex; ++i) sortedIndices[i] = indices[i];
2693     ierr = PetscSortInt(faceSizeQuadHex, sortedIndices);CHKERRQ(ierr);
2694     for (iFace = 0; iFace < 6; ++iFace) {
2695       const PetscInt ii = iFace*faceSizeQuadHex;
2696       PetscInt       fVertex, cVertex;
2697 
2698       if ((sortedIndices[0] == faceVerticesQuadHexSorted[ii+0]) &&
2699           (sortedIndices[1] == faceVerticesQuadHexSorted[ii+1]) &&
2700           (sortedIndices[2] == faceVerticesQuadHexSorted[ii+2]) &&
2701           (sortedIndices[3] == faceVerticesQuadHexSorted[ii+3])) {
2702         for (fVertex = 0; fVertex < faceSizeQuadHex; ++fVertex) {
2703           for (cVertex = 0; cVertex < faceSizeQuadHex; ++cVertex) {
2704             if (indices[cVertex] == faceVerticesQuadHex[ii+fVertex]) {
2705               faceVertices[fVertex] = origVertices[cVertex];
2706               break;
2707             }
2708           }
2709         }
2710         found = PETSC_TRUE;
2711         break;
2712       }
2713     }
2714     if (!found) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid hex crossface");
2715     if (posOriented) *posOriented = PETSC_TRUE;
2716     PetscFunctionReturn(0);
2717   } else SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Unknown cell type for faceOrientation().");
2718   if (!posOrient) {
2719     if (debug) {ierr = PetscPrintf(comm, "  Reversing initial face orientation\n");CHKERRQ(ierr);}
2720     for (f = 0; f < faceSize; ++f) faceVertices[f] = origVertices[faceSize-1 - f];
2721   } else {
2722     if (debug) {ierr = PetscPrintf(comm, "  Keeping initial face orientation\n");CHKERRQ(ierr);}
2723     for (f = 0; f < faceSize; ++f) faceVertices[f] = origVertices[f];
2724   }
2725   if (posOriented) *posOriented = posOrient;
2726   PetscFunctionReturn(0);
2727 }
2728 
2729 /*@
2730   DMPlexGetOrientedFace - Given a cell and a face, as a set of vertices, return the oriented face, as a set of vertices,
2731   in faceVertices. The orientation is such that the face normal points out of the cell
2732 
2733   Not collective
2734 
2735   Input Parameters:
2736 + dm           - The original mesh
2737 . cell         - The cell mesh point
2738 . faceSize     - The number of vertices on the face
2739 . face         - The face vertices
2740 . numCorners   - The number of vertices on the cell
2741 . indices      - Local numbering of face vertices in cell cone
2742 - origVertices - Original face vertices
2743 
2744   Output Parameter:
2745 + faceVertices - The face vertices properly oriented
2746 - posOriented  - PETSC_TRUE if the face was oriented with outward normal
2747 
2748   Level: developer
2749 
2750 .seealso: DMPlexGetCone()
2751 @*/
2752 PetscErrorCode DMPlexGetOrientedFace(DM dm, PetscInt cell, PetscInt faceSize, const PetscInt face[], PetscInt numCorners, PetscInt indices[], PetscInt origVertices[], PetscInt faceVertices[], PetscBool *posOriented)
2753 {
2754   const PetscInt *cone = NULL;
2755   PetscInt        coneSize, v, f, v2;
2756   PetscInt        oppositeVertex = -1;
2757   PetscErrorCode  ierr;
2758 
2759   PetscFunctionBegin;
2760   ierr = DMPlexGetConeSize(dm, cell, &coneSize);CHKERRQ(ierr);
2761   ierr = DMPlexGetCone(dm, cell, &cone);CHKERRQ(ierr);
2762   for (v = 0, v2 = 0; v < coneSize; ++v) {
2763     PetscBool found = PETSC_FALSE;
2764 
2765     for (f = 0; f < faceSize; ++f) {
2766       if (face[f] == cone[v]) {
2767         found = PETSC_TRUE; break;
2768       }
2769     }
2770     if (found) {
2771       indices[v2]      = v;
2772       origVertices[v2] = cone[v];
2773       ++v2;
2774     } else {
2775       oppositeVertex = v;
2776     }
2777   }
2778   ierr = DMPlexGetFaceOrientation(dm, cell, numCorners, indices, oppositeVertex, origVertices, faceVertices, posOriented);CHKERRQ(ierr);
2779   PetscFunctionReturn(0);
2780 }
2781 
2782 /*
2783   DMPlexInsertFace_Internal - Puts a face into the mesh
2784 
2785   Not collective
2786 
2787   Input Parameters:
2788   + dm              - The DMPlex
2789   . numFaceVertex   - The number of vertices in the face
2790   . faceVertices    - The vertices in the face for dm
2791   . subfaceVertices - The vertices in the face for subdm
2792   . numCorners      - The number of vertices in the cell
2793   . cell            - A cell in dm containing the face
2794   . subcell         - A cell in subdm containing the face
2795   . firstFace       - First face in the mesh
2796   - newFacePoint    - Next face in the mesh
2797 
2798   Output Parameters:
2799   . newFacePoint - Contains next face point number on input, updated on output
2800 
2801   Level: developer
2802 */
2803 static PetscErrorCode DMPlexInsertFace_Internal(DM dm, DM subdm, PetscInt numFaceVertices, const PetscInt faceVertices[], const PetscInt subfaceVertices[], PetscInt numCorners, PetscInt cell, PetscInt subcell, PetscInt firstFace, PetscInt *newFacePoint)
2804 {
2805   MPI_Comm        comm;
2806   DM_Plex        *submesh = (DM_Plex*) subdm->data;
2807   const PetscInt *faces;
2808   PetscInt        numFaces, coneSize;
2809   PetscErrorCode  ierr;
2810 
2811   PetscFunctionBegin;
2812   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
2813   ierr = DMPlexGetConeSize(subdm, subcell, &coneSize);CHKERRQ(ierr);
2814   if (coneSize != 1) SETERRQ2(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cone size of cell %d is %d != 1", cell, coneSize);
2815 #if 0
2816   /* Cannot use this because support() has not been constructed yet */
2817   ierr = DMPlexGetJoin(subdm, numFaceVertices, subfaceVertices, &numFaces, &faces);CHKERRQ(ierr);
2818 #else
2819   {
2820     PetscInt f;
2821 
2822     numFaces = 0;
2823     ierr     = DMGetWorkArray(subdm, 1, MPIU_INT, (void **) &faces);CHKERRQ(ierr);
2824     for (f = firstFace; f < *newFacePoint; ++f) {
2825       PetscInt dof, off, d;
2826 
2827       ierr = PetscSectionGetDof(submesh->coneSection, f, &dof);CHKERRQ(ierr);
2828       ierr = PetscSectionGetOffset(submesh->coneSection, f, &off);CHKERRQ(ierr);
2829       /* Yes, I know this is quadratic, but I expect the sizes to be <5 */
2830       for (d = 0; d < dof; ++d) {
2831         const PetscInt p = submesh->cones[off+d];
2832         PetscInt       v;
2833 
2834         for (v = 0; v < numFaceVertices; ++v) {
2835           if (subfaceVertices[v] == p) break;
2836         }
2837         if (v == numFaceVertices) break;
2838       }
2839       if (d == dof) {
2840         numFaces               = 1;
2841         ((PetscInt*) faces)[0] = f;
2842       }
2843     }
2844   }
2845 #endif
2846   if (numFaces > 1) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Vertex set had %d faces, not one", numFaces);
2847   else if (numFaces == 1) {
2848     /* Add the other cell neighbor for this face */
2849     ierr = DMPlexSetCone(subdm, subcell, faces);CHKERRQ(ierr);
2850   } else {
2851     PetscInt *indices, *origVertices, *orientedVertices, *orientedSubVertices, v, ov;
2852     PetscBool posOriented;
2853 
2854     ierr                = DMGetWorkArray(subdm, 4*numFaceVertices * sizeof(PetscInt), MPIU_INT, &orientedVertices);CHKERRQ(ierr);
2855     origVertices        = &orientedVertices[numFaceVertices];
2856     indices             = &orientedVertices[numFaceVertices*2];
2857     orientedSubVertices = &orientedVertices[numFaceVertices*3];
2858     ierr                = DMPlexGetOrientedFace(dm, cell, numFaceVertices, faceVertices, numCorners, indices, origVertices, orientedVertices, &posOriented);CHKERRQ(ierr);
2859     /* TODO: I know that routine should return a permutation, not the indices */
2860     for (v = 0; v < numFaceVertices; ++v) {
2861       const PetscInt vertex = faceVertices[v], subvertex = subfaceVertices[v];
2862       for (ov = 0; ov < numFaceVertices; ++ov) {
2863         if (orientedVertices[ov] == vertex) {
2864           orientedSubVertices[ov] = subvertex;
2865           break;
2866         }
2867       }
2868       if (ov == numFaceVertices) SETERRQ1(comm, PETSC_ERR_PLIB, "Could not find face vertex %d in orientated set", vertex);
2869     }
2870     ierr = DMPlexSetCone(subdm, *newFacePoint, orientedSubVertices);CHKERRQ(ierr);
2871     ierr = DMPlexSetCone(subdm, subcell, newFacePoint);CHKERRQ(ierr);
2872     ierr = DMRestoreWorkArray(subdm, 4*numFaceVertices * sizeof(PetscInt), MPIU_INT, &orientedVertices);CHKERRQ(ierr);
2873     ++(*newFacePoint);
2874   }
2875 #if 0
2876   ierr = DMPlexRestoreJoin(subdm, numFaceVertices, subfaceVertices, &numFaces, &faces);CHKERRQ(ierr);
2877 #else
2878   ierr = DMRestoreWorkArray(subdm, 1, MPIU_INT, (void **) &faces);CHKERRQ(ierr);
2879 #endif
2880   PetscFunctionReturn(0);
2881 }
2882 
2883 static PetscErrorCode DMPlexCreateSubmesh_Uninterpolated(DM dm, DMLabel vertexLabel, PetscInt value, DM subdm)
2884 {
2885   MPI_Comm        comm;
2886   DMLabel         subpointMap;
2887   IS              subvertexIS,  subcellIS;
2888   const PetscInt *subVertices, *subCells;
2889   PetscInt        numSubVertices, firstSubVertex, numSubCells;
2890   PetscInt       *subface, maxConeSize, numSubFaces = 0, firstSubFace, newFacePoint, nFV = 0;
2891   PetscInt        vStart, vEnd, c, f;
2892   PetscErrorCode  ierr;
2893 
2894   PetscFunctionBegin;
2895   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
2896   /* Create subpointMap which marks the submesh */
2897   ierr = DMLabelCreate(PETSC_COMM_SELF, "subpoint_map", &subpointMap);CHKERRQ(ierr);
2898   ierr = DMPlexSetSubpointMap(subdm, subpointMap);CHKERRQ(ierr);
2899   ierr = DMLabelDestroy(&subpointMap);CHKERRQ(ierr);
2900   if (vertexLabel) {ierr = DMPlexMarkSubmesh_Uninterpolated(dm, vertexLabel, value, subpointMap, &numSubFaces, &nFV, subdm);CHKERRQ(ierr);}
2901   /* Setup chart */
2902   ierr = DMLabelGetStratumSize(subpointMap, 0, &numSubVertices);CHKERRQ(ierr);
2903   ierr = DMLabelGetStratumSize(subpointMap, 2, &numSubCells);CHKERRQ(ierr);
2904   ierr = DMPlexSetChart(subdm, 0, numSubCells+numSubFaces+numSubVertices);CHKERRQ(ierr);
2905   ierr = DMPlexSetVTKCellHeight(subdm, 1);CHKERRQ(ierr);
2906   /* Set cone sizes */
2907   firstSubVertex = numSubCells;
2908   firstSubFace   = numSubCells+numSubVertices;
2909   newFacePoint   = firstSubFace;
2910   ierr = DMLabelGetStratumIS(subpointMap, 0, &subvertexIS);CHKERRQ(ierr);
2911   if (subvertexIS) {ierr = ISGetIndices(subvertexIS, &subVertices);CHKERRQ(ierr);}
2912   ierr = DMLabelGetStratumIS(subpointMap, 2, &subcellIS);CHKERRQ(ierr);
2913   if (subcellIS) {ierr = ISGetIndices(subcellIS, &subCells);CHKERRQ(ierr);}
2914   for (c = 0; c < numSubCells; ++c) {
2915     ierr = DMPlexSetConeSize(subdm, c, 1);CHKERRQ(ierr);
2916   }
2917   for (f = firstSubFace; f < firstSubFace+numSubFaces; ++f) {
2918     ierr = DMPlexSetConeSize(subdm, f, nFV);CHKERRQ(ierr);
2919   }
2920   ierr = DMSetUp(subdm);CHKERRQ(ierr);
2921   /* Create face cones */
2922   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
2923   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, NULL);CHKERRQ(ierr);
2924   ierr = DMGetWorkArray(subdm, maxConeSize, MPIU_INT, (void**) &subface);CHKERRQ(ierr);
2925   for (c = 0; c < numSubCells; ++c) {
2926     const PetscInt cell    = subCells[c];
2927     const PetscInt subcell = c;
2928     PetscInt      *closure = NULL;
2929     PetscInt       closureSize, cl, numCorners = 0, faceSize = 0;
2930 
2931     ierr = DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
2932     for (cl = 0; cl < closureSize*2; cl += 2) {
2933       const PetscInt point = closure[cl];
2934       PetscInt       subVertex;
2935 
2936       if ((point >= vStart) && (point < vEnd)) {
2937         ++numCorners;
2938         ierr = PetscFindInt(point, numSubVertices, subVertices, &subVertex);CHKERRQ(ierr);
2939         if (subVertex >= 0) {
2940           closure[faceSize] = point;
2941           subface[faceSize] = firstSubVertex+subVertex;
2942           ++faceSize;
2943         }
2944       }
2945     }
2946     if (faceSize > nFV) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Invalid submesh: Too many vertices %d of an element on the surface", faceSize);
2947     if (faceSize == nFV) {
2948       ierr = DMPlexInsertFace_Internal(dm, subdm, faceSize, closure, subface, numCorners, cell, subcell, firstSubFace, &newFacePoint);CHKERRQ(ierr);
2949     }
2950     ierr = DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
2951   }
2952   ierr = DMRestoreWorkArray(subdm, maxConeSize, MPIU_INT, (void**) &subface);CHKERRQ(ierr);
2953   ierr = DMPlexSymmetrize(subdm);CHKERRQ(ierr);
2954   ierr = DMPlexStratify(subdm);CHKERRQ(ierr);
2955   /* Build coordinates */
2956   {
2957     PetscSection coordSection, subCoordSection;
2958     Vec          coordinates, subCoordinates;
2959     PetscScalar *coords, *subCoords;
2960     PetscInt     numComp, coordSize, v;
2961     const char  *name;
2962 
2963     ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
2964     ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
2965     ierr = DMGetCoordinateSection(subdm, &subCoordSection);CHKERRQ(ierr);
2966     ierr = PetscSectionSetNumFields(subCoordSection, 1);CHKERRQ(ierr);
2967     ierr = PetscSectionGetFieldComponents(coordSection, 0, &numComp);CHKERRQ(ierr);
2968     ierr = PetscSectionSetFieldComponents(subCoordSection, 0, numComp);CHKERRQ(ierr);
2969     ierr = PetscSectionSetChart(subCoordSection, firstSubVertex, firstSubVertex+numSubVertices);CHKERRQ(ierr);
2970     for (v = 0; v < numSubVertices; ++v) {
2971       const PetscInt vertex    = subVertices[v];
2972       const PetscInt subvertex = firstSubVertex+v;
2973       PetscInt       dof;
2974 
2975       ierr = PetscSectionGetDof(coordSection, vertex, &dof);CHKERRQ(ierr);
2976       ierr = PetscSectionSetDof(subCoordSection, subvertex, dof);CHKERRQ(ierr);
2977       ierr = PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof);CHKERRQ(ierr);
2978     }
2979     ierr = PetscSectionSetUp(subCoordSection);CHKERRQ(ierr);
2980     ierr = PetscSectionGetStorageSize(subCoordSection, &coordSize);CHKERRQ(ierr);
2981     ierr = VecCreate(PETSC_COMM_SELF, &subCoordinates);CHKERRQ(ierr);
2982     ierr = PetscObjectGetName((PetscObject)coordinates,&name);CHKERRQ(ierr);
2983     ierr = PetscObjectSetName((PetscObject)subCoordinates,name);CHKERRQ(ierr);
2984     ierr = VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
2985     ierr = VecSetType(subCoordinates,VECSTANDARD);CHKERRQ(ierr);
2986     if (coordSize) {
2987       ierr = VecGetArray(coordinates,    &coords);CHKERRQ(ierr);
2988       ierr = VecGetArray(subCoordinates, &subCoords);CHKERRQ(ierr);
2989       for (v = 0; v < numSubVertices; ++v) {
2990         const PetscInt vertex    = subVertices[v];
2991         const PetscInt subvertex = firstSubVertex+v;
2992         PetscInt       dof, off, sdof, soff, d;
2993 
2994         ierr = PetscSectionGetDof(coordSection, vertex, &dof);CHKERRQ(ierr);
2995         ierr = PetscSectionGetOffset(coordSection, vertex, &off);CHKERRQ(ierr);
2996         ierr = PetscSectionGetDof(subCoordSection, subvertex, &sdof);CHKERRQ(ierr);
2997         ierr = PetscSectionGetOffset(subCoordSection, subvertex, &soff);CHKERRQ(ierr);
2998         if (dof != sdof) SETERRQ4(comm, PETSC_ERR_PLIB, "Coordinate dimension %d on subvertex %d, vertex %d should be %d", sdof, subvertex, vertex, dof);
2999         for (d = 0; d < dof; ++d) subCoords[soff+d] = coords[off+d];
3000       }
3001       ierr = VecRestoreArray(coordinates,    &coords);CHKERRQ(ierr);
3002       ierr = VecRestoreArray(subCoordinates, &subCoords);CHKERRQ(ierr);
3003     }
3004     ierr = DMSetCoordinatesLocal(subdm, subCoordinates);CHKERRQ(ierr);
3005     ierr = VecDestroy(&subCoordinates);CHKERRQ(ierr);
3006   }
3007   /* Cleanup */
3008   if (subvertexIS) {ierr = ISRestoreIndices(subvertexIS, &subVertices);CHKERRQ(ierr);}
3009   ierr = ISDestroy(&subvertexIS);CHKERRQ(ierr);
3010   if (subcellIS) {ierr = ISRestoreIndices(subcellIS, &subCells);CHKERRQ(ierr);}
3011   ierr = ISDestroy(&subcellIS);CHKERRQ(ierr);
3012   PetscFunctionReturn(0);
3013 }
3014 
3015 PETSC_STATIC_INLINE PetscInt DMPlexFilterPoint_Internal(PetscInt point, PetscInt firstSubPoint, PetscInt numSubPoints, const PetscInt subPoints[])
3016 {
3017   PetscInt       subPoint;
3018   PetscErrorCode ierr;
3019 
3020   ierr = PetscFindInt(point, numSubPoints, subPoints, &subPoint); if (ierr < 0) return ierr;
3021   return subPoint < 0 ? subPoint : firstSubPoint+subPoint;
3022 }
3023 
3024 static PetscErrorCode DMPlexCreateSubmeshGeneric_Interpolated(DM dm, DMLabel label, PetscInt value, PetscBool markedFaces, PetscBool isCohesive, PetscInt cellHeight, DM subdm)
3025 {
3026   MPI_Comm         comm;
3027   DMLabel          subpointMap;
3028   IS              *subpointIS;
3029   const PetscInt **subpoints;
3030   PetscInt        *numSubPoints, *firstSubPoint, *coneNew, *orntNew;
3031   PetscInt         totSubPoints = 0, maxConeSize, dim, p, d, v;
3032   PetscMPIInt      rank;
3033   PetscErrorCode   ierr;
3034 
3035   PetscFunctionBegin;
3036   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
3037   ierr = MPI_Comm_rank(comm, &rank);CHKERRMPI(ierr);
3038   /* Create subpointMap which marks the submesh */
3039   ierr = DMLabelCreate(PETSC_COMM_SELF, "subpoint_map", &subpointMap);CHKERRQ(ierr);
3040   ierr = DMPlexSetSubpointMap(subdm, subpointMap);CHKERRQ(ierr);
3041   if (cellHeight) {
3042     if (isCohesive) {ierr = DMPlexMarkCohesiveSubmesh_Interpolated(dm, label, value, subpointMap, subdm);CHKERRQ(ierr);}
3043     else            {ierr = DMPlexMarkSubmesh_Interpolated(dm, label, value, markedFaces, subpointMap, subdm);CHKERRQ(ierr);}
3044   } else {
3045     DMLabel         depth;
3046     IS              pointIS;
3047     const PetscInt *points;
3048     PetscInt        numPoints=0;
3049 
3050     ierr = DMPlexGetDepthLabel(dm, &depth);CHKERRQ(ierr);
3051     ierr = DMLabelGetStratumIS(label, value, &pointIS);CHKERRQ(ierr);
3052     if (pointIS) {
3053       ierr = ISGetIndices(pointIS, &points);CHKERRQ(ierr);
3054       ierr = ISGetLocalSize(pointIS, &numPoints);CHKERRQ(ierr);
3055     }
3056     for (p = 0; p < numPoints; ++p) {
3057       PetscInt *closure = NULL;
3058       PetscInt  closureSize, c, pdim;
3059 
3060       ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
3061       for (c = 0; c < closureSize*2; c += 2) {
3062         ierr = DMLabelGetValue(depth, closure[c], &pdim);CHKERRQ(ierr);
3063         ierr = DMLabelSetValue(subpointMap, closure[c], pdim);CHKERRQ(ierr);
3064       }
3065       ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
3066     }
3067     if (pointIS) {ierr = ISRestoreIndices(pointIS, &points);CHKERRQ(ierr);}
3068     ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
3069   }
3070   /* Setup chart */
3071   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
3072   ierr = PetscMalloc4(dim+1,&numSubPoints,dim+1,&firstSubPoint,dim+1,&subpointIS,dim+1,&subpoints);CHKERRQ(ierr);
3073   for (d = 0; d <= dim; ++d) {
3074     ierr = DMLabelGetStratumSize(subpointMap, d, &numSubPoints[d]);CHKERRQ(ierr);
3075     totSubPoints += numSubPoints[d];
3076   }
3077   ierr = DMPlexSetChart(subdm, 0, totSubPoints);CHKERRQ(ierr);
3078   ierr = DMPlexSetVTKCellHeight(subdm, cellHeight);CHKERRQ(ierr);
3079   /* Set cone sizes */
3080   firstSubPoint[dim] = 0;
3081   firstSubPoint[0]   = firstSubPoint[dim] + numSubPoints[dim];
3082   if (dim > 1) {firstSubPoint[dim-1] = firstSubPoint[0]     + numSubPoints[0];}
3083   if (dim > 2) {firstSubPoint[dim-2] = firstSubPoint[dim-1] + numSubPoints[dim-1];}
3084   for (d = 0; d <= dim; ++d) {
3085     ierr = DMLabelGetStratumIS(subpointMap, d, &subpointIS[d]);CHKERRQ(ierr);
3086     if (subpointIS[d]) {ierr = ISGetIndices(subpointIS[d], &subpoints[d]);CHKERRQ(ierr);}
3087   }
3088   /* We do not want this label automatically computed, instead we compute it here */
3089   ierr = DMCreateLabel(subdm, "celltype");CHKERRQ(ierr);
3090   for (d = 0; d <= dim; ++d) {
3091     for (p = 0; p < numSubPoints[d]; ++p) {
3092       const PetscInt  point    = subpoints[d][p];
3093       const PetscInt  subpoint = firstSubPoint[d] + p;
3094       const PetscInt *cone;
3095       PetscInt        coneSize, coneSizeNew, c, val;
3096       DMPolytopeType  ct;
3097 
3098       ierr = DMPlexGetConeSize(dm, point, &coneSize);CHKERRQ(ierr);
3099       ierr = DMPlexSetConeSize(subdm, subpoint, coneSize);CHKERRQ(ierr);
3100       ierr = DMPlexGetCellType(dm, point, &ct);CHKERRQ(ierr);
3101       ierr = DMPlexSetCellType(subdm, subpoint, ct);CHKERRQ(ierr);
3102       if (cellHeight && (d == dim)) {
3103         ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
3104         for (c = 0, coneSizeNew = 0; c < coneSize; ++c) {
3105           ierr = DMLabelGetValue(subpointMap, cone[c], &val);CHKERRQ(ierr);
3106           if (val >= 0) coneSizeNew++;
3107         }
3108         ierr = DMPlexSetConeSize(subdm, subpoint, coneSizeNew);CHKERRQ(ierr);
3109         ierr = DMPlexSetCellType(subdm, subpoint, DM_POLYTOPE_FV_GHOST);CHKERRQ(ierr);
3110       }
3111     }
3112   }
3113   ierr = DMLabelDestroy(&subpointMap);CHKERRQ(ierr);
3114   ierr = DMSetUp(subdm);CHKERRQ(ierr);
3115   /* Set cones */
3116   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, NULL);CHKERRQ(ierr);
3117   ierr = PetscMalloc2(maxConeSize,&coneNew,maxConeSize,&orntNew);CHKERRQ(ierr);
3118   for (d = 0; d <= dim; ++d) {
3119     for (p = 0; p < numSubPoints[d]; ++p) {
3120       const PetscInt  point    = subpoints[d][p];
3121       const PetscInt  subpoint = firstSubPoint[d] + p;
3122       const PetscInt *cone, *ornt;
3123       PetscInt        coneSize, subconeSize, coneSizeNew, c, subc, fornt = 0;
3124 
3125       if (d == dim-1) {
3126         const PetscInt *support, *cone, *ornt;
3127         PetscInt        supportSize, coneSize, s, subc;
3128 
3129         ierr = DMPlexGetSupport(dm, point, &support);CHKERRQ(ierr);
3130         ierr = DMPlexGetSupportSize(dm, point, &supportSize);CHKERRQ(ierr);
3131         for (s = 0; s < supportSize; ++s) {
3132           PetscBool isHybrid;
3133 
3134           ierr = DMPlexCellIsHybrid_Internal(dm, support[s], &isHybrid);CHKERRQ(ierr);
3135           if (!isHybrid) continue;
3136           ierr = PetscFindInt(support[s], numSubPoints[d+1], subpoints[d+1], &subc);CHKERRQ(ierr);
3137           if (subc >= 0) {
3138             const PetscInt ccell = subpoints[d+1][subc];
3139 
3140             ierr = DMPlexGetCone(dm, ccell, &cone);CHKERRQ(ierr);
3141             ierr = DMPlexGetConeSize(dm, ccell, &coneSize);CHKERRQ(ierr);
3142             ierr = DMPlexGetConeOrientation(dm, ccell, &ornt);CHKERRQ(ierr);
3143             for (c = 0; c < coneSize; ++c) {
3144               if (cone[c] == point) {
3145                 fornt = ornt[c];
3146                 break;
3147               }
3148             }
3149             break;
3150           }
3151         }
3152       }
3153       ierr = DMPlexGetConeSize(dm, point, &coneSize);CHKERRQ(ierr);
3154       ierr = DMPlexGetConeSize(subdm, subpoint, &subconeSize);CHKERRQ(ierr);
3155       ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
3156       ierr = DMPlexGetConeOrientation(dm, point, &ornt);CHKERRQ(ierr);
3157       for (c = 0, coneSizeNew = 0; c < coneSize; ++c) {
3158         ierr = PetscFindInt(cone[c], numSubPoints[d-1], subpoints[d-1], &subc);CHKERRQ(ierr);
3159         if (subc >= 0) {
3160           coneNew[coneSizeNew] = firstSubPoint[d-1] + subc;
3161           orntNew[coneSizeNew] = ornt[c];
3162           ++coneSizeNew;
3163         }
3164       }
3165       if (coneSizeNew != subconeSize) SETERRQ2(comm, PETSC_ERR_PLIB, "Number of cone points located %d does not match subcone size %d", coneSizeNew, subconeSize);
3166       ierr = DMPlexSetCone(subdm, subpoint, coneNew);CHKERRQ(ierr);
3167       ierr = DMPlexSetConeOrientation(subdm, subpoint, orntNew);CHKERRQ(ierr);
3168       if (fornt < 0) {ierr = DMPlexOrientPoint(subdm, subpoint, fornt);CHKERRQ(ierr);}
3169     }
3170   }
3171   ierr = PetscFree2(coneNew,orntNew);CHKERRQ(ierr);
3172   ierr = DMPlexSymmetrize(subdm);CHKERRQ(ierr);
3173   ierr = DMPlexStratify(subdm);CHKERRQ(ierr);
3174   /* Build coordinates */
3175   {
3176     PetscSection coordSection, subCoordSection;
3177     Vec          coordinates, subCoordinates;
3178     PetscScalar *coords, *subCoords;
3179     PetscInt     cdim, numComp, coordSize;
3180     const char  *name;
3181 
3182     ierr = DMGetCoordinateDim(dm, &cdim);CHKERRQ(ierr);
3183     ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
3184     ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
3185     ierr = DMGetCoordinateSection(subdm, &subCoordSection);CHKERRQ(ierr);
3186     ierr = PetscSectionSetNumFields(subCoordSection, 1);CHKERRQ(ierr);
3187     ierr = PetscSectionGetFieldComponents(coordSection, 0, &numComp);CHKERRQ(ierr);
3188     ierr = PetscSectionSetFieldComponents(subCoordSection, 0, numComp);CHKERRQ(ierr);
3189     ierr = PetscSectionSetChart(subCoordSection, firstSubPoint[0], firstSubPoint[0]+numSubPoints[0]);CHKERRQ(ierr);
3190     for (v = 0; v < numSubPoints[0]; ++v) {
3191       const PetscInt vertex    = subpoints[0][v];
3192       const PetscInt subvertex = firstSubPoint[0]+v;
3193       PetscInt       dof;
3194 
3195       ierr = PetscSectionGetDof(coordSection, vertex, &dof);CHKERRQ(ierr);
3196       ierr = PetscSectionSetDof(subCoordSection, subvertex, dof);CHKERRQ(ierr);
3197       ierr = PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof);CHKERRQ(ierr);
3198     }
3199     ierr = PetscSectionSetUp(subCoordSection);CHKERRQ(ierr);
3200     ierr = PetscSectionGetStorageSize(subCoordSection, &coordSize);CHKERRQ(ierr);
3201     ierr = VecCreate(PETSC_COMM_SELF, &subCoordinates);CHKERRQ(ierr);
3202     ierr = PetscObjectGetName((PetscObject)coordinates,&name);CHKERRQ(ierr);
3203     ierr = PetscObjectSetName((PetscObject)subCoordinates,name);CHKERRQ(ierr);
3204     ierr = VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
3205     ierr = VecSetBlockSize(subCoordinates, cdim);CHKERRQ(ierr);
3206     ierr = VecSetType(subCoordinates,VECSTANDARD);CHKERRQ(ierr);
3207     ierr = VecGetArray(coordinates,    &coords);CHKERRQ(ierr);
3208     ierr = VecGetArray(subCoordinates, &subCoords);CHKERRQ(ierr);
3209     for (v = 0; v < numSubPoints[0]; ++v) {
3210       const PetscInt vertex    = subpoints[0][v];
3211       const PetscInt subvertex = firstSubPoint[0]+v;
3212       PetscInt dof, off, sdof, soff, d;
3213 
3214       ierr = PetscSectionGetDof(coordSection, vertex, &dof);CHKERRQ(ierr);
3215       ierr = PetscSectionGetOffset(coordSection, vertex, &off);CHKERRQ(ierr);
3216       ierr = PetscSectionGetDof(subCoordSection, subvertex, &sdof);CHKERRQ(ierr);
3217       ierr = PetscSectionGetOffset(subCoordSection, subvertex, &soff);CHKERRQ(ierr);
3218       if (dof != sdof) SETERRQ4(comm, PETSC_ERR_PLIB, "Coordinate dimension %d on subvertex %d, vertex %d should be %d", sdof, subvertex, vertex, dof);
3219       for (d = 0; d < dof; ++d) subCoords[soff+d] = coords[off+d];
3220     }
3221     ierr = VecRestoreArray(coordinates,    &coords);CHKERRQ(ierr);
3222     ierr = VecRestoreArray(subCoordinates, &subCoords);CHKERRQ(ierr);
3223     ierr = DMSetCoordinatesLocal(subdm, subCoordinates);CHKERRQ(ierr);
3224     ierr = VecDestroy(&subCoordinates);CHKERRQ(ierr);
3225   }
3226   /* Build SF: We need this complexity because subpoints might not be selected on the owning process */
3227   {
3228     PetscSF            sfPoint, sfPointSub;
3229     IS                 subpIS;
3230     const PetscSFNode *remotePoints;
3231     PetscSFNode       *sremotePoints, *newLocalPoints, *newOwners;
3232     const PetscInt    *localPoints, *subpoints;
3233     PetscInt          *slocalPoints;
3234     PetscInt           numRoots, numLeaves, numSubpoints = 0, numSubroots, numSubleaves = 0, l, sl, ll, pStart, pEnd, p;
3235     PetscMPIInt        rank;
3236 
3237     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRMPI(ierr);
3238     ierr = DMGetPointSF(dm, &sfPoint);CHKERRQ(ierr);
3239     ierr = DMGetPointSF(subdm, &sfPointSub);CHKERRQ(ierr);
3240     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
3241     ierr = DMPlexGetChart(subdm, NULL, &numSubroots);CHKERRQ(ierr);
3242     ierr = DMPlexGetSubpointIS(subdm, &subpIS);CHKERRQ(ierr);
3243     if (subpIS) {
3244       ierr = ISGetIndices(subpIS, &subpoints);CHKERRQ(ierr);
3245       ierr = ISGetLocalSize(subpIS, &numSubpoints);CHKERRQ(ierr);
3246     }
3247     ierr = PetscSFGetGraph(sfPoint, &numRoots, &numLeaves, &localPoints, &remotePoints);CHKERRQ(ierr);
3248     if (numRoots >= 0) {
3249       ierr = PetscMalloc2(pEnd-pStart,&newLocalPoints,numRoots,&newOwners);CHKERRQ(ierr);
3250       for (p = 0; p < pEnd-pStart; ++p) {
3251         newLocalPoints[p].rank  = -2;
3252         newLocalPoints[p].index = -2;
3253       }
3254       /* Set subleaves */
3255       for (l = 0; l < numLeaves; ++l) {
3256         const PetscInt point    = localPoints[l];
3257         const PetscInt subpoint = DMPlexFilterPoint_Internal(point, 0, numSubpoints, subpoints);
3258 
3259         if (subpoint < 0) continue;
3260         newLocalPoints[point-pStart].rank  = rank;
3261         newLocalPoints[point-pStart].index = subpoint;
3262         ++numSubleaves;
3263       }
3264       /* Must put in owned subpoints */
3265       for (p = pStart; p < pEnd; ++p) {
3266         const PetscInt subpoint = DMPlexFilterPoint_Internal(p, 0, numSubpoints, subpoints);
3267 
3268         if (subpoint < 0) {
3269           newOwners[p-pStart].rank  = -3;
3270           newOwners[p-pStart].index = -3;
3271         } else {
3272           newOwners[p-pStart].rank  = rank;
3273           newOwners[p-pStart].index = subpoint;
3274         }
3275       }
3276       ierr = PetscSFReduceBegin(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC);CHKERRQ(ierr);
3277       ierr = PetscSFReduceEnd(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC);CHKERRQ(ierr);
3278       ierr = PetscSFBcastBegin(sfPoint, MPIU_2INT, newOwners, newLocalPoints,MPI_REPLACE);CHKERRQ(ierr);
3279       ierr = PetscSFBcastEnd(sfPoint, MPIU_2INT, newOwners, newLocalPoints,MPI_REPLACE);CHKERRQ(ierr);
3280       ierr = PetscMalloc1(numSubleaves, &slocalPoints);CHKERRQ(ierr);
3281       ierr = PetscMalloc1(numSubleaves, &sremotePoints);CHKERRQ(ierr);
3282       for (l = 0, sl = 0, ll = 0; l < numLeaves; ++l) {
3283         const PetscInt point    = localPoints[l];
3284         const PetscInt subpoint = DMPlexFilterPoint_Internal(point, 0, numSubpoints, subpoints);
3285 
3286         if (subpoint < 0) continue;
3287         if (newLocalPoints[point].rank == rank) {++ll; continue;}
3288         slocalPoints[sl]        = subpoint;
3289         sremotePoints[sl].rank  = newLocalPoints[point].rank;
3290         sremotePoints[sl].index = newLocalPoints[point].index;
3291         if (sremotePoints[sl].rank  < 0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote rank for local point %d", point);
3292         if (sremotePoints[sl].index < 0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote subpoint for local point %d", point);
3293         ++sl;
3294       }
3295       if (sl + ll != numSubleaves) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatch in number of subleaves %d + %d != %d", sl, ll, numSubleaves);
3296       ierr = PetscFree2(newLocalPoints,newOwners);CHKERRQ(ierr);
3297       ierr = PetscSFSetGraph(sfPointSub, numSubroots, sl, slocalPoints, PETSC_OWN_POINTER, sremotePoints, PETSC_OWN_POINTER);CHKERRQ(ierr);
3298     }
3299     if (subpIS) {
3300       ierr = ISRestoreIndices(subpIS, &subpoints);CHKERRQ(ierr);
3301     }
3302   }
3303   /* Cleanup */
3304   for (d = 0; d <= dim; ++d) {
3305     if (subpointIS[d]) {ierr = ISRestoreIndices(subpointIS[d], &subpoints[d]);CHKERRQ(ierr);}
3306     ierr = ISDestroy(&subpointIS[d]);CHKERRQ(ierr);
3307   }
3308   ierr = PetscFree4(numSubPoints,firstSubPoint,subpointIS,subpoints);CHKERRQ(ierr);
3309   PetscFunctionReturn(0);
3310 }
3311 
3312 static PetscErrorCode DMPlexCreateSubmesh_Interpolated(DM dm, DMLabel vertexLabel, PetscInt value, PetscBool markedFaces, DM subdm)
3313 {
3314   PetscErrorCode ierr;
3315 
3316   PetscFunctionBegin;
3317   ierr = DMPlexCreateSubmeshGeneric_Interpolated(dm, vertexLabel, value, markedFaces, PETSC_FALSE, 1, subdm);CHKERRQ(ierr);
3318   PetscFunctionReturn(0);
3319 }
3320 
3321 /*@
3322   DMPlexCreateSubmesh - Extract a hypersurface from the mesh using vertices defined by a label
3323 
3324   Input Parameters:
3325 + dm           - The original mesh
3326 . vertexLabel  - The DMLabel marking points contained in the surface
3327 . value        - The label value to use
3328 - markedFaces  - PETSC_TRUE if surface faces are marked in addition to vertices, PETSC_FALSE if only vertices are marked
3329 
3330   Output Parameter:
3331 . subdm - The surface mesh
3332 
3333   Note: This function produces a DMLabel mapping original points in the submesh to their depth. This can be obtained using DMPlexGetSubpointMap().
3334 
3335   Level: developer
3336 
3337 .seealso: DMPlexGetSubpointMap(), DMGetLabel(), DMLabelSetValue()
3338 @*/
3339 PetscErrorCode DMPlexCreateSubmesh(DM dm, DMLabel vertexLabel, PetscInt value, PetscBool markedFaces, DM *subdm)
3340 {
3341   DMPlexInterpolatedFlag interpolated;
3342   PetscInt       dim, cdim;
3343   PetscErrorCode ierr;
3344 
3345   PetscFunctionBegin;
3346   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3347   PetscValidPointer(subdm, 5);
3348   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
3349   ierr = DMCreate(PetscObjectComm((PetscObject)dm), subdm);CHKERRQ(ierr);
3350   ierr = DMSetType(*subdm, DMPLEX);CHKERRQ(ierr);
3351   ierr = DMSetDimension(*subdm, dim-1);CHKERRQ(ierr);
3352   ierr = DMGetCoordinateDim(dm, &cdim);CHKERRQ(ierr);
3353   ierr = DMSetCoordinateDim(*subdm, cdim);CHKERRQ(ierr);
3354   ierr = DMPlexIsInterpolated(dm, &interpolated);CHKERRQ(ierr);
3355   if (interpolated == DMPLEX_INTERPOLATED_PARTIAL) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Not for partially interpolated meshes");
3356   if (interpolated) {
3357     ierr = DMPlexCreateSubmesh_Interpolated(dm, vertexLabel, value, markedFaces, *subdm);CHKERRQ(ierr);
3358   } else {
3359     ierr = DMPlexCreateSubmesh_Uninterpolated(dm, vertexLabel, value, *subdm);CHKERRQ(ierr);
3360   }
3361   PetscFunctionReturn(0);
3362 }
3363 
3364 static PetscErrorCode DMPlexCreateCohesiveSubmesh_Uninterpolated(DM dm, PetscBool hasLagrange, const char label[], PetscInt value, DM subdm)
3365 {
3366   MPI_Comm        comm;
3367   DMLabel         subpointMap;
3368   IS              subvertexIS;
3369   const PetscInt *subVertices;
3370   PetscInt        numSubVertices, firstSubVertex, numSubCells, *subCells = NULL;
3371   PetscInt       *subface, maxConeSize, numSubFaces, firstSubFace, newFacePoint, nFV;
3372   PetscInt        c, f;
3373   PetscErrorCode  ierr;
3374 
3375   PetscFunctionBegin;
3376   ierr = PetscObjectGetComm((PetscObject)dm, &comm);CHKERRQ(ierr);
3377   /* Create subpointMap which marks the submesh */
3378   ierr = DMLabelCreate(PETSC_COMM_SELF, "subpoint_map", &subpointMap);CHKERRQ(ierr);
3379   ierr = DMPlexSetSubpointMap(subdm, subpointMap);CHKERRQ(ierr);
3380   ierr = DMLabelDestroy(&subpointMap);CHKERRQ(ierr);
3381   ierr = DMPlexMarkCohesiveSubmesh_Uninterpolated(dm, hasLagrange, label, value, subpointMap, &numSubFaces, &nFV, &subCells, subdm);CHKERRQ(ierr);
3382   /* Setup chart */
3383   ierr = DMLabelGetStratumSize(subpointMap, 0, &numSubVertices);CHKERRQ(ierr);
3384   ierr = DMLabelGetStratumSize(subpointMap, 2, &numSubCells);CHKERRQ(ierr);
3385   ierr = DMPlexSetChart(subdm, 0, numSubCells+numSubFaces+numSubVertices);CHKERRQ(ierr);
3386   ierr = DMPlexSetVTKCellHeight(subdm, 1);CHKERRQ(ierr);
3387   /* Set cone sizes */
3388   firstSubVertex = numSubCells;
3389   firstSubFace   = numSubCells+numSubVertices;
3390   newFacePoint   = firstSubFace;
3391   ierr = DMLabelGetStratumIS(subpointMap, 0, &subvertexIS);CHKERRQ(ierr);
3392   if (subvertexIS) {ierr = ISGetIndices(subvertexIS, &subVertices);CHKERRQ(ierr);}
3393   for (c = 0; c < numSubCells; ++c) {
3394     ierr = DMPlexSetConeSize(subdm, c, 1);CHKERRQ(ierr);
3395   }
3396   for (f = firstSubFace; f < firstSubFace+numSubFaces; ++f) {
3397     ierr = DMPlexSetConeSize(subdm, f, nFV);CHKERRQ(ierr);
3398   }
3399   ierr = DMSetUp(subdm);CHKERRQ(ierr);
3400   /* Create face cones */
3401   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, NULL);CHKERRQ(ierr);
3402   ierr = DMGetWorkArray(subdm, maxConeSize, MPIU_INT, (void**) &subface);CHKERRQ(ierr);
3403   for (c = 0; c < numSubCells; ++c) {
3404     const PetscInt  cell    = subCells[c];
3405     const PetscInt  subcell = c;
3406     const PetscInt *cone, *cells;
3407     PetscBool       isHybrid;
3408     PetscInt        numCells, subVertex, p, v;
3409 
3410     ierr = DMPlexCellIsHybrid_Internal(dm, cell, &isHybrid);CHKERRQ(ierr);
3411     if (!isHybrid) continue;
3412     ierr = DMPlexGetCone(dm, cell, &cone);CHKERRQ(ierr);
3413     for (v = 0; v < nFV; ++v) {
3414       ierr = PetscFindInt(cone[v], numSubVertices, subVertices, &subVertex);CHKERRQ(ierr);
3415       subface[v] = firstSubVertex+subVertex;
3416     }
3417     ierr = DMPlexSetCone(subdm, newFacePoint, subface);CHKERRQ(ierr);
3418     ierr = DMPlexSetCone(subdm, subcell, &newFacePoint);CHKERRQ(ierr);
3419     ierr = DMPlexGetJoin(dm, nFV, cone, &numCells, &cells);CHKERRQ(ierr);
3420     /* Not true in parallel
3421     if (numCells != 2) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive cells should separate two cells"); */
3422     for (p = 0; p < numCells; ++p) {
3423       PetscInt  negsubcell;
3424       PetscBool isHybrid;
3425 
3426       ierr = DMPlexCellIsHybrid_Internal(dm, cells[p], &isHybrid);CHKERRQ(ierr);
3427       if (isHybrid) continue;
3428       /* I know this is a crap search */
3429       for (negsubcell = 0; negsubcell < numSubCells; ++negsubcell) {
3430         if (subCells[negsubcell] == cells[p]) break;
3431       }
3432       if (negsubcell == numSubCells) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not find negative face neighbor for cohesive cell %d", cell);
3433       ierr = DMPlexSetCone(subdm, negsubcell, &newFacePoint);CHKERRQ(ierr);
3434     }
3435     ierr = DMPlexRestoreJoin(dm, nFV, cone, &numCells, &cells);CHKERRQ(ierr);
3436     ++newFacePoint;
3437   }
3438   ierr = DMRestoreWorkArray(subdm, maxConeSize, MPIU_INT, (void**) &subface);CHKERRQ(ierr);
3439   ierr = DMPlexSymmetrize(subdm);CHKERRQ(ierr);
3440   ierr = DMPlexStratify(subdm);CHKERRQ(ierr);
3441   /* Build coordinates */
3442   {
3443     PetscSection coordSection, subCoordSection;
3444     Vec          coordinates, subCoordinates;
3445     PetscScalar *coords, *subCoords;
3446     PetscInt     cdim, numComp, coordSize, v;
3447     const char  *name;
3448 
3449     ierr = DMGetCoordinateDim(dm, &cdim);CHKERRQ(ierr);
3450     ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
3451     ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
3452     ierr = DMGetCoordinateSection(subdm, &subCoordSection);CHKERRQ(ierr);
3453     ierr = PetscSectionSetNumFields(subCoordSection, 1);CHKERRQ(ierr);
3454     ierr = PetscSectionGetFieldComponents(coordSection, 0, &numComp);CHKERRQ(ierr);
3455     ierr = PetscSectionSetFieldComponents(subCoordSection, 0, numComp);CHKERRQ(ierr);
3456     ierr = PetscSectionSetChart(subCoordSection, firstSubVertex, firstSubVertex+numSubVertices);CHKERRQ(ierr);
3457     for (v = 0; v < numSubVertices; ++v) {
3458       const PetscInt vertex    = subVertices[v];
3459       const PetscInt subvertex = firstSubVertex+v;
3460       PetscInt       dof;
3461 
3462       ierr = PetscSectionGetDof(coordSection, vertex, &dof);CHKERRQ(ierr);
3463       ierr = PetscSectionSetDof(subCoordSection, subvertex, dof);CHKERRQ(ierr);
3464       ierr = PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof);CHKERRQ(ierr);
3465     }
3466     ierr = PetscSectionSetUp(subCoordSection);CHKERRQ(ierr);
3467     ierr = PetscSectionGetStorageSize(subCoordSection, &coordSize);CHKERRQ(ierr);
3468     ierr = VecCreate(PETSC_COMM_SELF, &subCoordinates);CHKERRQ(ierr);
3469     ierr = PetscObjectGetName((PetscObject)coordinates,&name);CHKERRQ(ierr);
3470     ierr = PetscObjectSetName((PetscObject)subCoordinates,name);CHKERRQ(ierr);
3471     ierr = VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
3472     ierr = VecSetBlockSize(subCoordinates, cdim);CHKERRQ(ierr);
3473     ierr = VecSetType(subCoordinates,VECSTANDARD);CHKERRQ(ierr);
3474     ierr = VecGetArray(coordinates,    &coords);CHKERRQ(ierr);
3475     ierr = VecGetArray(subCoordinates, &subCoords);CHKERRQ(ierr);
3476     for (v = 0; v < numSubVertices; ++v) {
3477       const PetscInt vertex    = subVertices[v];
3478       const PetscInt subvertex = firstSubVertex+v;
3479       PetscInt       dof, off, sdof, soff, d;
3480 
3481       ierr = PetscSectionGetDof(coordSection, vertex, &dof);CHKERRQ(ierr);
3482       ierr = PetscSectionGetOffset(coordSection, vertex, &off);CHKERRQ(ierr);
3483       ierr = PetscSectionGetDof(subCoordSection, subvertex, &sdof);CHKERRQ(ierr);
3484       ierr = PetscSectionGetOffset(subCoordSection, subvertex, &soff);CHKERRQ(ierr);
3485       if (dof != sdof) SETERRQ4(comm, PETSC_ERR_PLIB, "Coordinate dimension %d on subvertex %d, vertex %d should be %d", sdof, subvertex, vertex, dof);
3486       for (d = 0; d < dof; ++d) subCoords[soff+d] = coords[off+d];
3487     }
3488     ierr = VecRestoreArray(coordinates,    &coords);CHKERRQ(ierr);
3489     ierr = VecRestoreArray(subCoordinates, &subCoords);CHKERRQ(ierr);
3490     ierr = DMSetCoordinatesLocal(subdm, subCoordinates);CHKERRQ(ierr);
3491     ierr = VecDestroy(&subCoordinates);CHKERRQ(ierr);
3492   }
3493   /* Build SF */
3494   CHKMEMQ;
3495   {
3496     PetscSF            sfPoint, sfPointSub;
3497     const PetscSFNode *remotePoints;
3498     PetscSFNode       *sremotePoints, *newLocalPoints, *newOwners;
3499     const PetscInt    *localPoints;
3500     PetscInt          *slocalPoints;
3501     PetscInt           numRoots, numLeaves, numSubRoots = numSubCells+numSubFaces+numSubVertices, numSubLeaves = 0, l, sl, ll, pStart, pEnd, p, vStart, vEnd;
3502     PetscMPIInt        rank;
3503 
3504     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRMPI(ierr);
3505     ierr = DMGetPointSF(dm, &sfPoint);CHKERRQ(ierr);
3506     ierr = DMGetPointSF(subdm, &sfPointSub);CHKERRQ(ierr);
3507     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
3508     ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
3509     ierr = PetscSFGetGraph(sfPoint, &numRoots, &numLeaves, &localPoints, &remotePoints);CHKERRQ(ierr);
3510     if (numRoots >= 0) {
3511       /* Only vertices should be shared */
3512       ierr = PetscMalloc2(pEnd-pStart,&newLocalPoints,numRoots,&newOwners);CHKERRQ(ierr);
3513       for (p = 0; p < pEnd-pStart; ++p) {
3514         newLocalPoints[p].rank  = -2;
3515         newLocalPoints[p].index = -2;
3516       }
3517       /* Set subleaves */
3518       for (l = 0; l < numLeaves; ++l) {
3519         const PetscInt point    = localPoints[l];
3520         const PetscInt subPoint = DMPlexFilterPoint_Internal(point, firstSubVertex, numSubVertices, subVertices);
3521 
3522         if ((point < vStart) && (point >= vEnd)) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Should not be mapping anything but vertices, %d", point);
3523         if (subPoint < 0) continue;
3524         newLocalPoints[point-pStart].rank  = rank;
3525         newLocalPoints[point-pStart].index = subPoint;
3526         ++numSubLeaves;
3527       }
3528       /* Must put in owned subpoints */
3529       for (p = pStart; p < pEnd; ++p) {
3530         const PetscInt subPoint = DMPlexFilterPoint_Internal(p, firstSubVertex, numSubVertices, subVertices);
3531 
3532         if (subPoint < 0) {
3533           newOwners[p-pStart].rank  = -3;
3534           newOwners[p-pStart].index = -3;
3535         } else {
3536           newOwners[p-pStart].rank  = rank;
3537           newOwners[p-pStart].index = subPoint;
3538         }
3539       }
3540       ierr = PetscSFReduceBegin(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC);CHKERRQ(ierr);
3541       ierr = PetscSFReduceEnd(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC);CHKERRQ(ierr);
3542       ierr = PetscSFBcastBegin(sfPoint, MPIU_2INT, newOwners, newLocalPoints,MPI_REPLACE);CHKERRQ(ierr);
3543       ierr = PetscSFBcastEnd(sfPoint, MPIU_2INT, newOwners, newLocalPoints,MPI_REPLACE);CHKERRQ(ierr);
3544       ierr = PetscMalloc1(numSubLeaves,    &slocalPoints);CHKERRQ(ierr);
3545       ierr = PetscMalloc1(numSubLeaves, &sremotePoints);CHKERRQ(ierr);
3546       for (l = 0, sl = 0, ll = 0; l < numLeaves; ++l) {
3547         const PetscInt point    = localPoints[l];
3548         const PetscInt subPoint = DMPlexFilterPoint_Internal(point, firstSubVertex, numSubVertices, subVertices);
3549 
3550         if (subPoint < 0) continue;
3551         if (newLocalPoints[point].rank == rank) {++ll; continue;}
3552         slocalPoints[sl]        = subPoint;
3553         sremotePoints[sl].rank  = newLocalPoints[point].rank;
3554         sremotePoints[sl].index = newLocalPoints[point].index;
3555         if (sremotePoints[sl].rank  < 0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote rank for local point %d", point);
3556         if (sremotePoints[sl].index < 0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote subpoint for local point %d", point);
3557         ++sl;
3558       }
3559       ierr = PetscFree2(newLocalPoints,newOwners);CHKERRQ(ierr);
3560       if (sl + ll != numSubLeaves) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatch in number of subleaves %d + %d != %d", sl, ll, numSubLeaves);
3561       ierr = PetscSFSetGraph(sfPointSub, numSubRoots, sl, slocalPoints, PETSC_OWN_POINTER, sremotePoints, PETSC_OWN_POINTER);CHKERRQ(ierr);
3562     }
3563   }
3564   CHKMEMQ;
3565   /* Cleanup */
3566   if (subvertexIS) {ierr = ISRestoreIndices(subvertexIS, &subVertices);CHKERRQ(ierr);}
3567   ierr = ISDestroy(&subvertexIS);CHKERRQ(ierr);
3568   ierr = PetscFree(subCells);CHKERRQ(ierr);
3569   PetscFunctionReturn(0);
3570 }
3571 
3572 static PetscErrorCode DMPlexCreateCohesiveSubmesh_Interpolated(DM dm, const char labelname[], PetscInt value, DM subdm)
3573 {
3574   DMLabel        label = NULL;
3575   PetscErrorCode ierr;
3576 
3577   PetscFunctionBegin;
3578   if (labelname) {ierr = DMGetLabel(dm, labelname, &label);CHKERRQ(ierr);}
3579   ierr = DMPlexCreateSubmeshGeneric_Interpolated(dm, label, value, PETSC_FALSE, PETSC_TRUE, 1, subdm);CHKERRQ(ierr);
3580   PetscFunctionReturn(0);
3581 }
3582 
3583 /*@C
3584   DMPlexCreateCohesiveSubmesh - Extract from a mesh with cohesive cells the hypersurface defined by one face of the cells. Optionally, a Label an be given to restrict the cells.
3585 
3586   Input Parameters:
3587 + dm          - The original mesh
3588 . hasLagrange - The mesh has Lagrange unknowns in the cohesive cells
3589 . label       - A label name, or NULL
3590 - value  - A label value
3591 
3592   Output Parameter:
3593 . subdm - The surface mesh
3594 
3595   Note: This function produces a DMLabel mapping original points in the submesh to their depth. This can be obtained using DMPlexGetSubpointMap().
3596 
3597   Level: developer
3598 
3599 .seealso: DMPlexGetSubpointMap(), DMPlexCreateSubmesh()
3600 @*/
3601 PetscErrorCode DMPlexCreateCohesiveSubmesh(DM dm, PetscBool hasLagrange, const char label[], PetscInt value, DM *subdm)
3602 {
3603   PetscInt       dim, cdim, depth;
3604   PetscErrorCode ierr;
3605 
3606   PetscFunctionBegin;
3607   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3608   PetscValidPointer(subdm, 5);
3609   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
3610   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3611   ierr = DMCreate(PetscObjectComm((PetscObject)dm), subdm);CHKERRQ(ierr);
3612   ierr = DMSetType(*subdm, DMPLEX);CHKERRQ(ierr);
3613   ierr = DMSetDimension(*subdm, dim-1);CHKERRQ(ierr);
3614   ierr = DMGetCoordinateDim(dm, &cdim);CHKERRQ(ierr);
3615   ierr = DMSetCoordinateDim(*subdm, cdim);CHKERRQ(ierr);
3616   if (depth == dim) {
3617     ierr = DMPlexCreateCohesiveSubmesh_Interpolated(dm, label, value, *subdm);CHKERRQ(ierr);
3618   } else {
3619     ierr = DMPlexCreateCohesiveSubmesh_Uninterpolated(dm, hasLagrange, label, value, *subdm);CHKERRQ(ierr);
3620   }
3621   PetscFunctionReturn(0);
3622 }
3623 
3624 /*@
3625   DMPlexFilter - Extract a subset of mesh cells defined by a label as a separate mesh
3626 
3627   Input Parameters:
3628 + dm        - The original mesh
3629 . cellLabel - The DMLabel marking cells contained in the new mesh
3630 - value     - The label value to use
3631 
3632   Output Parameter:
3633 . subdm - The new mesh
3634 
3635   Note: This function produces a DMLabel mapping original points in the submesh to their depth. This can be obtained using DMPlexGetSubpointMap().
3636 
3637   Level: developer
3638 
3639 .seealso: DMPlexGetSubpointMap(), DMGetLabel(), DMLabelSetValue()
3640 @*/
3641 PetscErrorCode DMPlexFilter(DM dm, DMLabel cellLabel, PetscInt value, DM *subdm)
3642 {
3643   PetscInt       dim;
3644   PetscErrorCode ierr;
3645 
3646   PetscFunctionBegin;
3647   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3648   PetscValidPointer(subdm, 4);
3649   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
3650   ierr = DMCreate(PetscObjectComm((PetscObject) dm), subdm);CHKERRQ(ierr);
3651   ierr = DMSetType(*subdm, DMPLEX);CHKERRQ(ierr);
3652   ierr = DMSetDimension(*subdm, dim);CHKERRQ(ierr);
3653   /* Extract submesh in place, could be empty on some procs, could have inconsistency if procs do not both extract a shared cell */
3654   ierr = DMPlexCreateSubmeshGeneric_Interpolated(dm, cellLabel, value, PETSC_FALSE, PETSC_FALSE, 0, *subdm);CHKERRQ(ierr);
3655   PetscFunctionReturn(0);
3656 }
3657 
3658 /*@
3659   DMPlexGetSubpointMap - Returns a DMLabel with point dimension as values
3660 
3661   Input Parameter:
3662 . dm - The submesh DM
3663 
3664   Output Parameter:
3665 . subpointMap - The DMLabel of all the points from the original mesh in this submesh, or NULL if this is not a submesh
3666 
3667   Level: developer
3668 
3669 .seealso: DMPlexCreateSubmesh(), DMPlexGetSubpointIS()
3670 @*/
3671 PetscErrorCode DMPlexGetSubpointMap(DM dm, DMLabel *subpointMap)
3672 {
3673   PetscFunctionBegin;
3674   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3675   PetscValidPointer(subpointMap, 2);
3676   *subpointMap = ((DM_Plex*) dm->data)->subpointMap;
3677   PetscFunctionReturn(0);
3678 }
3679 
3680 /*@
3681   DMPlexSetSubpointMap - Sets the DMLabel with point dimension as values
3682 
3683   Input Parameters:
3684 + dm - The submesh DM
3685 - subpointMap - The DMLabel of all the points from the original mesh in this submesh
3686 
3687   Note: Should normally not be called by the user, since it is set in DMPlexCreateSubmesh()
3688 
3689   Level: developer
3690 
3691 .seealso: DMPlexCreateSubmesh(), DMPlexGetSubpointIS()
3692 @*/
3693 PetscErrorCode DMPlexSetSubpointMap(DM dm, DMLabel subpointMap)
3694 {
3695   DM_Plex       *mesh = (DM_Plex *) dm->data;
3696   DMLabel        tmp;
3697   PetscErrorCode ierr;
3698 
3699   PetscFunctionBegin;
3700   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3701   tmp  = mesh->subpointMap;
3702   mesh->subpointMap = subpointMap;
3703   ierr = PetscObjectReference((PetscObject) mesh->subpointMap);CHKERRQ(ierr);
3704   ierr = DMLabelDestroy(&tmp);CHKERRQ(ierr);
3705   PetscFunctionReturn(0);
3706 }
3707 
3708 static PetscErrorCode DMPlexCreateSubpointIS_Internal(DM dm, IS *subpointIS)
3709 {
3710   DMLabel        spmap;
3711   PetscInt       depth, d;
3712   PetscErrorCode ierr;
3713 
3714   PetscFunctionBegin;
3715   ierr = DMPlexGetSubpointMap(dm, &spmap);CHKERRQ(ierr);
3716   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3717   if (spmap && depth >= 0) {
3718     DM_Plex  *mesh = (DM_Plex *) dm->data;
3719     PetscInt *points, *depths;
3720     PetscInt  pStart, pEnd, p, off;
3721 
3722     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
3723     if (pStart) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Submeshes must start the point numbering at 0, not %d", pStart);
3724     ierr = PetscMalloc1(pEnd, &points);CHKERRQ(ierr);
3725     ierr = DMGetWorkArray(dm, depth+1, MPIU_INT, &depths);CHKERRQ(ierr);
3726     depths[0] = depth;
3727     depths[1] = 0;
3728     for (d = 2; d <= depth; ++d) {depths[d] = depth+1 - d;}
3729     for (d = 0, off = 0; d <= depth; ++d) {
3730       const PetscInt dep = depths[d];
3731       PetscInt       depStart, depEnd, n;
3732 
3733       ierr = DMPlexGetDepthStratum(dm, dep, &depStart, &depEnd);CHKERRQ(ierr);
3734       ierr = DMLabelGetStratumSize(spmap, dep, &n);CHKERRQ(ierr);
3735       if (((d < 2) && (depth > 1)) || (d == 1)) { /* Only check vertices and cells for now since the map is broken for others */
3736         if (n != depEnd-depStart) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "The number of mapped submesh points %d at depth %d should be %d", n, dep, depEnd-depStart);
3737       } else {
3738         if (!n) {
3739           if (d == 0) {
3740             /* Missing cells */
3741             for (p = 0; p < depEnd-depStart; ++p, ++off) points[off] = -1;
3742           } else {
3743             /* Missing faces */
3744             for (p = 0; p < depEnd-depStart; ++p, ++off) points[off] = PETSC_MAX_INT;
3745           }
3746         }
3747       }
3748       if (n) {
3749         IS              is;
3750         const PetscInt *opoints;
3751 
3752         ierr = DMLabelGetStratumIS(spmap, dep, &is);CHKERRQ(ierr);
3753         ierr = ISGetIndices(is, &opoints);CHKERRQ(ierr);
3754         for (p = 0; p < n; ++p, ++off) points[off] = opoints[p];
3755         ierr = ISRestoreIndices(is, &opoints);CHKERRQ(ierr);
3756         ierr = ISDestroy(&is);CHKERRQ(ierr);
3757       }
3758     }
3759     ierr = DMRestoreWorkArray(dm, depth+1, MPIU_INT, &depths);CHKERRQ(ierr);
3760     if (off != pEnd) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "The number of mapped submesh points %d should be %d", off, pEnd);
3761     ierr = ISCreateGeneral(PETSC_COMM_SELF, pEnd, points, PETSC_OWN_POINTER, subpointIS);CHKERRQ(ierr);
3762     ierr = PetscObjectStateGet((PetscObject) spmap, &mesh->subpointState);CHKERRQ(ierr);
3763   }
3764   PetscFunctionReturn(0);
3765 }
3766 
3767 /*@
3768   DMPlexGetSubpointIS - Returns an IS covering the entire subdm chart with the original points as data
3769 
3770   Input Parameter:
3771 . dm - The submesh DM
3772 
3773   Output Parameter:
3774 . subpointIS - The IS of all the points from the original mesh in this submesh, or NULL if this is not a submesh
3775 
3776   Note: This IS is guaranteed to be sorted by the construction of the submesh
3777 
3778   Level: developer
3779 
3780 .seealso: DMPlexCreateSubmesh(), DMPlexGetSubpointMap()
3781 @*/
3782 PetscErrorCode DMPlexGetSubpointIS(DM dm, IS *subpointIS)
3783 {
3784   DM_Plex         *mesh = (DM_Plex *) dm->data;
3785   DMLabel          spmap;
3786   PetscObjectState state;
3787   PetscErrorCode   ierr;
3788 
3789   PetscFunctionBegin;
3790   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3791   PetscValidPointer(subpointIS, 2);
3792   ierr = DMPlexGetSubpointMap(dm, &spmap);CHKERRQ(ierr);
3793   ierr = PetscObjectStateGet((PetscObject) spmap, &state);CHKERRQ(ierr);
3794   if (state != mesh->subpointState || !mesh->subpointIS) {ierr = DMPlexCreateSubpointIS_Internal(dm, &mesh->subpointIS);CHKERRQ(ierr);}
3795   *subpointIS = mesh->subpointIS;
3796   PetscFunctionReturn(0);
3797 }
3798 
3799 /*@
3800   DMGetEnclosureRelation - Get the relationship between dmA and dmB
3801 
3802   Input Parameters:
3803 + dmA - The first DM
3804 - dmB - The second DM
3805 
3806   Output Parameter:
3807 . rel - The relation of dmA to dmB
3808 
3809   Level: intermediate
3810 
3811 .seealso: DMGetEnclosurePoint()
3812 @*/
3813 PetscErrorCode DMGetEnclosureRelation(DM dmA, DM dmB, DMEnclosureType *rel)
3814 {
3815   DM             plexA, plexB, sdm;
3816   DMLabel        spmap;
3817   PetscInt       pStartA, pEndA, pStartB, pEndB, NpA, NpB;
3818   PetscErrorCode ierr;
3819 
3820   PetscFunctionBegin;
3821   PetscValidPointer(rel, 3);
3822   *rel = DM_ENC_NONE;
3823   if (!dmA || !dmB) PetscFunctionReturn(0);
3824   PetscValidHeaderSpecific(dmA, DM_CLASSID, 1);
3825   PetscValidHeaderSpecific(dmB, DM_CLASSID, 2);
3826   if (dmA == dmB) {*rel = DM_ENC_EQUALITY; PetscFunctionReturn(0);}
3827   ierr = DMConvert(dmA, DMPLEX, &plexA);CHKERRQ(ierr);
3828   ierr = DMConvert(dmB, DMPLEX, &plexB);CHKERRQ(ierr);
3829   ierr = DMPlexGetChart(plexA, &pStartA, &pEndA);CHKERRQ(ierr);
3830   ierr = DMPlexGetChart(plexB, &pStartB, &pEndB);CHKERRQ(ierr);
3831   /* Assumption 1: subDMs have smaller charts than the DMs that they originate from
3832     - The degenerate case of a subdomain which includes all of the domain on some process can be treated as equality */
3833   if ((pStartA == pStartB) && (pEndA == pEndB)) {
3834     *rel = DM_ENC_EQUALITY;
3835     goto end;
3836   }
3837   NpA = pEndA - pStartA;
3838   NpB = pEndB - pStartB;
3839   if (NpA == NpB) goto end;
3840   sdm = NpA > NpB ? plexB : plexA; /* The other is the original, enclosing dm */
3841   ierr = DMPlexGetSubpointMap(sdm, &spmap);CHKERRQ(ierr);
3842   if (!spmap) goto end;
3843   /* TODO Check the space mapped to by subpointMap is same size as dm */
3844   if (NpA > NpB) {
3845     *rel = DM_ENC_SUPERMESH;
3846   } else {
3847     *rel = DM_ENC_SUBMESH;
3848   }
3849   end:
3850   ierr = DMDestroy(&plexA);CHKERRQ(ierr);
3851   ierr = DMDestroy(&plexB);CHKERRQ(ierr);
3852   PetscFunctionReturn(0);
3853 }
3854 
3855 /*@
3856   DMGetEnclosurePoint - Get the point pA in dmA which corresponds to the point pB in dmB
3857 
3858   Input Parameters:
3859 + dmA   - The first DM
3860 . dmB   - The second DM
3861 . etype - The type of enclosure relation that dmA has to dmB
3862 - pB    - A point of dmB
3863 
3864   Output Parameter:
3865 . pA    - The corresponding point of dmA
3866 
3867   Level: intermediate
3868 
3869 .seealso: DMGetEnclosureRelation()
3870 @*/
3871 PetscErrorCode DMGetEnclosurePoint(DM dmA, DM dmB, DMEnclosureType etype, PetscInt pB, PetscInt *pA)
3872 {
3873   DM              sdm;
3874   IS              subpointIS;
3875   const PetscInt *subpoints;
3876   PetscInt        numSubpoints;
3877   PetscErrorCode  ierr;
3878 
3879   PetscFunctionBegin;
3880   /* TODO Cache the IS, making it look like an index */
3881   switch (etype) {
3882     case DM_ENC_SUPERMESH:
3883     sdm  = dmB;
3884     ierr = DMPlexGetSubpointIS(sdm, &subpointIS);CHKERRQ(ierr);
3885     ierr = ISGetIndices(subpointIS, &subpoints);CHKERRQ(ierr);
3886     *pA  = subpoints[pB];
3887     ierr = ISRestoreIndices(subpointIS, &subpoints);CHKERRQ(ierr);
3888     break;
3889     case DM_ENC_SUBMESH:
3890     sdm  = dmA;
3891     ierr = DMPlexGetSubpointIS(sdm, &subpointIS);CHKERRQ(ierr);
3892     ierr = ISGetLocalSize(subpointIS, &numSubpoints);CHKERRQ(ierr);
3893     ierr = ISGetIndices(subpointIS, &subpoints);CHKERRQ(ierr);
3894     ierr = PetscFindInt(pB, numSubpoints, subpoints, pA);CHKERRQ(ierr);
3895     if (*pA < 0) {
3896       ierr = DMViewFromOptions(dmA, NULL, "-dm_enc_A_view");CHKERRQ(ierr);
3897       ierr = DMViewFromOptions(dmB, NULL, "-dm_enc_B_view");CHKERRQ(ierr);
3898       SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Point %d not found in submesh", pB);
3899     }
3900     ierr = ISRestoreIndices(subpointIS, &subpoints);CHKERRQ(ierr);
3901     break;
3902     case DM_ENC_EQUALITY:
3903     case DM_ENC_NONE:
3904     *pA = pB;break;
3905     case DM_ENC_UNKNOWN:
3906     {
3907       DMEnclosureType enc;
3908 
3909       ierr = DMGetEnclosureRelation(dmA, dmB, &enc);CHKERRQ(ierr);
3910       ierr = DMGetEnclosurePoint(dmA, dmB, enc, pB, pA);CHKERRQ(ierr);
3911     }
3912     break;
3913     default: SETERRQ1(PetscObjectComm((PetscObject) dmA), PETSC_ERR_ARG_OUTOFRANGE, "Invalid enclosure type %d", (int) etype);
3914   }
3915   PetscFunctionReturn(0);
3916 }
3917