xref: /petsc/src/dm/impls/plex/plexsubmesh.c (revision 561bb3f4137092920721da7c220a372726fb5d48)
1 #include <petsc/private/dmpleximpl.h>    /*I      "petscdmplex.h"    I*/
2 #include <petsc/private/dmlabelimpl.h>   /*I      "petscdmlabel.h"   I*/
3 #include <petscsf.h>
4 
5 static PetscErrorCode DMPlexMarkBoundaryFaces_Internal(DM dm, PetscInt cellHeight, DMLabel label)
6 {
7   PetscInt       fStart, fEnd, f;
8   PetscErrorCode ierr;
9 
10   PetscFunctionBegin;
11   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
12   ierr = DMPlexGetHeightStratum(dm, cellHeight+1, &fStart, &fEnd);CHKERRQ(ierr);
13   for (f = fStart; f < fEnd; ++f) {
14     PetscInt supportSize;
15 
16     ierr = DMPlexGetSupportSize(dm, f, &supportSize);CHKERRQ(ierr);
17     if (supportSize == 1) {ierr = DMLabelSetValue(label, f, 1);CHKERRQ(ierr);}
18   }
19   PetscFunctionReturn(0);
20 }
21 
22 /*@
23   DMPlexMarkBoundaryFaces - Mark all faces on the boundary
24 
25   Not Collective
26 
27   Input Parameter:
28 . dm - The original DM
29 
30   Output Parameter:
31 . label - The DMLabel marking boundary faces with value 1
32 
33   Level: developer
34 
35 .seealso: DMLabelCreate(), DMCreateLabel()
36 @*/
37 PetscErrorCode DMPlexMarkBoundaryFaces(DM dm, DMLabel label)
38 {
39   PetscErrorCode ierr;
40 
41   PetscFunctionBegin;
42   ierr = DMPlexMarkBoundaryFaces_Internal(dm, 0, label);CHKERRQ(ierr);
43   PetscFunctionReturn(0);
44 }
45 
46 static PetscErrorCode DMPlexLabelComplete_Internal(DM dm, DMLabel label, PetscBool completeCells)
47 {
48   IS              valueIS;
49   const PetscInt *values;
50   PetscInt        numValues, v, cStart, cEnd;
51   PetscErrorCode  ierr;
52 
53   PetscFunctionBegin;
54   ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
55   ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
56   ierr = DMPlexGetHeightStratum(dm,0,&cStart,&cEnd);CHKERRQ(ierr);
57   ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
58   for (v = 0; v < numValues; ++v) {
59     IS              pointIS;
60     const PetscInt *points;
61     PetscInt        numPoints, p;
62 
63     ierr = DMLabelGetStratumSize(label, values[v], &numPoints);CHKERRQ(ierr);
64     ierr = DMLabelGetStratumIS(label, values[v], &pointIS);CHKERRQ(ierr);
65     ierr = ISGetIndices(pointIS, &points);CHKERRQ(ierr);
66     for (p = 0; p < numPoints; ++p) {
67       PetscInt  q = points[p];
68       PetscInt *closure = NULL;
69       PetscInt  closureSize, c;
70 
71       if (cStart <= q && q < cEnd && !completeCells) { /* skip cells */
72         continue;
73       }
74       ierr = DMPlexGetTransitiveClosure(dm, q, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
75       for (c = 0; c < closureSize*2; c += 2) {
76         ierr = DMLabelSetValue(label, closure[c], values[v]);CHKERRQ(ierr);
77       }
78       ierr = DMPlexRestoreTransitiveClosure(dm, q, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
79     }
80     ierr = ISRestoreIndices(pointIS, &points);CHKERRQ(ierr);
81     ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
82   }
83   ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
84   ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
85   PetscFunctionReturn(0);
86 }
87 
88 /*@
89   DMPlexLabelComplete - Starting with a label marking points on a surface, we add the transitive closure to the surface
90 
91   Input Parameters:
92 + dm - The DM
93 - label - A DMLabel marking the surface points
94 
95   Output Parameter:
96 . label - A DMLabel marking all surface points in the transitive closure
97 
98   Level: developer
99 
100 .seealso: DMPlexLabelCohesiveComplete()
101 @*/
102 PetscErrorCode DMPlexLabelComplete(DM dm, DMLabel label)
103 {
104   PetscErrorCode ierr;
105 
106   PetscFunctionBegin;
107   ierr = DMPlexLabelComplete_Internal(dm, label, PETSC_TRUE);CHKERRQ(ierr);
108   PetscFunctionReturn(0);
109 }
110 
111 /*@
112   DMPlexLabelAddCells - Starting with a label marking faces on a surface, we add a cell for each face
113 
114   Input Parameters:
115 + dm - The DM
116 - label - A DMLabel marking the surface points
117 
118   Output Parameter:
119 . label - A DMLabel incorporating cells
120 
121   Level: developer
122 
123   Note: The cells allow FEM boundary conditions to be applied using the cell geometry
124 
125 .seealso: DMPlexLabelComplete(), DMPlexLabelCohesiveComplete()
126 @*/
127 PetscErrorCode DMPlexLabelAddCells(DM dm, DMLabel label)
128 {
129   IS              valueIS;
130   const PetscInt *values;
131   PetscInt        numValues, v, cStart, cEnd, cEndInterior;
132   PetscErrorCode  ierr;
133 
134   PetscFunctionBegin;
135   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
136   ierr = DMPlexGetHybridBounds(dm, &cEndInterior, NULL, NULL, NULL);CHKERRQ(ierr);
137   cEnd = cEndInterior < 0 ? cEnd : cEndInterior;
138   ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
139   ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
140   ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
141   for (v = 0; v < numValues; ++v) {
142     IS              pointIS;
143     const PetscInt *points;
144     PetscInt        numPoints, p;
145 
146     ierr = DMLabelGetStratumSize(label, values[v], &numPoints);CHKERRQ(ierr);
147     ierr = DMLabelGetStratumIS(label, values[v], &pointIS);CHKERRQ(ierr);
148     ierr = ISGetIndices(pointIS, &points);CHKERRQ(ierr);
149     for (p = 0; p < numPoints; ++p) {
150       PetscInt *closure = NULL;
151       PetscInt  closureSize, point, cl;
152 
153       ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_FALSE, &closureSize, &closure);CHKERRQ(ierr);
154       for (cl = closureSize-1; cl > 0; --cl) {
155         point = closure[cl*2];
156         if ((point >= cStart) && (point < cEnd)) {ierr = DMLabelSetValue(label, point, values[v]);CHKERRQ(ierr); break;}
157       }
158       ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_FALSE, &closureSize, &closure);CHKERRQ(ierr);
159     }
160     ierr = ISRestoreIndices(pointIS, &points);CHKERRQ(ierr);
161     ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
162   }
163   ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
164   ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
165   PetscFunctionReturn(0);
166 }
167 
168 /*@
169   DMPlexLabelClearCells - Remove cells from a label
170 
171   Input Parameters:
172 + dm - The DM
173 - label - A DMLabel marking surface points and their adjacent cells
174 
175   Output Parameter:
176 . label - A DMLabel without cells
177 
178   Level: developer
179 
180   Note: This undoes DMPlexLabelAddCells()
181 
182 .seealso: DMPlexLabelComplete(), DMPlexLabelCohesiveComplete(), DMPlexLabelAddCells()
183 @*/
184 PetscErrorCode DMPlexLabelClearCells(DM dm, DMLabel label)
185 {
186   IS              valueIS;
187   const PetscInt *values;
188   PetscInt        numValues, v, cStart, cEnd, cEndInterior;
189   PetscErrorCode  ierr;
190 
191   PetscFunctionBegin;
192   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
193   ierr = DMPlexGetHybridBounds(dm, &cEndInterior, NULL, NULL, NULL);CHKERRQ(ierr);
194   cEnd = cEndInterior < 0 ? cEnd : cEndInterior;
195   ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
196   ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
197   ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
198   for (v = 0; v < numValues; ++v) {
199     IS              pointIS;
200     const PetscInt *points;
201     PetscInt        numPoints, p;
202 
203     ierr = DMLabelGetStratumSize(label, values[v], &numPoints);CHKERRQ(ierr);
204     ierr = DMLabelGetStratumIS(label, values[v], &pointIS);CHKERRQ(ierr);
205     ierr = ISGetIndices(pointIS, &points);CHKERRQ(ierr);
206     for (p = 0; p < numPoints; ++p) {
207       PetscInt point = points[p];
208 
209       if (point >= cStart && point < cEnd) {
210         ierr = DMLabelClearValue(label,point,values[v]);CHKERRQ(ierr);
211       }
212     }
213     ierr = ISRestoreIndices(pointIS, &points);CHKERRQ(ierr);
214     ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
215   }
216   ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
217   ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
218   PetscFunctionReturn(0);
219 }
220 
221 /* take (oldEnd, added) pairs, ordered by height and convert them to (oldstart, newstart) pairs, ordered by ascending
222  * index (skipping first, which is (0,0)) */
223 PETSC_STATIC_INLINE PetscErrorCode DMPlexShiftPointSetUp_Internal(PetscInt depth, PetscInt depthShift[])
224 {
225   PetscInt d, off = 0;
226 
227   PetscFunctionBegin;
228   /* sort by (oldend): yes this is an O(n^2) sort, we expect depth <= 3 */
229   for (d = 0; d < depth; d++) {
230     PetscInt firstd = d;
231     PetscInt firstStart = depthShift[2*d];
232     PetscInt e;
233 
234     for (e = d+1; e <= depth; e++) {
235       if (depthShift[2*e] < firstStart) {
236         firstd = e;
237         firstStart = depthShift[2*d];
238       }
239     }
240     if (firstd != d) {
241       PetscInt swap[2];
242 
243       e = firstd;
244       swap[0] = depthShift[2*d];
245       swap[1] = depthShift[2*d+1];
246       depthShift[2*d]   = depthShift[2*e];
247       depthShift[2*d+1] = depthShift[2*e+1];
248       depthShift[2*e]   = swap[0];
249       depthShift[2*e+1] = swap[1];
250     }
251   }
252   /* convert (oldstart, added) to (oldstart, newstart) */
253   for (d = 0; d <= depth; d++) {
254     off += depthShift[2*d+1];
255     depthShift[2*d+1] = depthShift[2*d] + off;
256   }
257   PetscFunctionReturn(0);
258 }
259 
260 /* depthShift is a list of (old, new) pairs */
261 PETSC_STATIC_INLINE PetscInt DMPlexShiftPoint_Internal(PetscInt p, PetscInt depth, PetscInt depthShift[])
262 {
263   PetscInt d;
264   PetscInt newOff = 0;
265 
266   for (d = 0; d <= depth; d++) {
267     if (p < depthShift[2*d]) return p + newOff;
268     else newOff = depthShift[2*d+1] - depthShift[2*d];
269   }
270   return p + newOff;
271 }
272 
273 /* depthShift is a list of (old, new) pairs */
274 PETSC_STATIC_INLINE PetscInt DMPlexShiftPointInverse_Internal(PetscInt p, PetscInt depth, PetscInt depthShift[])
275 {
276   PetscInt d;
277   PetscInt newOff = 0;
278 
279   for (d = 0; d <= depth; d++) {
280     if (p < depthShift[2*d+1]) return p + newOff;
281     else newOff = depthShift[2*d] - depthShift[2*d+1];
282   }
283   return p + newOff;
284 }
285 
286 static PetscErrorCode DMPlexShiftSizes_Internal(DM dm, PetscInt depthShift[], DM dmNew)
287 {
288   PetscInt       depth = 0, d, pStart, pEnd, p;
289   PetscErrorCode ierr;
290 
291   PetscFunctionBegin;
292   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
293   if (depth < 0) PetscFunctionReturn(0);
294   /* Step 1: Expand chart */
295   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
296   pEnd = DMPlexShiftPoint_Internal(pEnd,depth,depthShift);
297   ierr = DMPlexSetChart(dmNew, pStart, pEnd);CHKERRQ(ierr);
298   /* Step 2: Set cone and support sizes */
299   for (d = 0; d <= depth; ++d) {
300     ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
301     for (p = pStart; p < pEnd; ++p) {
302       PetscInt newp = DMPlexShiftPoint_Internal(p, depth, depthShift);
303       PetscInt size;
304 
305       ierr = DMPlexGetConeSize(dm, p, &size);CHKERRQ(ierr);
306       ierr = DMPlexSetConeSize(dmNew, newp, size);CHKERRQ(ierr);
307       ierr = DMPlexGetSupportSize(dm, p, &size);CHKERRQ(ierr);
308       ierr = DMPlexSetSupportSize(dmNew, newp, size);CHKERRQ(ierr);
309     }
310   }
311   PetscFunctionReturn(0);
312 }
313 
314 static PetscErrorCode DMPlexShiftPoints_Internal(DM dm, PetscInt depthShift[], DM dmNew)
315 {
316   PetscInt      *newpoints;
317   PetscInt       depth = 0, maxConeSize, maxSupportSize, maxConeSizeNew, maxSupportSizeNew, pStart, pEnd, p;
318   PetscErrorCode ierr;
319 
320   PetscFunctionBegin;
321   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
322   if (depth < 0) PetscFunctionReturn(0);
323   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);CHKERRQ(ierr);
324   ierr = DMPlexGetMaxSizes(dmNew, &maxConeSizeNew, &maxSupportSizeNew);CHKERRQ(ierr);
325   ierr = PetscMalloc1(PetscMax(PetscMax(maxConeSize, maxSupportSize), PetscMax(maxConeSizeNew, maxSupportSizeNew)),&newpoints);CHKERRQ(ierr);
326   /* Step 5: Set cones and supports */
327   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
328   for (p = pStart; p < pEnd; ++p) {
329     const PetscInt *points = NULL, *orientations = NULL;
330     PetscInt        size,sizeNew, i, newp = DMPlexShiftPoint_Internal(p, depth, depthShift);
331 
332     ierr = DMPlexGetConeSize(dm, p, &size);CHKERRQ(ierr);
333     ierr = DMPlexGetCone(dm, p, &points);CHKERRQ(ierr);
334     ierr = DMPlexGetConeOrientation(dm, p, &orientations);CHKERRQ(ierr);
335     for (i = 0; i < size; ++i) {
336       newpoints[i] = DMPlexShiftPoint_Internal(points[i], depth, depthShift);
337     }
338     ierr = DMPlexSetCone(dmNew, newp, newpoints);CHKERRQ(ierr);
339     ierr = DMPlexSetConeOrientation(dmNew, newp, orientations);CHKERRQ(ierr);
340     ierr = DMPlexGetSupportSize(dm, p, &size);CHKERRQ(ierr);
341     ierr = DMPlexGetSupportSize(dmNew, newp, &sizeNew);CHKERRQ(ierr);
342     ierr = DMPlexGetSupport(dm, p, &points);CHKERRQ(ierr);
343     for (i = 0; i < size; ++i) {
344       newpoints[i] = DMPlexShiftPoint_Internal(points[i], depth, depthShift);
345     }
346     for (i = size; i < sizeNew; ++i) newpoints[i] = 0;
347     ierr = DMPlexSetSupport(dmNew, newp, newpoints);CHKERRQ(ierr);
348   }
349   ierr = PetscFree(newpoints);CHKERRQ(ierr);
350   PetscFunctionReturn(0);
351 }
352 
353 static PetscErrorCode DMPlexShiftCoordinates_Internal(DM dm, PetscInt depthShift[], DM dmNew)
354 {
355   PetscSection   coordSection, newCoordSection;
356   Vec            coordinates, newCoordinates;
357   PetscScalar   *coords, *newCoords;
358   PetscInt       coordSize, sStart, sEnd;
359   PetscInt       dim, depth = 0, cStart, cEnd, cStartNew, cEndNew, c, vStart, vEnd, vStartNew, vEndNew, v;
360   PetscBool      hasCells;
361   PetscErrorCode ierr;
362 
363   PetscFunctionBegin;
364   ierr = DMGetCoordinateDim(dm, &dim);CHKERRQ(ierr);
365   ierr = DMSetCoordinateDim(dmNew, dim);CHKERRQ(ierr);
366   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
367   /* Step 8: Convert coordinates */
368   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
369   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
370   ierr = DMPlexGetDepthStratum(dmNew, 0, &vStartNew, &vEndNew);CHKERRQ(ierr);
371   ierr = DMPlexGetHeightStratum(dmNew, 0, &cStartNew, &cEndNew);CHKERRQ(ierr);
372   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
373   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &newCoordSection);CHKERRQ(ierr);
374   ierr = PetscSectionSetNumFields(newCoordSection, 1);CHKERRQ(ierr);
375   ierr = PetscSectionSetFieldComponents(newCoordSection, 0, dim);CHKERRQ(ierr);
376   ierr = PetscSectionGetChart(coordSection, &sStart, &sEnd);CHKERRQ(ierr);
377   hasCells = sStart == cStart ? PETSC_TRUE : PETSC_FALSE;
378   ierr = PetscSectionSetChart(newCoordSection, hasCells ? cStartNew : vStartNew, vEndNew);CHKERRQ(ierr);
379   if (hasCells) {
380     for (c = cStart; c < cEnd; ++c) {
381       PetscInt cNew = DMPlexShiftPoint_Internal(c, depth, depthShift), dof;
382 
383       ierr = PetscSectionGetDof(coordSection, c, &dof);CHKERRQ(ierr);
384       ierr = PetscSectionSetDof(newCoordSection, cNew, dof);CHKERRQ(ierr);
385       ierr = PetscSectionSetFieldDof(newCoordSection, cNew, 0, dof);CHKERRQ(ierr);
386     }
387   }
388   for (v = vStartNew; v < vEndNew; ++v) {
389     ierr = PetscSectionSetDof(newCoordSection, v, dim);CHKERRQ(ierr);
390     ierr = PetscSectionSetFieldDof(newCoordSection, v, 0, dim);CHKERRQ(ierr);
391   }
392   ierr = PetscSectionSetUp(newCoordSection);CHKERRQ(ierr);
393   ierr = DMSetCoordinateSection(dmNew, PETSC_DETERMINE, newCoordSection);CHKERRQ(ierr);
394   ierr = PetscSectionGetStorageSize(newCoordSection, &coordSize);CHKERRQ(ierr);
395   ierr = VecCreate(PETSC_COMM_SELF, &newCoordinates);CHKERRQ(ierr);
396   ierr = PetscObjectSetName((PetscObject) newCoordinates, "coordinates");CHKERRQ(ierr);
397   ierr = VecSetSizes(newCoordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
398   ierr = VecSetBlockSize(newCoordinates, dim);CHKERRQ(ierr);
399   ierr = VecSetType(newCoordinates,VECSTANDARD);CHKERRQ(ierr);
400   ierr = DMSetCoordinatesLocal(dmNew, newCoordinates);CHKERRQ(ierr);
401   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
402   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
403   ierr = VecGetArray(newCoordinates, &newCoords);CHKERRQ(ierr);
404   if (hasCells) {
405     for (c = cStart; c < cEnd; ++c) {
406       PetscInt cNew = DMPlexShiftPoint_Internal(c, depth, depthShift), dof, off, noff, d;
407 
408       ierr = PetscSectionGetDof(coordSection, c, &dof);CHKERRQ(ierr);
409       ierr = PetscSectionGetOffset(coordSection, c, &off);CHKERRQ(ierr);
410       ierr = PetscSectionGetOffset(newCoordSection, cNew, &noff);CHKERRQ(ierr);
411       for (d = 0; d < dof; ++d) newCoords[noff+d] = coords[off+d];
412     }
413   }
414   for (v = vStart; v < vEnd; ++v) {
415     PetscInt dof, off, noff, d;
416 
417     ierr = PetscSectionGetDof(coordSection, v, &dof);CHKERRQ(ierr);
418     ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
419     ierr = PetscSectionGetOffset(newCoordSection, DMPlexShiftPoint_Internal(v, depth, depthShift), &noff);CHKERRQ(ierr);
420     for (d = 0; d < dof; ++d) newCoords[noff+d] = coords[off+d];
421   }
422   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
423   ierr = VecRestoreArray(newCoordinates, &newCoords);CHKERRQ(ierr);
424   ierr = VecDestroy(&newCoordinates);CHKERRQ(ierr);
425   ierr = PetscSectionDestroy(&newCoordSection);CHKERRQ(ierr);
426   PetscFunctionReturn(0);
427 }
428 
429 static PetscErrorCode DMPlexShiftSF_Internal(DM dm, PetscInt depthShift[], DM dmNew)
430 {
431   PetscInt           depth = 0;
432   PetscSF            sfPoint, sfPointNew;
433   const PetscSFNode *remotePoints;
434   PetscSFNode       *gremotePoints;
435   const PetscInt    *localPoints;
436   PetscInt          *glocalPoints, *newLocation, *newRemoteLocation;
437   PetscInt           numRoots, numLeaves, l, pStart, pEnd, totShift = 0;
438   PetscErrorCode     ierr;
439 
440   PetscFunctionBegin;
441   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
442   /* Step 9: Convert pointSF */
443   ierr = DMGetPointSF(dm, &sfPoint);CHKERRQ(ierr);
444   ierr = DMGetPointSF(dmNew, &sfPointNew);CHKERRQ(ierr);
445   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
446   ierr = PetscSFGetGraph(sfPoint, &numRoots, &numLeaves, &localPoints, &remotePoints);CHKERRQ(ierr);
447   totShift = DMPlexShiftPoint_Internal(pEnd,depth,depthShift) - pEnd;
448   if (numRoots >= 0) {
449     ierr = PetscMalloc2(numRoots,&newLocation,pEnd-pStart,&newRemoteLocation);CHKERRQ(ierr);
450     for (l=0; l<numRoots; l++) newLocation[l] = DMPlexShiftPoint_Internal(l, depth, depthShift);
451     ierr = PetscSFBcastBegin(sfPoint, MPIU_INT, newLocation, newRemoteLocation);CHKERRQ(ierr);
452     ierr = PetscSFBcastEnd(sfPoint, MPIU_INT, newLocation, newRemoteLocation);CHKERRQ(ierr);
453     ierr = PetscMalloc1(numLeaves,    &glocalPoints);CHKERRQ(ierr);
454     ierr = PetscMalloc1(numLeaves, &gremotePoints);CHKERRQ(ierr);
455     for (l = 0; l < numLeaves; ++l) {
456       glocalPoints[l]        = DMPlexShiftPoint_Internal(localPoints[l], depth, depthShift);
457       gremotePoints[l].rank  = remotePoints[l].rank;
458       gremotePoints[l].index = newRemoteLocation[localPoints[l]];
459     }
460     ierr = PetscFree2(newLocation,newRemoteLocation);CHKERRQ(ierr);
461     ierr = PetscSFSetGraph(sfPointNew, numRoots + totShift, numLeaves, glocalPoints, PETSC_OWN_POINTER, gremotePoints, PETSC_OWN_POINTER);CHKERRQ(ierr);
462   }
463   PetscFunctionReturn(0);
464 }
465 
466 static PetscErrorCode DMPlexShiftLabels_Internal(DM dm, PetscInt depthShift[], DM dmNew)
467 {
468   PetscSF            sfPoint;
469   DMLabel            vtkLabel, ghostLabel;
470   const PetscSFNode *leafRemote;
471   const PetscInt    *leafLocal;
472   PetscInt           depth = 0, numLeaves, numLabels, l, cStart, cEnd, c, fStart, fEnd, f;
473   PetscMPIInt        rank;
474   PetscErrorCode     ierr;
475 
476   PetscFunctionBegin;
477   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
478   /* Step 10: Convert labels */
479   ierr = DMGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
480   for (l = 0; l < numLabels; ++l) {
481     DMLabel         label, newlabel;
482     const char     *lname;
483     PetscBool       isDepth;
484     IS              valueIS;
485     const PetscInt *values;
486     PetscInt        numValues, val;
487 
488     ierr = DMGetLabelName(dm, l, &lname);CHKERRQ(ierr);
489     ierr = PetscStrcmp(lname, "depth", &isDepth);CHKERRQ(ierr);
490     if (isDepth) continue;
491     ierr = DMCreateLabel(dmNew, lname);CHKERRQ(ierr);
492     ierr = DMGetLabel(dm, lname, &label);CHKERRQ(ierr);
493     ierr = DMGetLabel(dmNew, lname, &newlabel);CHKERRQ(ierr);
494     ierr = DMLabelGetDefaultValue(label,&val);CHKERRQ(ierr);
495     ierr = DMLabelSetDefaultValue(newlabel,val);CHKERRQ(ierr);
496     ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
497     ierr = ISGetLocalSize(valueIS, &numValues);CHKERRQ(ierr);
498     ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
499     for (val = 0; val < numValues; ++val) {
500       IS              pointIS;
501       const PetscInt *points;
502       PetscInt        numPoints, p;
503 
504       ierr = DMLabelGetStratumIS(label, values[val], &pointIS);CHKERRQ(ierr);
505       ierr = ISGetLocalSize(pointIS, &numPoints);CHKERRQ(ierr);
506       ierr = ISGetIndices(pointIS, &points);CHKERRQ(ierr);
507       for (p = 0; p < numPoints; ++p) {
508         const PetscInt newpoint = DMPlexShiftPoint_Internal(points[p], depth, depthShift);
509 
510         ierr = DMLabelSetValue(newlabel, newpoint, values[val]);CHKERRQ(ierr);
511       }
512       ierr = ISRestoreIndices(pointIS, &points);CHKERRQ(ierr);
513       ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
514     }
515     ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
516     ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
517   }
518   /* Step 11: Make label for output (vtk) and to mark ghost points (ghost) */
519   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
520   ierr = DMGetPointSF(dm, &sfPoint);CHKERRQ(ierr);
521   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
522   ierr = PetscSFGetGraph(sfPoint, NULL, &numLeaves, &leafLocal, &leafRemote);CHKERRQ(ierr);
523   ierr = DMCreateLabel(dmNew, "vtk");CHKERRQ(ierr);
524   ierr = DMCreateLabel(dmNew, "ghost");CHKERRQ(ierr);
525   ierr = DMGetLabel(dmNew, "vtk", &vtkLabel);CHKERRQ(ierr);
526   ierr = DMGetLabel(dmNew, "ghost", &ghostLabel);CHKERRQ(ierr);
527   for (l = 0, c = cStart; l < numLeaves && c < cEnd; ++l, ++c) {
528     for (; c < leafLocal[l] && c < cEnd; ++c) {
529       ierr = DMLabelSetValue(vtkLabel, c, 1);CHKERRQ(ierr);
530     }
531     if (leafLocal[l] >= cEnd) break;
532     if (leafRemote[l].rank == rank) {
533       ierr = DMLabelSetValue(vtkLabel, c, 1);CHKERRQ(ierr);
534     } else {
535       ierr = DMLabelSetValue(ghostLabel, c, 2);CHKERRQ(ierr);
536     }
537   }
538   for (; c < cEnd; ++c) {
539     ierr = DMLabelSetValue(vtkLabel, c, 1);CHKERRQ(ierr);
540   }
541   if (0) {
542     ierr = DMLabelView(vtkLabel, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
543   }
544   ierr = DMPlexGetHeightStratum(dmNew, 1, &fStart, &fEnd);CHKERRQ(ierr);
545   for (f = fStart; f < fEnd; ++f) {
546     PetscInt numCells;
547 
548     ierr = DMPlexGetSupportSize(dmNew, f, &numCells);CHKERRQ(ierr);
549     if (numCells < 2) {
550       ierr = DMLabelSetValue(ghostLabel, f, 1);CHKERRQ(ierr);
551     } else {
552       const PetscInt *cells = NULL;
553       PetscInt        vA, vB;
554 
555       ierr = DMPlexGetSupport(dmNew, f, &cells);CHKERRQ(ierr);
556       ierr = DMLabelGetValue(vtkLabel, cells[0], &vA);CHKERRQ(ierr);
557       ierr = DMLabelGetValue(vtkLabel, cells[1], &vB);CHKERRQ(ierr);
558       if (vA != 1 && vB != 1) {ierr = DMLabelSetValue(ghostLabel, f, 1);CHKERRQ(ierr);}
559     }
560   }
561   if (0) {
562     ierr = DMLabelView(ghostLabel, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
563   }
564   PetscFunctionReturn(0);
565 }
566 
567 static PetscErrorCode DMPlexShiftTree_Internal(DM dm, PetscInt depthShift[], DM dmNew)
568 {
569   DM             refTree;
570   PetscSection   pSec;
571   PetscInt       *parents, *childIDs;
572   PetscErrorCode ierr;
573 
574   PetscFunctionBegin;
575   ierr = DMPlexGetReferenceTree(dm,&refTree);CHKERRQ(ierr);
576   ierr = DMPlexSetReferenceTree(dmNew,refTree);CHKERRQ(ierr);
577   ierr = DMPlexGetTree(dm,&pSec,&parents,&childIDs,NULL,NULL);CHKERRQ(ierr);
578   if (pSec) {
579     PetscInt p, pStart, pEnd, *parentsShifted, pStartShifted, pEndShifted, depth;
580     PetscInt *childIDsShifted;
581     PetscSection pSecShifted;
582 
583     ierr = PetscSectionGetChart(pSec,&pStart,&pEnd);CHKERRQ(ierr);
584     ierr = DMPlexGetDepth(dm,&depth);CHKERRQ(ierr);
585     pStartShifted = DMPlexShiftPoint_Internal(pStart,depth,depthShift);
586     pEndShifted   = DMPlexShiftPoint_Internal(pEnd,depth,depthShift);
587     ierr = PetscMalloc2(pEndShifted - pStartShifted,&parentsShifted,pEndShifted-pStartShifted,&childIDsShifted);CHKERRQ(ierr);
588     ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dmNew),&pSecShifted);CHKERRQ(ierr);
589     ierr = PetscSectionSetChart(pSecShifted,pStartShifted,pEndShifted);CHKERRQ(ierr);
590     for (p = pStartShifted; p < pEndShifted; p++) {
591       /* start off assuming no children */
592       ierr = PetscSectionSetDof(pSecShifted,p,0);CHKERRQ(ierr);
593     }
594     for (p = pStart; p < pEnd; p++) {
595       PetscInt dof;
596       PetscInt pNew = DMPlexShiftPoint_Internal(p,depth,depthShift);
597 
598       ierr = PetscSectionGetDof(pSec,p,&dof);CHKERRQ(ierr);
599       ierr = PetscSectionSetDof(pSecShifted,pNew,dof);CHKERRQ(ierr);
600     }
601     ierr = PetscSectionSetUp(pSecShifted);CHKERRQ(ierr);
602     for (p = pStart; p < pEnd; p++) {
603       PetscInt dof;
604       PetscInt pNew = DMPlexShiftPoint_Internal(p,depth,depthShift);
605 
606       ierr = PetscSectionGetDof(pSec,p,&dof);CHKERRQ(ierr);
607       if (dof) {
608         PetscInt off, offNew;
609 
610         ierr = PetscSectionGetOffset(pSec,p,&off);CHKERRQ(ierr);
611         ierr = PetscSectionGetOffset(pSecShifted,pNew,&offNew);CHKERRQ(ierr);
612         parentsShifted[offNew] = DMPlexShiftPoint_Internal(parents[off],depth,depthShift);
613         childIDsShifted[offNew] = childIDs[off];
614       }
615     }
616     ierr = DMPlexSetTree(dmNew,pSecShifted,parentsShifted,childIDsShifted);CHKERRQ(ierr);
617     ierr = PetscFree2(parentsShifted,childIDsShifted);CHKERRQ(ierr);
618     ierr = PetscSectionDestroy(&pSecShifted);CHKERRQ(ierr);
619   }
620   PetscFunctionReturn(0);
621 }
622 
623 static PetscErrorCode DMPlexConstructGhostCells_Internal(DM dm, DMLabel label, PetscInt *numGhostCells, DM gdm)
624 {
625   PetscSF         sf;
626   IS              valueIS;
627   const PetscInt *values, *leaves;
628   PetscInt       *depthShift;
629   PetscInt        d, depth = 0, nleaves, loc, Ng, numFS, fs, fStart, fEnd, ghostCell, cEnd, c;
630   PetscErrorCode  ierr;
631 
632   PetscFunctionBegin;
633   ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
634   ierr = PetscSFGetGraph(sf, NULL, &nleaves, &leaves, NULL);CHKERRQ(ierr);
635   nleaves = PetscMax(0, nleaves);
636   ierr = DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd);CHKERRQ(ierr);
637   /* Count ghost cells */
638   ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
639   ierr = ISGetLocalSize(valueIS, &numFS);CHKERRQ(ierr);
640   ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
641   Ng   = 0;
642   for (fs = 0; fs < numFS; ++fs) {
643     IS              faceIS;
644     const PetscInt *faces;
645     PetscInt        numFaces, f, numBdFaces = 0;
646 
647     ierr = DMLabelGetStratumIS(label, values[fs], &faceIS);CHKERRQ(ierr);
648     ierr = ISGetLocalSize(faceIS, &numFaces);CHKERRQ(ierr);
649     ierr = ISGetIndices(faceIS, &faces);CHKERRQ(ierr);
650     for (f = 0; f < numFaces; ++f) {
651       PetscInt numChildren;
652 
653       ierr = PetscFindInt(faces[f], nleaves, leaves, &loc);CHKERRQ(ierr);
654       ierr = DMPlexGetTreeChildren(dm,faces[f],&numChildren,NULL);CHKERRQ(ierr);
655       /* non-local and ancestors points don't get to register ghosts */
656       if (loc >= 0 || numChildren) continue;
657       if ((faces[f] >= fStart) && (faces[f] < fEnd)) ++numBdFaces;
658     }
659     Ng += numBdFaces;
660     ierr = ISDestroy(&faceIS);CHKERRQ(ierr);
661   }
662   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
663   ierr = PetscMalloc1(2*(depth+1), &depthShift);CHKERRQ(ierr);
664   for (d = 0; d <= depth; d++) {
665     PetscInt dEnd;
666 
667     ierr = DMPlexGetDepthStratum(dm,d,NULL,&dEnd);CHKERRQ(ierr);
668     depthShift[2*d]   = dEnd;
669     depthShift[2*d+1] = 0;
670   }
671   if (depth >= 0) depthShift[2*depth+1] = Ng;
672   ierr = DMPlexShiftPointSetUp_Internal(depth,depthShift);CHKERRQ(ierr);
673   ierr = DMPlexShiftSizes_Internal(dm, depthShift, gdm);CHKERRQ(ierr);
674   /* Step 3: Set cone/support sizes for new points */
675   ierr = DMPlexGetHeightStratum(dm, 0, NULL, &cEnd);CHKERRQ(ierr);
676   ierr = DMPlexSetHybridBounds(gdm, cEnd, PETSC_DETERMINE, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
677   for (c = cEnd; c < cEnd + Ng; ++c) {
678     ierr = DMPlexSetConeSize(gdm, c, 1);CHKERRQ(ierr);
679   }
680   for (fs = 0; fs < numFS; ++fs) {
681     IS              faceIS;
682     const PetscInt *faces;
683     PetscInt        numFaces, f;
684 
685     ierr = DMLabelGetStratumIS(label, values[fs], &faceIS);CHKERRQ(ierr);
686     ierr = ISGetLocalSize(faceIS, &numFaces);CHKERRQ(ierr);
687     ierr = ISGetIndices(faceIS, &faces);CHKERRQ(ierr);
688     for (f = 0; f < numFaces; ++f) {
689       PetscInt size, numChildren;
690 
691       ierr = PetscFindInt(faces[f], nleaves, leaves, &loc);CHKERRQ(ierr);
692       ierr = DMPlexGetTreeChildren(dm,faces[f],&numChildren,NULL);CHKERRQ(ierr);
693       if (loc >= 0 || numChildren) continue;
694       if ((faces[f] < fStart) || (faces[f] >= fEnd)) continue;
695       ierr = DMPlexGetSupportSize(dm, faces[f], &size);CHKERRQ(ierr);
696       if (size != 1) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM has boundary face %d with %d support cells", faces[f], size);
697       ierr = DMPlexSetSupportSize(gdm, faces[f] + Ng, 2);CHKERRQ(ierr);
698     }
699     ierr = ISRestoreIndices(faceIS, &faces);CHKERRQ(ierr);
700     ierr = ISDestroy(&faceIS);CHKERRQ(ierr);
701   }
702   /* Step 4: Setup ghosted DM */
703   ierr = DMSetUp(gdm);CHKERRQ(ierr);
704   ierr = DMPlexShiftPoints_Internal(dm, depthShift, gdm);CHKERRQ(ierr);
705   /* Step 6: Set cones and supports for new points */
706   ghostCell = cEnd;
707   for (fs = 0; fs < numFS; ++fs) {
708     IS              faceIS;
709     const PetscInt *faces;
710     PetscInt        numFaces, f;
711 
712     ierr = DMLabelGetStratumIS(label, values[fs], &faceIS);CHKERRQ(ierr);
713     ierr = ISGetLocalSize(faceIS, &numFaces);CHKERRQ(ierr);
714     ierr = ISGetIndices(faceIS, &faces);CHKERRQ(ierr);
715     for (f = 0; f < numFaces; ++f) {
716       PetscInt newFace = faces[f] + Ng, numChildren;
717 
718       ierr = PetscFindInt(faces[f], nleaves, leaves, &loc);CHKERRQ(ierr);
719       ierr = DMPlexGetTreeChildren(dm,faces[f],&numChildren,NULL);CHKERRQ(ierr);
720       if (loc >= 0 || numChildren) continue;
721       if ((faces[f] < fStart) || (faces[f] >= fEnd)) continue;
722       ierr = DMPlexSetCone(gdm, ghostCell, &newFace);CHKERRQ(ierr);
723       ierr = DMPlexInsertSupport(gdm, newFace, 1, ghostCell);CHKERRQ(ierr);
724       ++ghostCell;
725     }
726     ierr = ISRestoreIndices(faceIS, &faces);CHKERRQ(ierr);
727     ierr = ISDestroy(&faceIS);CHKERRQ(ierr);
728   }
729   ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
730   ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
731   /* Step 7: Stratify */
732   ierr = DMPlexStratify(gdm);CHKERRQ(ierr);
733   ierr = DMPlexShiftCoordinates_Internal(dm, depthShift, gdm);CHKERRQ(ierr);
734   ierr = DMPlexShiftSF_Internal(dm, depthShift, gdm);CHKERRQ(ierr);
735   ierr = DMPlexShiftLabels_Internal(dm, depthShift, gdm);CHKERRQ(ierr);
736   ierr = DMPlexShiftTree_Internal(dm, depthShift, gdm);CHKERRQ(ierr);
737   ierr = PetscFree(depthShift);CHKERRQ(ierr);
738   /* Step 7: Periodicity */
739   if (dm->maxCell) {
740     const PetscReal *maxCell, *L;
741     const DMBoundaryType *bd;
742     ierr = DMGetPeriodicity(dm,  &maxCell, &L, &bd);CHKERRQ(ierr);
743     ierr = DMSetPeriodicity(gdm,  maxCell,  L,  bd);CHKERRQ(ierr);
744   }
745   if (numGhostCells) *numGhostCells = Ng;
746   PetscFunctionReturn(0);
747 }
748 
749 /*@C
750   DMPlexConstructGhostCells - Construct ghost cells which connect to every boundary face
751 
752   Collective on dm
753 
754   Input Parameters:
755 + dm - The original DM
756 - labelName - The label specifying the boundary faces, or "Face Sets" if this is NULL
757 
758   Output Parameters:
759 + numGhostCells - The number of ghost cells added to the DM
760 - dmGhosted - The new DM
761 
762   Note: If no label exists of that name, one will be created marking all boundary faces
763 
764   Level: developer
765 
766 .seealso: DMCreate()
767 @*/
768 PetscErrorCode DMPlexConstructGhostCells(DM dm, const char labelName[], PetscInt *numGhostCells, DM *dmGhosted)
769 {
770   DM             gdm;
771   DMLabel        label;
772   const char    *name = labelName ? labelName : "Face Sets";
773   PetscInt       dim;
774   PetscBool      flag;
775   PetscErrorCode ierr;
776 
777   PetscFunctionBegin;
778   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
779   if (numGhostCells) PetscValidPointer(numGhostCells, 3);
780   PetscValidPointer(dmGhosted, 4);
781   ierr = DMCreate(PetscObjectComm((PetscObject)dm), &gdm);CHKERRQ(ierr);
782   ierr = DMSetType(gdm, DMPLEX);CHKERRQ(ierr);
783   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
784   ierr = DMSetDimension(gdm, dim);CHKERRQ(ierr);
785   ierr = DMPlexGetAdjacencyUseCone(dm, &flag);CHKERRQ(ierr);
786   ierr = DMPlexSetAdjacencyUseCone(gdm, flag);CHKERRQ(ierr);
787   ierr = DMPlexGetAdjacencyUseClosure(dm, &flag);CHKERRQ(ierr);
788   ierr = DMPlexSetAdjacencyUseClosure(gdm, flag);CHKERRQ(ierr);
789   ierr = DMGetLabel(dm, name, &label);CHKERRQ(ierr);
790   if (!label) {
791     /* Get label for boundary faces */
792     ierr = DMCreateLabel(dm, name);CHKERRQ(ierr);
793     ierr = DMGetLabel(dm, name, &label);CHKERRQ(ierr);
794     ierr = DMPlexMarkBoundaryFaces(dm, label);CHKERRQ(ierr);
795   }
796   ierr = DMPlexConstructGhostCells_Internal(dm, label, numGhostCells, gdm);CHKERRQ(ierr);
797   ierr = DMCopyBoundary(dm, gdm);CHKERRQ(ierr);
798   *dmGhosted = gdm;
799   PetscFunctionReturn(0);
800 }
801 
802 /*
803   We are adding three kinds of points here:
804     Replicated:     Copies of points which exist in the mesh, such as vertices identified across a fault
805     Non-replicated: Points which exist on the fault, but are not replicated
806     Hybrid:         Entirely new points, such as cohesive cells
807 
808   When creating subsequent cohesive cells, we shift the old hybrid cells to the end of the numbering at
809   each depth so that the new split/hybrid points can be inserted as a block.
810 */
811 static PetscErrorCode DMPlexConstructCohesiveCells_Internal(DM dm, DMLabel label, DM sdm)
812 {
813   MPI_Comm         comm;
814   IS               valueIS;
815   PetscInt         numSP = 0;       /* The number of depths for which we have replicated points */
816   const PetscInt  *values;          /* List of depths for which we have replicated points */
817   IS              *splitIS;
818   IS              *unsplitIS;
819   PetscInt        *numSplitPoints;     /* The number of replicated points at each depth */
820   PetscInt        *numUnsplitPoints;   /* The number of non-replicated points at each depth which still give rise to hybrid points */
821   PetscInt        *numHybridPoints;    /* The number of new hybrid points at each depth */
822   PetscInt        *numHybridPointsOld; /* The number of existing hybrid points at each depth */
823   const PetscInt **splitPoints;        /* Replicated points for each depth */
824   const PetscInt **unsplitPoints;      /* Non-replicated points for each depth */
825   PetscSection     coordSection;
826   Vec              coordinates;
827   PetscScalar     *coords;
828   PetscInt        *depthMax;           /* The first hybrid point at each depth in the original mesh */
829   PetscInt        *depthEnd;           /* The point limit at each depth in the original mesh */
830   PetscInt        *depthShift;         /* Number of replicated+hybrid points at each depth */
831   PetscInt        *pMaxNew;            /* The first replicated point at each depth in the new mesh, hybrids come after this */
832   PetscInt        *coneNew, *coneONew, *supportNew;
833   PetscInt         shift = 100, shift2 = 200, depth = 0, dep, dim, d, sp, maxConeSize, maxSupportSize, maxConeSizeNew, maxSupportSizeNew, numLabels, vStart, vEnd, pEnd, p, v;
834   PetscErrorCode   ierr;
835 
836   PetscFunctionBegin;
837   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
838   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
839   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
840   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
841   /* Count split points and add cohesive cells */
842   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);CHKERRQ(ierr);
843   ierr = PetscMalloc5(depth+1,&depthMax,depth+1,&depthEnd,2*(depth+1),&depthShift,depth+1,&pMaxNew,depth+1,&numHybridPointsOld);CHKERRQ(ierr);
844   ierr = PetscMalloc7(depth+1,&splitIS,depth+1,&unsplitIS,depth+1,&numSplitPoints,depth+1,&numUnsplitPoints,depth+1,&numHybridPoints,depth+1,&splitPoints,depth+1,&unsplitPoints);CHKERRQ(ierr);
845   ierr = DMPlexGetHybridBounds(dm, depth >= 0 ? &depthMax[depth] : NULL, depth>1 ? &depthMax[depth-1] : NULL, depth>2 ? &depthMax[1] : NULL, depth >= 0 ? &depthMax[0] : NULL);CHKERRQ(ierr);
846   for (d = 0; d <= depth; ++d) {
847     ierr = DMPlexGetDepthStratum(dm, d, NULL, &pMaxNew[d]);CHKERRQ(ierr);
848     depthEnd[d]           = pMaxNew[d];
849     depthMax[d]           = depthMax[d] < 0 ? depthEnd[d] : depthMax[d];
850     numSplitPoints[d]     = 0;
851     numUnsplitPoints[d]   = 0;
852     numHybridPoints[d]    = 0;
853     numHybridPointsOld[d] = depthMax[d] < 0 ? 0 : depthEnd[d] - depthMax[d];
854     splitPoints[d]        = NULL;
855     unsplitPoints[d]      = NULL;
856     splitIS[d]            = NULL;
857     unsplitIS[d]          = NULL;
858     /* we are shifting the existing hybrid points with the stratum behind them, so
859      * the split comes at the end of the normal points, i.e., at depthMax[d] */
860     depthShift[2*d]       = depthMax[d];
861     depthShift[2*d+1]     = 0;
862   }
863   if (label) {
864     ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
865     ierr = ISGetLocalSize(valueIS, &numSP);CHKERRQ(ierr);
866     ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
867   }
868   for (sp = 0; sp < numSP; ++sp) {
869     const PetscInt dep = values[sp];
870 
871     if ((dep < 0) || (dep > depth)) continue;
872     ierr = DMLabelGetStratumIS(label, dep, &splitIS[dep]);CHKERRQ(ierr);
873     if (splitIS[dep]) {
874       ierr = ISGetLocalSize(splitIS[dep], &numSplitPoints[dep]);CHKERRQ(ierr);
875       ierr = ISGetIndices(splitIS[dep], &splitPoints[dep]);CHKERRQ(ierr);
876     }
877     ierr = DMLabelGetStratumIS(label, shift2+dep, &unsplitIS[dep]);CHKERRQ(ierr);
878     if (unsplitIS[dep]) {
879       ierr = ISGetLocalSize(unsplitIS[dep], &numUnsplitPoints[dep]);CHKERRQ(ierr);
880       ierr = ISGetIndices(unsplitIS[dep], &unsplitPoints[dep]);CHKERRQ(ierr);
881     }
882   }
883   /* Calculate number of hybrid points */
884   for (d = 1; d <= depth; ++d) numHybridPoints[d]     = numSplitPoints[d-1] + numUnsplitPoints[d-1]; /* There is a hybrid cell/face/edge for every split face/edge/vertex   */
885   for (d = 0; d <= depth; ++d) depthShift[2*d+1]      = numSplitPoints[d] + numHybridPoints[d];
886   ierr = DMPlexShiftPointSetUp_Internal(depth,depthShift);CHKERRQ(ierr);
887   /* the end of the points in this stratum that come before the new points:
888    * shifting pMaxNew[d] gets the new start of the next stratum, then count back the old hybrid points and the newly
889    * added points */
890   for (d = 0; d <= depth; ++d) pMaxNew[d]             = DMPlexShiftPoint_Internal(pMaxNew[d],depth,depthShift) - (numHybridPointsOld[d] + numSplitPoints[d] + numHybridPoints[d]);
891   ierr = DMPlexShiftSizes_Internal(dm, depthShift, sdm);CHKERRQ(ierr);
892   /* Step 3: Set cone/support sizes for new points */
893   for (dep = 0; dep <= depth; ++dep) {
894     for (p = 0; p < numSplitPoints[dep]; ++p) {
895       const PetscInt  oldp   = splitPoints[dep][p];
896       const PetscInt  newp   = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*oldp + depthOffset[dep]*/;
897       const PetscInt  splitp = p    + pMaxNew[dep];
898       const PetscInt *support;
899       PetscInt        coneSize, supportSize, qf, qn, qp, e;
900 
901       ierr = DMPlexGetConeSize(dm, oldp, &coneSize);CHKERRQ(ierr);
902       ierr = DMPlexSetConeSize(sdm, splitp, coneSize);CHKERRQ(ierr);
903       ierr = DMPlexGetSupportSize(dm, oldp, &supportSize);CHKERRQ(ierr);
904       ierr = DMPlexSetSupportSize(sdm, splitp, supportSize);CHKERRQ(ierr);
905       if (dep == depth-1) {
906         const PetscInt hybcell = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
907 
908         /* Add cohesive cells, they are prisms */
909         ierr = DMPlexSetConeSize(sdm, hybcell, 2 + coneSize);CHKERRQ(ierr);
910       } else if (dep == 0) {
911         const PetscInt hybedge = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
912 
913         ierr = DMPlexGetSupport(dm, oldp, &support);CHKERRQ(ierr);
914         for (e = 0, qn = 0, qp = 0, qf = 0; e < supportSize; ++e) {
915           PetscInt val;
916 
917           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
918           if (val == 1) ++qf;
919           if ((val == 1) || (val ==  (shift + 1))) ++qn;
920           if ((val == 1) || (val == -(shift + 1))) ++qp;
921         }
922         /* Split old vertex: Edges into original vertex and new cohesive edge */
923         ierr = DMPlexSetSupportSize(sdm, newp, qn+1);CHKERRQ(ierr);
924         /* Split new vertex: Edges into split vertex and new cohesive edge */
925         ierr = DMPlexSetSupportSize(sdm, splitp, qp+1);CHKERRQ(ierr);
926         /* Add hybrid edge */
927         ierr = DMPlexSetConeSize(sdm, hybedge, 2);CHKERRQ(ierr);
928         ierr = DMPlexSetSupportSize(sdm, hybedge, qf);CHKERRQ(ierr);
929       } else if (dep == dim-2) {
930         const PetscInt hybface = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
931 
932         ierr = DMPlexGetSupport(dm, oldp, &support);CHKERRQ(ierr);
933         for (e = 0, qn = 0, qp = 0, qf = 0; e < supportSize; ++e) {
934           PetscInt val;
935 
936           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
937           if (val == dim-1) ++qf;
938           if ((val == dim-1) || (val ==  (shift + dim-1))) ++qn;
939           if ((val == dim-1) || (val == -(shift + dim-1))) ++qp;
940         }
941         /* Split old edge: Faces into original edge and cohesive face (positive side?) */
942         ierr = DMPlexSetSupportSize(sdm, newp, qn+1);CHKERRQ(ierr);
943         /* Split new edge: Faces into split edge and cohesive face (negative side?) */
944         ierr = DMPlexSetSupportSize(sdm, splitp, qp+1);CHKERRQ(ierr);
945         /* Add hybrid face */
946         ierr = DMPlexSetConeSize(sdm, hybface, 4);CHKERRQ(ierr);
947         ierr = DMPlexSetSupportSize(sdm, hybface, qf);CHKERRQ(ierr);
948       }
949     }
950   }
951   for (dep = 0; dep <= depth; ++dep) {
952     for (p = 0; p < numUnsplitPoints[dep]; ++p) {
953       const PetscInt  oldp   = unsplitPoints[dep][p];
954       const PetscInt  newp   = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*oldp + depthOffset[dep]*/;
955       const PetscInt *support;
956       PetscInt        coneSize, supportSize, qf, e, s;
957 
958       ierr = DMPlexGetConeSize(dm, oldp, &coneSize);CHKERRQ(ierr);
959       ierr = DMPlexGetSupportSize(dm, oldp, &supportSize);CHKERRQ(ierr);
960       ierr = DMPlexGetSupport(dm, oldp, &support);CHKERRQ(ierr);
961       if (dep == 0) {
962         const PetscInt hybedge = p + pMaxNew[dep+1] + numSplitPoints[dep+1] + numSplitPoints[dep];
963 
964         /* Unsplit vertex: Edges into original vertex, split edges, and new cohesive edge twice */
965         for (s = 0, qf = 0; s < supportSize; ++s, ++qf) {
966           ierr = PetscFindInt(support[s], numSplitPoints[dep+1], splitPoints[dep+1], &e);CHKERRQ(ierr);
967           if (e >= 0) ++qf;
968         }
969         ierr = DMPlexSetSupportSize(sdm, newp, qf+2);CHKERRQ(ierr);
970         /* Add hybrid edge */
971         ierr = DMPlexSetConeSize(sdm, hybedge, 2);CHKERRQ(ierr);
972         for (e = 0, qf = 0; e < supportSize; ++e) {
973           PetscInt val;
974 
975           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
976           /* Split and unsplit edges produce hybrid faces */
977           if (val == 1) ++qf;
978           if (val == (shift2 + 1)) ++qf;
979         }
980         ierr = DMPlexSetSupportSize(sdm, hybedge, qf);CHKERRQ(ierr);
981       } else if (dep == dim-2) {
982         const PetscInt hybface = p + pMaxNew[dep+1] + numSplitPoints[dep+1] + numSplitPoints[dep];
983         PetscInt       val;
984 
985         for (e = 0, qf = 0; e < supportSize; ++e) {
986           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
987           if (val == dim-1) qf += 2;
988           else              ++qf;
989         }
990         /* Unsplit edge: Faces into original edge, split face, and cohesive face twice */
991         ierr = DMPlexSetSupportSize(sdm, newp, qf+2);CHKERRQ(ierr);
992         /* Add hybrid face */
993         for (e = 0, qf = 0; e < supportSize; ++e) {
994           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
995           if (val == dim-1) ++qf;
996         }
997         ierr = DMPlexSetConeSize(sdm, hybface, 4);CHKERRQ(ierr);
998         ierr = DMPlexSetSupportSize(sdm, hybface, qf);CHKERRQ(ierr);
999       }
1000     }
1001   }
1002   /* Step 4: Setup split DM */
1003   ierr = DMSetUp(sdm);CHKERRQ(ierr);
1004   ierr = DMPlexShiftPoints_Internal(dm, depthShift, sdm);CHKERRQ(ierr);
1005   ierr = DMPlexGetMaxSizes(sdm, &maxConeSizeNew, &maxSupportSizeNew);CHKERRQ(ierr);
1006   ierr = PetscMalloc3(PetscMax(maxConeSize, maxConeSizeNew)*3,&coneNew,PetscMax(maxConeSize, maxConeSizeNew)*3,&coneONew,PetscMax(maxSupportSize, maxSupportSizeNew),&supportNew);CHKERRQ(ierr);
1007   /* Step 6: Set cones and supports for new points */
1008   for (dep = 0; dep <= depth; ++dep) {
1009     for (p = 0; p < numSplitPoints[dep]; ++p) {
1010       const PetscInt  oldp   = splitPoints[dep][p];
1011       const PetscInt  newp   = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*oldp + depthOffset[dep]*/;
1012       const PetscInt  splitp = p    + pMaxNew[dep];
1013       const PetscInt *cone, *support, *ornt;
1014       PetscInt        coneSize, supportSize, q, qf, qn, qp, v, e, s;
1015 
1016       ierr = DMPlexGetConeSize(dm, oldp, &coneSize);CHKERRQ(ierr);
1017       ierr = DMPlexGetCone(dm, oldp, &cone);CHKERRQ(ierr);
1018       ierr = DMPlexGetConeOrientation(dm, oldp, &ornt);CHKERRQ(ierr);
1019       ierr = DMPlexGetSupportSize(dm, oldp, &supportSize);CHKERRQ(ierr);
1020       ierr = DMPlexGetSupport(dm, oldp, &support);CHKERRQ(ierr);
1021       if (dep == depth-1) {
1022         PetscBool       hasUnsplit = PETSC_FALSE;
1023         const PetscInt  hybcell    = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1024         const PetscInt *supportF;
1025 
1026         /* Split face:       copy in old face to new face to start */
1027         ierr = DMPlexGetSupport(sdm, newp,  &supportF);CHKERRQ(ierr);
1028         ierr = DMPlexSetSupport(sdm, splitp, supportF);CHKERRQ(ierr);
1029         /* Split old face:   old vertices/edges in cone so no change */
1030         /* Split new face:   new vertices/edges in cone */
1031         for (q = 0; q < coneSize; ++q) {
1032           ierr = PetscFindInt(cone[q], numSplitPoints[dep-1], splitPoints[dep-1], &v);CHKERRQ(ierr);
1033           if (v < 0) {
1034             ierr = PetscFindInt(cone[q], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &v);CHKERRQ(ierr);
1035             if (v < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %d in split or unsplit points of depth %d", cone[q], dep-1);
1036             coneNew[2+q] = DMPlexShiftPoint_Internal(cone[q], depth, depthShift) /*cone[q] + depthOffset[dep-1]*/;
1037             hasUnsplit   = PETSC_TRUE;
1038           } else {
1039             coneNew[2+q] = v + pMaxNew[dep-1];
1040             if (dep > 1) {
1041               const PetscInt *econe;
1042               PetscInt        econeSize, r, vs, vu;
1043 
1044               ierr = DMPlexGetConeSize(dm, cone[q], &econeSize);CHKERRQ(ierr);
1045               ierr = DMPlexGetCone(dm, cone[q], &econe);CHKERRQ(ierr);
1046               for (r = 0; r < econeSize; ++r) {
1047                 ierr = PetscFindInt(econe[r], numSplitPoints[dep-2],   splitPoints[dep-2],   &vs);CHKERRQ(ierr);
1048                 ierr = PetscFindInt(econe[r], numUnsplitPoints[dep-2], unsplitPoints[dep-2], &vu);CHKERRQ(ierr);
1049                 if (vs >= 0) continue;
1050                 if (vu < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %d in split or unsplit points of depth %d", econe[r], dep-2);
1051                 hasUnsplit   = PETSC_TRUE;
1052               }
1053             }
1054           }
1055         }
1056         ierr = DMPlexSetCone(sdm, splitp, &coneNew[2]);CHKERRQ(ierr);
1057         ierr = DMPlexSetConeOrientation(sdm, splitp, ornt);CHKERRQ(ierr);
1058         /* Face support */
1059         for (s = 0; s < supportSize; ++s) {
1060           PetscInt val;
1061 
1062           ierr = DMLabelGetValue(label, support[s], &val);CHKERRQ(ierr);
1063           if (val < 0) {
1064             /* Split old face:   Replace negative side cell with cohesive cell */
1065              ierr = DMPlexInsertSupport(sdm, newp, s, hybcell);CHKERRQ(ierr);
1066           } else {
1067             /* Split new face:   Replace positive side cell with cohesive cell */
1068             ierr = DMPlexInsertSupport(sdm, splitp, s, hybcell);CHKERRQ(ierr);
1069             /* Get orientation for cohesive face */
1070             {
1071               const PetscInt *ncone, *nconeO;
1072               PetscInt        nconeSize, nc;
1073 
1074               ierr = DMPlexGetConeSize(dm, support[s], &nconeSize);CHKERRQ(ierr);
1075               ierr = DMPlexGetCone(dm, support[s], &ncone);CHKERRQ(ierr);
1076               ierr = DMPlexGetConeOrientation(dm, support[s], &nconeO);CHKERRQ(ierr);
1077               for (nc = 0; nc < nconeSize; ++nc) {
1078                 if (ncone[nc] == oldp) {
1079                   coneONew[0] = nconeO[nc];
1080                   break;
1081                 }
1082               }
1083               if (nc >= nconeSize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate face %d in neighboring cell %d", oldp, support[s]);
1084             }
1085           }
1086         }
1087         /* Cohesive cell:    Old and new split face, then new cohesive faces */
1088         coneNew[0]  = newp;   /* Extracted negative side orientation above */
1089         coneNew[1]  = splitp;
1090         coneONew[1] = coneONew[0];
1091         for (q = 0; q < coneSize; ++q) {
1092           ierr = PetscFindInt(cone[q], numSplitPoints[dep-1], splitPoints[dep-1], &v);CHKERRQ(ierr);
1093           if (v < 0) {
1094             ierr = PetscFindInt(cone[q], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &v);CHKERRQ(ierr);
1095             coneNew[2+q]  = v + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep-1];
1096             coneONew[2+q] = 0;
1097           } else {
1098             coneNew[2+q]  = v + pMaxNew[dep] + numSplitPoints[dep];
1099           }
1100           coneONew[2+q] = 0;
1101         }
1102         ierr = DMPlexSetCone(sdm, hybcell, coneNew);CHKERRQ(ierr);
1103         ierr = DMPlexSetConeOrientation(sdm, hybcell, coneONew);CHKERRQ(ierr);
1104         /* Label the hybrid cells on the boundary of the split */
1105         if (hasUnsplit) {ierr = DMLabelSetValue(label, -hybcell, dim);CHKERRQ(ierr);}
1106       } else if (dep == 0) {
1107         const PetscInt hybedge = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1108 
1109         /* Split old vertex: Edges in old split faces and new cohesive edge */
1110         for (e = 0, qn = 0; e < supportSize; ++e) {
1111           PetscInt val;
1112 
1113           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
1114           if ((val == 1) || (val == (shift + 1))) {
1115             supportNew[qn++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1116           }
1117         }
1118         supportNew[qn] = hybedge;
1119         ierr = DMPlexSetSupport(sdm, newp, supportNew);CHKERRQ(ierr);
1120         /* Split new vertex: Edges in new split faces and new cohesive edge */
1121         for (e = 0, qp = 0; e < supportSize; ++e) {
1122           PetscInt val, edge;
1123 
1124           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
1125           if (val == 1) {
1126             ierr = PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &edge);CHKERRQ(ierr);
1127             if (edge < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Edge %d is not a split edge", support[e]);
1128             supportNew[qp++] = edge + pMaxNew[dep+1];
1129           } else if (val == -(shift + 1)) {
1130             supportNew[qp++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1131           }
1132         }
1133         supportNew[qp] = hybedge;
1134         ierr = DMPlexSetSupport(sdm, splitp, supportNew);CHKERRQ(ierr);
1135         /* Hybrid edge:    Old and new split vertex */
1136         coneNew[0] = newp;
1137         coneNew[1] = splitp;
1138         ierr = DMPlexSetCone(sdm, hybedge, coneNew);CHKERRQ(ierr);
1139         for (e = 0, qf = 0; e < supportSize; ++e) {
1140           PetscInt val, edge;
1141 
1142           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
1143           if (val == 1) {
1144             ierr = PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &edge);CHKERRQ(ierr);
1145             if (edge < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Edge %d is not a split edge", support[e]);
1146             supportNew[qf++] = edge + pMaxNew[dep+2] + numSplitPoints[dep+2];
1147           }
1148         }
1149         ierr = DMPlexSetSupport(sdm, hybedge, supportNew);CHKERRQ(ierr);
1150       } else if (dep == dim-2) {
1151         const PetscInt hybface = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1152 
1153         /* Split old edge:   old vertices in cone so no change */
1154         /* Split new edge:   new vertices in cone */
1155         for (q = 0; q < coneSize; ++q) {
1156           ierr = PetscFindInt(cone[q], numSplitPoints[dep-1], splitPoints[dep-1], &v);CHKERRQ(ierr);
1157           if (v < 0) {
1158             ierr = PetscFindInt(cone[q], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &v);CHKERRQ(ierr);
1159             if (v < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %d in split or unsplit points of depth %d", cone[q], dep-1);
1160             coneNew[q] = DMPlexShiftPoint_Internal(cone[q], depth, depthShift) /*cone[q] + depthOffset[dep-1]*/;
1161           } else {
1162             coneNew[q] = v + pMaxNew[dep-1];
1163           }
1164         }
1165         ierr = DMPlexSetCone(sdm, splitp, coneNew);CHKERRQ(ierr);
1166         /* Split old edge: Faces in positive side cells and old split faces */
1167         for (e = 0, q = 0; e < supportSize; ++e) {
1168           PetscInt val;
1169 
1170           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
1171           if (val == dim-1) {
1172             supportNew[q++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1173           } else if (val == (shift + dim-1)) {
1174             supportNew[q++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1175           }
1176         }
1177         supportNew[q++] = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1178         ierr = DMPlexSetSupport(sdm, newp, supportNew);CHKERRQ(ierr);
1179         /* Split new edge: Faces in negative side cells and new split faces */
1180         for (e = 0, q = 0; e < supportSize; ++e) {
1181           PetscInt val, face;
1182 
1183           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
1184           if (val == dim-1) {
1185             ierr = PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &face);CHKERRQ(ierr);
1186             if (face < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Face %d is not a split face", support[e]);
1187             supportNew[q++] = face + pMaxNew[dep+1];
1188           } else if (val == -(shift + dim-1)) {
1189             supportNew[q++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1190           }
1191         }
1192         supportNew[q++] = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1193         ierr = DMPlexSetSupport(sdm, splitp, supportNew);CHKERRQ(ierr);
1194         /* Hybrid face */
1195         coneNew[0] = newp;
1196         coneNew[1] = splitp;
1197         for (v = 0; v < coneSize; ++v) {
1198           PetscInt vertex;
1199           ierr = PetscFindInt(cone[v], numSplitPoints[dep-1], splitPoints[dep-1], &vertex);CHKERRQ(ierr);
1200           if (vertex < 0) {
1201             ierr = PetscFindInt(cone[v], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &vertex);CHKERRQ(ierr);
1202             if (vertex < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %d in split or unsplit points of depth %d", cone[v], dep-1);
1203             coneNew[2+v] = vertex + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep-1];
1204           } else {
1205             coneNew[2+v] = vertex + pMaxNew[dep] + numSplitPoints[dep];
1206           }
1207         }
1208         ierr = DMPlexSetCone(sdm, hybface, coneNew);CHKERRQ(ierr);
1209         for (e = 0, qf = 0; e < supportSize; ++e) {
1210           PetscInt val, face;
1211 
1212           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
1213           if (val == dim-1) {
1214             ierr = PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &face);CHKERRQ(ierr);
1215             if (face < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Face %d is not a split face", support[e]);
1216             supportNew[qf++] = face + pMaxNew[dep+2] + numSplitPoints[dep+2];
1217           }
1218         }
1219         ierr = DMPlexSetSupport(sdm, hybface, supportNew);CHKERRQ(ierr);
1220       }
1221     }
1222   }
1223   for (dep = 0; dep <= depth; ++dep) {
1224     for (p = 0; p < numUnsplitPoints[dep]; ++p) {
1225       const PetscInt  oldp   = unsplitPoints[dep][p];
1226       const PetscInt  newp   = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*oldp + depthOffset[dep]*/;
1227       const PetscInt *cone, *support, *ornt;
1228       PetscInt        coneSize, supportSize, supportSizeNew, q, qf, e, f, s;
1229 
1230       ierr = DMPlexGetConeSize(dm, oldp, &coneSize);CHKERRQ(ierr);
1231       ierr = DMPlexGetCone(dm, oldp, &cone);CHKERRQ(ierr);
1232       ierr = DMPlexGetConeOrientation(dm, oldp, &ornt);CHKERRQ(ierr);
1233       ierr = DMPlexGetSupportSize(dm, oldp, &supportSize);CHKERRQ(ierr);
1234       ierr = DMPlexGetSupport(dm, oldp, &support);CHKERRQ(ierr);
1235       if (dep == 0) {
1236         const PetscInt hybedge = p + pMaxNew[dep+1] + numSplitPoints[dep+1] + numSplitPoints[dep];
1237 
1238         /* Unsplit vertex */
1239         ierr = DMPlexGetSupportSize(sdm, newp, &supportSizeNew);CHKERRQ(ierr);
1240         for (s = 0, q = 0; s < supportSize; ++s) {
1241           supportNew[q++] = DMPlexShiftPoint_Internal(support[s], depth, depthShift) /*support[s] + depthOffset[dep+1]*/;
1242           ierr = PetscFindInt(support[s], numSplitPoints[dep+1], splitPoints[dep+1], &e);CHKERRQ(ierr);
1243           if (e >= 0) {
1244             supportNew[q++] = e + pMaxNew[dep+1];
1245           }
1246         }
1247         supportNew[q++] = hybedge;
1248         supportNew[q++] = hybedge;
1249         if (q != supportSizeNew) SETERRQ3(comm, PETSC_ERR_ARG_WRONG, "Support size %d != %d for vertex %d", q, supportSizeNew, newp);
1250         ierr = DMPlexSetSupport(sdm, newp, supportNew);CHKERRQ(ierr);
1251         /* Hybrid edge */
1252         coneNew[0] = newp;
1253         coneNew[1] = newp;
1254         ierr = DMPlexSetCone(sdm, hybedge, coneNew);CHKERRQ(ierr);
1255         for (e = 0, qf = 0; e < supportSize; ++e) {
1256           PetscInt val, edge;
1257 
1258           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
1259           if (val == 1) {
1260             ierr = PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &edge);CHKERRQ(ierr);
1261             if (edge < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Edge %d is not a split edge", support[e]);
1262             supportNew[qf++] = edge + pMaxNew[dep+2] + numSplitPoints[dep+2];
1263           } else if  (val ==  (shift2 + 1)) {
1264             ierr = PetscFindInt(support[e], numUnsplitPoints[dep+1], unsplitPoints[dep+1], &edge);CHKERRQ(ierr);
1265             if (edge < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Edge %d is not a unsplit edge", support[e]);
1266             supportNew[qf++] = edge + pMaxNew[dep+2] + numSplitPoints[dep+2] + numSplitPoints[dep+1];
1267           }
1268         }
1269         ierr = DMPlexSetSupport(sdm, hybedge, supportNew);CHKERRQ(ierr);
1270       } else if (dep == dim-2) {
1271         const PetscInt hybface = p + pMaxNew[dep+1] + numSplitPoints[dep+1] + numSplitPoints[dep];
1272 
1273         /* Unsplit edge: Faces into original edge, split face, and hybrid face twice */
1274         for (f = 0, qf = 0; f < supportSize; ++f) {
1275           PetscInt val, face;
1276 
1277           ierr = DMLabelGetValue(label, support[f], &val);CHKERRQ(ierr);
1278           if (val == dim-1) {
1279             ierr = PetscFindInt(support[f], numSplitPoints[dep+1], splitPoints[dep+1], &face);CHKERRQ(ierr);
1280             if (face < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Face %d is not a split face", support[f]);
1281             supportNew[qf++] = DMPlexShiftPoint_Internal(support[f], depth, depthShift) /*support[f] + depthOffset[dep+1]*/;
1282             supportNew[qf++] = face + pMaxNew[dep+1];
1283           } else {
1284             supportNew[qf++] = DMPlexShiftPoint_Internal(support[f], depth, depthShift) /*support[f] + depthOffset[dep+1]*/;
1285           }
1286         }
1287         supportNew[qf++] = hybface;
1288         supportNew[qf++] = hybface;
1289         ierr = DMPlexGetSupportSize(sdm, newp, &supportSizeNew);CHKERRQ(ierr);
1290         if (qf != supportSizeNew) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Support size for unsplit edge %d is %d != %d\n", newp, qf, supportSizeNew);
1291         ierr = DMPlexSetSupport(sdm, newp, supportNew);CHKERRQ(ierr);
1292         /* Add hybrid face */
1293         coneNew[0] = newp;
1294         coneNew[1] = newp;
1295         ierr = PetscFindInt(cone[0], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &v);CHKERRQ(ierr);
1296         if (v < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Vertex %d is not an unsplit vertex", cone[0]);
1297         coneNew[2] = v + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep-1];
1298         ierr = PetscFindInt(cone[1], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &v);CHKERRQ(ierr);
1299         if (v < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Vertex %d is not an unsplit vertex", cone[1]);
1300         coneNew[3] = v + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep-1];
1301         ierr = DMPlexSetCone(sdm, hybface, coneNew);CHKERRQ(ierr);
1302         for (f = 0, qf = 0; f < supportSize; ++f) {
1303           PetscInt val, face;
1304 
1305           ierr = DMLabelGetValue(label, support[f], &val);CHKERRQ(ierr);
1306           if (val == dim-1) {
1307             ierr = PetscFindInt(support[f], numSplitPoints[dep+1], splitPoints[dep+1], &face);CHKERRQ(ierr);
1308             supportNew[qf++] = face + pMaxNew[dep+2] + numSplitPoints[dep+2];
1309           }
1310         }
1311         ierr = DMPlexGetSupportSize(sdm, hybface, &supportSizeNew);CHKERRQ(ierr);
1312         if (qf != supportSizeNew) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Support size for hybrid face %d is %d != %d\n", hybface, qf, supportSizeNew);
1313         ierr = DMPlexSetSupport(sdm, hybface, supportNew);CHKERRQ(ierr);
1314       }
1315     }
1316   }
1317   /* Step 6b: Replace split points in negative side cones */
1318   for (sp = 0; sp < numSP; ++sp) {
1319     PetscInt        dep = values[sp];
1320     IS              pIS;
1321     PetscInt        numPoints;
1322     const PetscInt *points;
1323 
1324     if (dep >= 0) continue;
1325     ierr = DMLabelGetStratumIS(label, dep, &pIS);CHKERRQ(ierr);
1326     if (!pIS) continue;
1327     dep  = -dep - shift;
1328     ierr = ISGetLocalSize(pIS, &numPoints);CHKERRQ(ierr);
1329     ierr = ISGetIndices(pIS, &points);CHKERRQ(ierr);
1330     for (p = 0; p < numPoints; ++p) {
1331       const PetscInt  oldp = points[p];
1332       const PetscInt  newp = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*depthOffset[dep] + oldp*/;
1333       const PetscInt *cone;
1334       PetscInt        coneSize, c;
1335       /* PetscBool       replaced = PETSC_FALSE; */
1336 
1337       /* Negative edge: replace split vertex */
1338       /* Negative cell: replace split face */
1339       ierr = DMPlexGetConeSize(sdm, newp, &coneSize);CHKERRQ(ierr);
1340       ierr = DMPlexGetCone(sdm, newp, &cone);CHKERRQ(ierr);
1341       for (c = 0; c < coneSize; ++c) {
1342         const PetscInt coldp = DMPlexShiftPointInverse_Internal(cone[c],depth,depthShift);
1343         PetscInt       csplitp, cp, val;
1344 
1345         ierr = DMLabelGetValue(label, coldp, &val);CHKERRQ(ierr);
1346         if (val == dep-1) {
1347           ierr = PetscFindInt(coldp, numSplitPoints[dep-1], splitPoints[dep-1], &cp);CHKERRQ(ierr);
1348           if (cp < 0) SETERRQ2(comm, PETSC_ERR_ARG_WRONG, "Point %d is not a split point of dimension %d", oldp, dep-1);
1349           csplitp  = pMaxNew[dep-1] + cp;
1350           ierr     = DMPlexInsertCone(sdm, newp, c, csplitp);CHKERRQ(ierr);
1351           /* replaced = PETSC_TRUE; */
1352         }
1353       }
1354       /* Cells with only a vertex or edge on the submesh have no replacement */
1355       /* if (!replaced) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "The cone of point %d does not contain split points", oldp); */
1356     }
1357     ierr = ISRestoreIndices(pIS, &points);CHKERRQ(ierr);
1358     ierr = ISDestroy(&pIS);CHKERRQ(ierr);
1359   }
1360   /* Step 7: Stratify */
1361   ierr = DMPlexStratify(sdm);CHKERRQ(ierr);
1362   /* Step 8: Coordinates */
1363   ierr = DMPlexShiftCoordinates_Internal(dm, depthShift, sdm);CHKERRQ(ierr);
1364   ierr = DMGetCoordinateSection(sdm, &coordSection);CHKERRQ(ierr);
1365   ierr = DMGetCoordinatesLocal(sdm, &coordinates);CHKERRQ(ierr);
1366   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
1367   for (v = 0; v < (numSplitPoints ? numSplitPoints[0] : 0); ++v) {
1368     const PetscInt newp   = DMPlexShiftPoint_Internal(splitPoints[0][v], depth, depthShift) /*depthOffset[0] + splitPoints[0][v]*/;
1369     const PetscInt splitp = pMaxNew[0] + v;
1370     PetscInt       dof, off, soff, d;
1371 
1372     ierr = PetscSectionGetDof(coordSection, newp, &dof);CHKERRQ(ierr);
1373     ierr = PetscSectionGetOffset(coordSection, newp, &off);CHKERRQ(ierr);
1374     ierr = PetscSectionGetOffset(coordSection, splitp, &soff);CHKERRQ(ierr);
1375     for (d = 0; d < dof; ++d) coords[soff+d] = coords[off+d];
1376   }
1377   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
1378   /* Step 9: SF, if I can figure this out we can split the mesh in parallel */
1379   ierr = DMPlexShiftSF_Internal(dm, depthShift, sdm);CHKERRQ(ierr);
1380   /* Step 10: Labels */
1381   ierr = DMPlexShiftLabels_Internal(dm, depthShift, sdm);CHKERRQ(ierr);
1382   ierr = DMGetNumLabels(sdm, &numLabels);CHKERRQ(ierr);
1383   for (dep = 0; dep <= depth; ++dep) {
1384     for (p = 0; p < numSplitPoints[dep]; ++p) {
1385       const PetscInt newp   = DMPlexShiftPoint_Internal(splitPoints[dep][p], depth, depthShift) /*depthOffset[dep] + splitPoints[dep][p]*/;
1386       const PetscInt splitp = pMaxNew[dep] + p;
1387       PetscInt       l;
1388 
1389       for (l = 0; l < numLabels; ++l) {
1390         DMLabel     mlabel;
1391         const char *lname;
1392         PetscInt    val;
1393         PetscBool   isDepth;
1394 
1395         ierr = DMGetLabelName(sdm, l, &lname);CHKERRQ(ierr);
1396         ierr = PetscStrcmp(lname, "depth", &isDepth);CHKERRQ(ierr);
1397         if (isDepth) continue;
1398         ierr = DMGetLabel(sdm, lname, &mlabel);CHKERRQ(ierr);
1399         ierr = DMLabelGetValue(mlabel, newp, &val);CHKERRQ(ierr);
1400         if (val >= 0) {
1401           ierr = DMLabelSetValue(mlabel, splitp, val);CHKERRQ(ierr);
1402         }
1403       }
1404     }
1405   }
1406   for (sp = 0; sp < numSP; ++sp) {
1407     const PetscInt dep = values[sp];
1408 
1409     if ((dep < 0) || (dep > depth)) continue;
1410     if (splitIS[dep]) {ierr = ISRestoreIndices(splitIS[dep], &splitPoints[dep]);CHKERRQ(ierr);}
1411     ierr = ISDestroy(&splitIS[dep]);CHKERRQ(ierr);
1412     if (unsplitIS[dep]) {ierr = ISRestoreIndices(unsplitIS[dep], &unsplitPoints[dep]);CHKERRQ(ierr);}
1413     ierr = ISDestroy(&unsplitIS[dep]);CHKERRQ(ierr);
1414   }
1415   if (label) {
1416     ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
1417     ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
1418   }
1419   for (d = 0; d <= depth; ++d) {
1420     ierr = DMPlexGetDepthStratum(sdm, d, NULL, &pEnd);CHKERRQ(ierr);
1421     pMaxNew[d] = pEnd - numHybridPoints[d] - numHybridPointsOld[d];
1422   }
1423   ierr = DMPlexSetHybridBounds(sdm, depth >= 0 ? pMaxNew[depth] : PETSC_DETERMINE, depth>1 ? pMaxNew[depth-1] : PETSC_DETERMINE, depth>2 ? pMaxNew[1] : PETSC_DETERMINE, depth >= 0 ? pMaxNew[0] : PETSC_DETERMINE);CHKERRQ(ierr);
1424   ierr = PetscFree3(coneNew, coneONew, supportNew);CHKERRQ(ierr);
1425   ierr = PetscFree5(depthMax, depthEnd, depthShift, pMaxNew, numHybridPointsOld);CHKERRQ(ierr);
1426   ierr = PetscFree7(splitIS, unsplitIS, numSplitPoints, numUnsplitPoints, numHybridPoints, splitPoints, unsplitPoints);CHKERRQ(ierr);
1427   PetscFunctionReturn(0);
1428 }
1429 
1430 /*@C
1431   DMPlexConstructCohesiveCells - Construct cohesive cells which split the face along an internal interface
1432 
1433   Collective on dm
1434 
1435   Input Parameters:
1436 + dm - The original DM
1437 - label - The label specifying the boundary faces (this could be auto-generated)
1438 
1439   Output Parameters:
1440 - dmSplit - The new DM
1441 
1442   Level: developer
1443 
1444 .seealso: DMCreate(), DMPlexLabelCohesiveComplete()
1445 @*/
1446 PetscErrorCode DMPlexConstructCohesiveCells(DM dm, DMLabel label, DM *dmSplit)
1447 {
1448   DM             sdm;
1449   PetscInt       dim;
1450   PetscErrorCode ierr;
1451 
1452   PetscFunctionBegin;
1453   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1454   PetscValidPointer(dmSplit, 3);
1455   ierr = DMCreate(PetscObjectComm((PetscObject)dm), &sdm);CHKERRQ(ierr);
1456   ierr = DMSetType(sdm, DMPLEX);CHKERRQ(ierr);
1457   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
1458   ierr = DMSetDimension(sdm, dim);CHKERRQ(ierr);
1459   switch (dim) {
1460   case 2:
1461   case 3:
1462     ierr = DMPlexConstructCohesiveCells_Internal(dm, label, sdm);CHKERRQ(ierr);
1463     break;
1464   default:
1465     SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cannot construct cohesive cells for dimension %d", dim);
1466   }
1467   *dmSplit = sdm;
1468   PetscFunctionReturn(0);
1469 }
1470 
1471 /* Returns the side of the surface for a given cell with a face on the surface */
1472 static PetscErrorCode GetSurfaceSide_Static(DM dm, DM subdm, PetscInt numSubpoints, const PetscInt *subpoints, PetscInt cell, PetscInt face, PetscBool *pos)
1473 {
1474   const PetscInt *cone, *ornt;
1475   PetscInt        dim, coneSize, c;
1476   PetscErrorCode  ierr;
1477 
1478   PetscFunctionBegin;
1479   *pos = PETSC_TRUE;
1480   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
1481   ierr = DMPlexGetConeSize(dm, cell, &coneSize);CHKERRQ(ierr);
1482   ierr = DMPlexGetCone(dm, cell, &cone);CHKERRQ(ierr);
1483   ierr = DMPlexGetConeOrientation(dm, cell, &ornt);CHKERRQ(ierr);
1484   for (c = 0; c < coneSize; ++c) {
1485     if (cone[c] == face) {
1486       PetscInt o = ornt[c];
1487 
1488       if (subdm) {
1489         const PetscInt *subcone, *subornt;
1490         PetscInt        subpoint, subface, subconeSize, sc;
1491 
1492         ierr = PetscFindInt(cell, numSubpoints, subpoints, &subpoint);CHKERRQ(ierr);
1493         ierr = PetscFindInt(face, numSubpoints, subpoints, &subface);CHKERRQ(ierr);
1494         ierr = DMPlexGetConeSize(subdm, subpoint, &subconeSize);CHKERRQ(ierr);
1495         ierr = DMPlexGetCone(subdm, subpoint, &subcone);CHKERRQ(ierr);
1496         ierr = DMPlexGetConeOrientation(subdm, subpoint, &subornt);CHKERRQ(ierr);
1497         for (sc = 0; sc < subconeSize; ++sc) {
1498           if (subcone[sc] == subface) {
1499             o = subornt[0];
1500             break;
1501           }
1502         }
1503         if (sc >= subconeSize) SETERRQ4(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not find subpoint %d (%d) in cone for subpoint %d (%d)", subface, face, subpoint, cell);
1504       }
1505       if (o >= 0) *pos = PETSC_TRUE;
1506       else        *pos = PETSC_FALSE;
1507       break;
1508     }
1509   }
1510   if (c == coneSize) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Cell %d in split face %d support does not have it in the cone", cell, face);
1511   PetscFunctionReturn(0);
1512 }
1513 
1514 /*@
1515   DMPlexLabelCohesiveComplete - Starting with a label marking points on an internal surface, we add all other mesh pieces
1516   to complete the surface
1517 
1518   Input Parameters:
1519 + dm     - The DM
1520 . label  - A DMLabel marking the surface
1521 . blabel - A DMLabel marking the vertices on the boundary which will not be duplicated, or NULL to find them automatically
1522 . flip   - Flag to flip the submesh normal and replace points on the other side
1523 - subdm  - The subDM associated with the label, or NULL
1524 
1525   Output Parameter:
1526 . label - A DMLabel marking all surface points
1527 
1528   Note: The vertices in blabel are called "unsplit" in the terminology from hybrid cell creation.
1529 
1530   Level: developer
1531 
1532 .seealso: DMPlexConstructCohesiveCells(), DMPlexLabelComplete()
1533 @*/
1534 PetscErrorCode DMPlexLabelCohesiveComplete(DM dm, DMLabel label, DMLabel blabel, PetscBool flip, DM subdm)
1535 {
1536   DMLabel         depthLabel;
1537   IS              dimIS, subpointIS = NULL, facePosIS, faceNegIS, crossEdgeIS = NULL;
1538   const PetscInt *points, *subpoints;
1539   const PetscInt  rev   = flip ? -1 : 1;
1540   PetscInt       *pMax;
1541   PetscInt        shift = 100, shift2 = 200, dim, depth, pSize, dep, cStart, cEnd, cMax, fStart, fEnd, vStart, vEnd, numPoints, numSubpoints, p, val;
1542   PetscErrorCode  ierr;
1543 
1544   PetscFunctionBegin;
1545   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
1546   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
1547   pSize = PetscMax(depth, dim) + 1;
1548   ierr = PetscMalloc1(pSize,&pMax);CHKERRQ(ierr);
1549   ierr = DMPlexGetHybridBounds(dm, depth >= 0 ? &pMax[depth] : NULL, depth>1 ? &pMax[depth-1] : NULL, depth>2 ? &pMax[1] : NULL, &pMax[0]);CHKERRQ(ierr);
1550   ierr = DMPlexGetDepthLabel(dm, &depthLabel);CHKERRQ(ierr);
1551   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
1552   if (subdm) {
1553     ierr = DMPlexCreateSubpointIS(subdm, &subpointIS);CHKERRQ(ierr);
1554     if (subpointIS) {
1555       ierr = ISGetLocalSize(subpointIS, &numSubpoints);CHKERRQ(ierr);
1556       ierr = ISGetIndices(subpointIS, &subpoints);CHKERRQ(ierr);
1557     }
1558   }
1559   /* Mark cell on the fault, and its faces which touch the fault: cell orientation for face gives the side of the fault */
1560   ierr = DMLabelGetStratumIS(label, dim-1, &dimIS);CHKERRQ(ierr);
1561   if (!dimIS) {
1562     ierr = PetscFree(pMax);CHKERRQ(ierr);
1563     ierr = ISDestroy(&subpointIS);CHKERRQ(ierr);
1564     PetscFunctionReturn(0);
1565   }
1566   ierr = ISGetLocalSize(dimIS, &numPoints);CHKERRQ(ierr);
1567   ierr = ISGetIndices(dimIS, &points);CHKERRQ(ierr);
1568   for (p = 0; p < numPoints; ++p) { /* Loop over fault faces */
1569     const PetscInt *support;
1570     PetscInt        supportSize, s;
1571 
1572     ierr = DMPlexGetSupportSize(dm, points[p], &supportSize);CHKERRQ(ierr);
1573     if (supportSize != 2) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Split face %d has %d != 2 supports", points[p], supportSize);
1574     ierr = DMPlexGetSupport(dm, points[p], &support);CHKERRQ(ierr);
1575     for (s = 0; s < supportSize; ++s) {
1576       const PetscInt *cone;
1577       PetscInt        coneSize, c;
1578       PetscBool       pos;
1579 
1580       ierr = GetSurfaceSide_Static(dm, subdm, numSubpoints, subpoints, support[s], points[p], &pos);CHKERRQ(ierr);
1581       if (pos) {ierr = DMLabelSetValue(label, support[s],  rev*(shift+dim));CHKERRQ(ierr);}
1582       else     {ierr = DMLabelSetValue(label, support[s], -rev*(shift+dim));CHKERRQ(ierr);}
1583       if (rev < 0) pos = !pos ? PETSC_TRUE : PETSC_FALSE;
1584       /* Put faces touching the fault in the label */
1585       ierr = DMPlexGetConeSize(dm, support[s], &coneSize);CHKERRQ(ierr);
1586       ierr = DMPlexGetCone(dm, support[s], &cone);CHKERRQ(ierr);
1587       for (c = 0; c < coneSize; ++c) {
1588         const PetscInt point = cone[c];
1589 
1590         ierr = DMLabelGetValue(label, point, &val);CHKERRQ(ierr);
1591         if (val == -1) {
1592           PetscInt *closure = NULL;
1593           PetscInt  closureSize, cl;
1594 
1595           ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1596           for (cl = 0; cl < closureSize*2; cl += 2) {
1597             const PetscInt clp  = closure[cl];
1598             PetscInt       bval = -1;
1599 
1600             ierr = DMLabelGetValue(label, clp, &val);CHKERRQ(ierr);
1601             if (blabel) {ierr = DMLabelGetValue(blabel, clp, &bval);CHKERRQ(ierr);}
1602             if ((val >= 0) && (val < dim-1) && (bval < 0)) {
1603               ierr = DMLabelSetValue(label, point, pos == PETSC_TRUE ? shift+dim-1 : -(shift+dim-1));CHKERRQ(ierr);
1604               break;
1605             }
1606           }
1607           ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1608         }
1609       }
1610     }
1611   }
1612   ierr = ISRestoreIndices(dimIS, &points);CHKERRQ(ierr);
1613   ierr = ISDestroy(&dimIS);CHKERRQ(ierr);
1614   if (subpointIS) {ierr = ISRestoreIndices(subpointIS, &subpoints);CHKERRQ(ierr);}
1615   ierr = ISDestroy(&subpointIS);CHKERRQ(ierr);
1616   /* Mark boundary points as unsplit */
1617   if (blabel) {
1618     ierr = DMLabelGetStratumIS(blabel, 1, &dimIS);CHKERRQ(ierr);
1619     ierr = ISGetLocalSize(dimIS, &numPoints);CHKERRQ(ierr);
1620     ierr = ISGetIndices(dimIS, &points);CHKERRQ(ierr);
1621     for (p = 0; p < numPoints; ++p) {
1622       const PetscInt point = points[p];
1623       PetscInt       val, bval;
1624 
1625       ierr = DMLabelGetValue(blabel, point, &bval);CHKERRQ(ierr);
1626       if (bval >= 0) {
1627         ierr = DMLabelGetValue(label, point, &val);CHKERRQ(ierr);
1628         if ((val < 0) || (val > dim)) {
1629           /* This could be a point added from splitting a vertex on an adjacent fault, otherwise its just wrong */
1630           ierr = DMLabelClearValue(blabel, point, bval);CHKERRQ(ierr);
1631         }
1632       }
1633     }
1634     for (p = 0; p < numPoints; ++p) {
1635       const PetscInt point = points[p];
1636       PetscInt       val, bval;
1637 
1638       ierr = DMLabelGetValue(blabel, point, &bval);CHKERRQ(ierr);
1639       if (bval >= 0) {
1640         const PetscInt *cone,    *support;
1641         PetscInt        coneSize, supportSize, s, valA, valB, valE;
1642 
1643         /* Mark as unsplit */
1644         ierr = DMLabelGetValue(label, point, &val);CHKERRQ(ierr);
1645         if ((val < 0) || (val > dim)) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %d has label value %d, should be part of the fault", point, val);
1646         ierr = DMLabelClearValue(label, point, val);CHKERRQ(ierr);
1647         ierr = DMLabelSetValue(label, point, shift2+val);CHKERRQ(ierr);
1648         /* Check for cross-edge
1649              A cross-edge has endpoints which are both on the boundary of the surface, but the edge itself is not. */
1650         if (val != 0) continue;
1651         ierr = DMPlexGetSupport(dm, point, &support);CHKERRQ(ierr);
1652         ierr = DMPlexGetSupportSize(dm, point, &supportSize);CHKERRQ(ierr);
1653         for (s = 0; s < supportSize; ++s) {
1654           ierr = DMPlexGetCone(dm, support[s], &cone);CHKERRQ(ierr);
1655           ierr = DMPlexGetConeSize(dm, support[s], &coneSize);CHKERRQ(ierr);
1656           if (coneSize != 2) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Edge %D has %D vertices != 2", support[s], coneSize);
1657           ierr = DMLabelGetValue(blabel, cone[0], &valA);CHKERRQ(ierr);
1658           ierr = DMLabelGetValue(blabel, cone[1], &valB);CHKERRQ(ierr);
1659           ierr = DMLabelGetValue(blabel, support[s], &valE);CHKERRQ(ierr);
1660           if ((valE < 0) && (valA >= 0) && (valB >= 0) && (cone[0] != cone[1])) {ierr = DMLabelSetValue(blabel, support[s], 2);CHKERRQ(ierr);}
1661         }
1662       }
1663     }
1664     ierr = ISRestoreIndices(dimIS, &points);CHKERRQ(ierr);
1665     ierr = ISDestroy(&dimIS);CHKERRQ(ierr);
1666   }
1667   /* Search for other cells/faces/edges connected to the fault by a vertex */
1668   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
1669   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
1670   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
1671   cMax = cMax < 0 ? cEnd : cMax;
1672   ierr = DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd);CHKERRQ(ierr);
1673   ierr = DMLabelGetStratumIS(label, 0, &dimIS);CHKERRQ(ierr);
1674   if (blabel) {ierr = DMLabelGetStratumIS(blabel, 2, &crossEdgeIS);CHKERRQ(ierr);}
1675   if (dimIS && crossEdgeIS) {
1676     IS vertIS = dimIS;
1677 
1678     ierr = ISExpand(vertIS, crossEdgeIS, &dimIS);CHKERRQ(ierr);
1679     ierr = ISDestroy(&crossEdgeIS);CHKERRQ(ierr);
1680     ierr = ISDestroy(&vertIS);CHKERRQ(ierr);
1681   }
1682   if (!dimIS) {
1683     ierr = PetscFree(pMax);CHKERRQ(ierr);
1684     PetscFunctionReturn(0);
1685   }
1686   ierr = ISGetLocalSize(dimIS, &numPoints);CHKERRQ(ierr);
1687   ierr = ISGetIndices(dimIS, &points);CHKERRQ(ierr);
1688   for (p = 0; p < numPoints; ++p) { /* Loop over fault vertices */
1689     PetscInt *star = NULL;
1690     PetscInt  starSize, s;
1691     PetscInt  again = 1;  /* 0: Finished 1: Keep iterating after a change 2: No change */
1692 
1693     /* All points connected to the fault are inside a cell, so at the top level we will only check cells */
1694     ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_FALSE, &starSize, &star);CHKERRQ(ierr);
1695     while (again) {
1696       if (again > 1) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Could not classify all cells connected to the fault");
1697       again = 0;
1698       for (s = 0; s < starSize*2; s += 2) {
1699         const PetscInt  point = star[s];
1700         const PetscInt *cone;
1701         PetscInt        coneSize, c;
1702 
1703         if ((point < cStart) || (point >= cMax)) continue;
1704         ierr = DMLabelGetValue(label, point, &val);CHKERRQ(ierr);
1705         if (val != -1) continue;
1706         again = again == 1 ? 1 : 2;
1707         ierr  = DMPlexGetConeSize(dm, point, &coneSize);CHKERRQ(ierr);
1708         ierr  = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
1709         for (c = 0; c < coneSize; ++c) {
1710           ierr = DMLabelGetValue(label, cone[c], &val);CHKERRQ(ierr);
1711           if (val != -1) {
1712             const PetscInt *ccone;
1713             PetscInt        cconeSize, cc, side;
1714 
1715             if (PetscAbs(val) < shift) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Face %d on cell %d has an invalid label %d", cone[c], point, val);
1716             if (val > 0) side =  1;
1717             else         side = -1;
1718             ierr = DMLabelSetValue(label, point, side*(shift+dim));CHKERRQ(ierr);
1719             /* Mark cell faces which touch the fault */
1720             ierr = DMPlexGetConeSize(dm, point, &cconeSize);CHKERRQ(ierr);
1721             ierr = DMPlexGetCone(dm, point, &ccone);CHKERRQ(ierr);
1722             for (cc = 0; cc < cconeSize; ++cc) {
1723               PetscInt *closure = NULL;
1724               PetscInt  closureSize, cl;
1725 
1726               ierr = DMLabelGetValue(label, ccone[cc], &val);CHKERRQ(ierr);
1727               if (val != -1) continue;
1728               ierr = DMPlexGetTransitiveClosure(dm, ccone[cc], PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1729               for (cl = 0; cl < closureSize*2; cl += 2) {
1730                 const PetscInt clp = closure[cl];
1731 
1732                 ierr = DMLabelGetValue(label, clp, &val);CHKERRQ(ierr);
1733                 if (val == -1) continue;
1734                 ierr = DMLabelSetValue(label, ccone[cc], side*(shift+dim-1));CHKERRQ(ierr);
1735                 break;
1736               }
1737               ierr = DMPlexRestoreTransitiveClosure(dm, ccone[cc], PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1738             }
1739             again = 1;
1740             break;
1741           }
1742         }
1743       }
1744     }
1745     /* Classify the rest by cell membership */
1746     for (s = 0; s < starSize*2; s += 2) {
1747       const PetscInt point = star[s];
1748 
1749       ierr = DMLabelGetValue(label, point, &val);CHKERRQ(ierr);
1750       if (val == -1) {
1751         PetscInt  *sstar = NULL;
1752         PetscInt   sstarSize, ss;
1753         PetscBool  marked = PETSC_FALSE;
1754 
1755         ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_FALSE, &sstarSize, &sstar);CHKERRQ(ierr);
1756         for (ss = 0; ss < sstarSize*2; ss += 2) {
1757           const PetscInt spoint = sstar[ss];
1758 
1759           if ((spoint < cStart) || (spoint >= cMax)) continue;
1760           ierr = DMLabelGetValue(label, spoint, &val);CHKERRQ(ierr);
1761           if (val == -1) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Cell %d in star of %d does not have a valid label", spoint, point);
1762           ierr = DMLabelGetValue(depthLabel, point, &dep);CHKERRQ(ierr);
1763           if (val > 0) {
1764             ierr = DMLabelSetValue(label, point,   shift+dep);CHKERRQ(ierr);
1765           } else {
1766             ierr = DMLabelSetValue(label, point, -(shift+dep));CHKERRQ(ierr);
1767           }
1768           marked = PETSC_TRUE;
1769           break;
1770         }
1771         ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_FALSE, &sstarSize, &sstar);CHKERRQ(ierr);
1772         ierr = DMLabelGetValue(depthLabel, point, &dep);CHKERRQ(ierr);
1773         if (point < pMax[dep] && !marked) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d could not be classified", point);
1774       }
1775     }
1776     ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_FALSE, &starSize, &star);CHKERRQ(ierr);
1777   }
1778   ierr = ISRestoreIndices(dimIS, &points);CHKERRQ(ierr);
1779   ierr = ISDestroy(&dimIS);CHKERRQ(ierr);
1780   /* If any faces touching the fault divide cells on either side, split them */
1781   ierr = DMLabelGetStratumIS(label,   shift+dim-1,  &facePosIS);CHKERRQ(ierr);
1782   ierr = DMLabelGetStratumIS(label, -(shift+dim-1), &faceNegIS);CHKERRQ(ierr);
1783   ierr = ISExpand(facePosIS, faceNegIS, &dimIS);CHKERRQ(ierr);
1784   ierr = ISDestroy(&facePosIS);CHKERRQ(ierr);
1785   ierr = ISDestroy(&faceNegIS);CHKERRQ(ierr);
1786   ierr = ISGetLocalSize(dimIS, &numPoints);CHKERRQ(ierr);
1787   ierr = ISGetIndices(dimIS, &points);CHKERRQ(ierr);
1788   for (p = 0; p < numPoints; ++p) {
1789     const PetscInt  point = points[p];
1790     const PetscInt *support;
1791     PetscInt        supportSize, valA, valB;
1792 
1793     ierr = DMPlexGetSupportSize(dm, point, &supportSize);CHKERRQ(ierr);
1794     if (supportSize != 2) continue;
1795     ierr = DMPlexGetSupport(dm, point, &support);CHKERRQ(ierr);
1796     ierr = DMLabelGetValue(label, support[0], &valA);CHKERRQ(ierr);
1797     ierr = DMLabelGetValue(label, support[1], &valB);CHKERRQ(ierr);
1798     if ((valA == -1) || (valB == -1)) continue;
1799     if (valA*valB > 0) continue;
1800     /* Split the face */
1801     ierr = DMLabelGetValue(label, point, &valA);CHKERRQ(ierr);
1802     ierr = DMLabelClearValue(label, point, valA);CHKERRQ(ierr);
1803     ierr = DMLabelSetValue(label, point, dim-1);CHKERRQ(ierr);
1804     /* Label its closure:
1805       unmarked: label as unsplit
1806       incident: relabel as split
1807       split:    do nothing
1808     */
1809     {
1810       PetscInt *closure = NULL;
1811       PetscInt  closureSize, cl;
1812 
1813       ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1814       for (cl = 0; cl < closureSize*2; cl += 2) {
1815         ierr = DMLabelGetValue(label, closure[cl], &valA);CHKERRQ(ierr);
1816         if (valA == -1) { /* Mark as unsplit */
1817           ierr = DMLabelGetValue(depthLabel, closure[cl], &dep);CHKERRQ(ierr);
1818           ierr = DMLabelSetValue(label, closure[cl], shift2+dep);CHKERRQ(ierr);
1819         } else if (((valA >= shift) && (valA < shift2)) || ((valA <= -shift) && (valA > -shift2))) {
1820           ierr = DMLabelGetValue(depthLabel, closure[cl], &dep);CHKERRQ(ierr);
1821           ierr = DMLabelClearValue(label, closure[cl], valA);CHKERRQ(ierr);
1822           ierr = DMLabelSetValue(label, closure[cl], dep);CHKERRQ(ierr);
1823         }
1824       }
1825       ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1826     }
1827   }
1828   ierr = ISRestoreIndices(dimIS, &points);CHKERRQ(ierr);
1829   ierr = ISDestroy(&dimIS);CHKERRQ(ierr);
1830   ierr = PetscFree(pMax);CHKERRQ(ierr);
1831   PetscFunctionReturn(0);
1832 }
1833 
1834 /*@
1835   DMPlexCreateHybridMesh - Create a mesh with hybrid cells along an internal interface
1836 
1837   Collective on dm
1838 
1839   Input Parameters:
1840 + dm - The original DM
1841 - labelName - The label specifying the interface vertices
1842 
1843   Output Parameters:
1844 + hybridLabel - The label fully marking the interface
1845 - dmHybrid - The new DM
1846 
1847   Level: developer
1848 
1849 .seealso: DMPlexConstructCohesiveCells(), DMPlexLabelCohesiveComplete(), DMCreate()
1850 @*/
1851 PetscErrorCode DMPlexCreateHybridMesh(DM dm, DMLabel label, DMLabel *hybridLabel, DM *dmHybrid)
1852 {
1853   DM             idm;
1854   DMLabel        subpointMap, hlabel;
1855   PetscInt       dim;
1856   PetscErrorCode ierr;
1857 
1858   PetscFunctionBegin;
1859   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1860   if (hybridLabel) PetscValidPointer(hybridLabel, 3);
1861   PetscValidPointer(dmHybrid, 4);
1862   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
1863   ierr = DMPlexCreateSubmesh(dm, label, 1, &idm);CHKERRQ(ierr);
1864   ierr = DMPlexOrient(idm);CHKERRQ(ierr);
1865   ierr = DMPlexGetSubpointMap(idm, &subpointMap);CHKERRQ(ierr);
1866   ierr = DMLabelDuplicate(subpointMap, &hlabel);CHKERRQ(ierr);
1867   ierr = DMLabelClearStratum(hlabel, dim);CHKERRQ(ierr);
1868   ierr = DMPlexLabelCohesiveComplete(dm, hlabel, NULL, PETSC_FALSE, idm);CHKERRQ(ierr);
1869   ierr = DMDestroy(&idm);CHKERRQ(ierr);
1870   ierr = DMPlexConstructCohesiveCells(dm, hlabel, dmHybrid);CHKERRQ(ierr);
1871   if (hybridLabel) *hybridLabel = hlabel;
1872   else             {ierr = DMLabelDestroy(&hlabel);CHKERRQ(ierr);}
1873   PetscFunctionReturn(0);
1874 }
1875 
1876 /* Here we need the explicit assumption that:
1877 
1878      For any marked cell, the marked vertices constitute a single face
1879 */
1880 static PetscErrorCode DMPlexMarkSubmesh_Uninterpolated(DM dm, DMLabel vertexLabel, PetscInt value, DMLabel subpointMap, PetscInt *numFaces, PetscInt *nFV, DM subdm)
1881 {
1882   IS               subvertexIS = NULL;
1883   const PetscInt  *subvertices;
1884   PetscInt        *pStart, *pEnd, *pMax, pSize;
1885   PetscInt         depth, dim, d, numSubVerticesInitial = 0, v;
1886   PetscErrorCode   ierr;
1887 
1888   PetscFunctionBegin;
1889   *numFaces = 0;
1890   *nFV      = 0;
1891   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
1892   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
1893   pSize = PetscMax(depth, dim) + 1;
1894   ierr = PetscMalloc3(pSize,&pStart,pSize,&pEnd,pSize,&pMax);CHKERRQ(ierr);
1895   ierr = DMPlexGetHybridBounds(dm, depth >= 0 ? &pMax[depth] : NULL, depth>1 ? &pMax[depth-1] : NULL, depth>2 ? &pMax[1] : NULL, &pMax[0]);CHKERRQ(ierr);
1896   for (d = 0; d <= depth; ++d) {
1897     ierr = DMPlexGetDepthStratum(dm, d, &pStart[d], &pEnd[d]);CHKERRQ(ierr);
1898     if (pMax[d] >= 0) pEnd[d] = PetscMin(pEnd[d], pMax[d]);
1899   }
1900   /* Loop over initial vertices and mark all faces in the collective star() */
1901   if (vertexLabel) {ierr = DMLabelGetStratumIS(vertexLabel, value, &subvertexIS);CHKERRQ(ierr);}
1902   if (subvertexIS) {
1903     ierr = ISGetSize(subvertexIS, &numSubVerticesInitial);CHKERRQ(ierr);
1904     ierr = ISGetIndices(subvertexIS, &subvertices);CHKERRQ(ierr);
1905   }
1906   for (v = 0; v < numSubVerticesInitial; ++v) {
1907     const PetscInt vertex = subvertices[v];
1908     PetscInt      *star   = NULL;
1909     PetscInt       starSize, s, numCells = 0, c;
1910 
1911     ierr = DMPlexGetTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star);CHKERRQ(ierr);
1912     for (s = 0; s < starSize*2; s += 2) {
1913       const PetscInt point = star[s];
1914       if ((point >= pStart[depth]) && (point < pEnd[depth])) star[numCells++] = point;
1915     }
1916     for (c = 0; c < numCells; ++c) {
1917       const PetscInt cell    = star[c];
1918       PetscInt      *closure = NULL;
1919       PetscInt       closureSize, cl;
1920       PetscInt       cellLoc, numCorners = 0, faceSize = 0;
1921 
1922       ierr = DMLabelGetValue(subpointMap, cell, &cellLoc);CHKERRQ(ierr);
1923       if (cellLoc == 2) continue;
1924       if (cellLoc >= 0) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Cell %d has dimension %d in the surface label", cell, cellLoc);
1925       ierr = DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1926       for (cl = 0; cl < closureSize*2; cl += 2) {
1927         const PetscInt point = closure[cl];
1928         PetscInt       vertexLoc;
1929 
1930         if ((point >= pStart[0]) && (point < pEnd[0])) {
1931           ++numCorners;
1932           ierr = DMLabelGetValue(vertexLabel, point, &vertexLoc);CHKERRQ(ierr);
1933           if (vertexLoc == value) closure[faceSize++] = point;
1934         }
1935       }
1936       if (!(*nFV)) {ierr = DMPlexGetNumFaceVertices(dm, dim, numCorners, nFV);CHKERRQ(ierr);}
1937       if (faceSize > *nFV) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Invalid submesh: Too many vertices %d of an element on the surface", faceSize);
1938       if (faceSize == *nFV) {
1939         const PetscInt *cells = NULL;
1940         PetscInt        numCells, nc;
1941 
1942         ++(*numFaces);
1943         for (cl = 0; cl < faceSize; ++cl) {
1944           ierr = DMLabelSetValue(subpointMap, closure[cl], 0);CHKERRQ(ierr);
1945         }
1946         ierr = DMPlexGetJoin(dm, faceSize, closure, &numCells, &cells);CHKERRQ(ierr);
1947         for (nc = 0; nc < numCells; ++nc) {
1948           ierr = DMLabelSetValue(subpointMap, cells[nc], 2);CHKERRQ(ierr);
1949         }
1950         ierr = DMPlexRestoreJoin(dm, faceSize, closure, &numCells, &cells);CHKERRQ(ierr);
1951       }
1952       ierr = DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1953     }
1954     ierr = DMPlexRestoreTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star);CHKERRQ(ierr);
1955   }
1956   if (subvertexIS) {
1957     ierr = ISRestoreIndices(subvertexIS, &subvertices);CHKERRQ(ierr);
1958   }
1959   ierr = ISDestroy(&subvertexIS);CHKERRQ(ierr);
1960   ierr = PetscFree3(pStart,pEnd,pMax);CHKERRQ(ierr);
1961   PetscFunctionReturn(0);
1962 }
1963 
1964 static PetscErrorCode DMPlexMarkSubmesh_Interpolated(DM dm, DMLabel vertexLabel, PetscInt value, DMLabel subpointMap, DM subdm)
1965 {
1966   IS               subvertexIS = NULL;
1967   const PetscInt  *subvertices;
1968   PetscInt        *pStart, *pEnd, *pMax;
1969   PetscInt         dim, d, numSubVerticesInitial = 0, v;
1970   PetscErrorCode   ierr;
1971 
1972   PetscFunctionBegin;
1973   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
1974   ierr = PetscMalloc3(dim+1,&pStart,dim+1,&pEnd,dim+1,&pMax);CHKERRQ(ierr);
1975   ierr = DMPlexGetHybridBounds(dm, &pMax[dim], dim>1 ? &pMax[dim-1] : NULL, dim > 2 ? &pMax[1] : NULL, &pMax[0]);CHKERRQ(ierr);
1976   for (d = 0; d <= dim; ++d) {
1977     ierr = DMPlexGetDepthStratum(dm, d, &pStart[d], &pEnd[d]);CHKERRQ(ierr);
1978     if (pMax[d] >= 0) pEnd[d] = PetscMin(pEnd[d], pMax[d]);
1979   }
1980   /* Loop over initial vertices and mark all faces in the collective star() */
1981   if (vertexLabel) {
1982     ierr = DMLabelGetStratumIS(vertexLabel, value, &subvertexIS);CHKERRQ(ierr);
1983     if (subvertexIS) {
1984       ierr = ISGetSize(subvertexIS, &numSubVerticesInitial);CHKERRQ(ierr);
1985       ierr = ISGetIndices(subvertexIS, &subvertices);CHKERRQ(ierr);
1986     }
1987   }
1988   for (v = 0; v < numSubVerticesInitial; ++v) {
1989     const PetscInt vertex = subvertices[v];
1990     PetscInt      *star   = NULL;
1991     PetscInt       starSize, s, numFaces = 0, f;
1992 
1993     ierr = DMPlexGetTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star);CHKERRQ(ierr);
1994     for (s = 0; s < starSize*2; s += 2) {
1995       const PetscInt point = star[s];
1996       if ((point >= pStart[dim-1]) && (point < pEnd[dim-1])) star[numFaces++] = point;
1997     }
1998     for (f = 0; f < numFaces; ++f) {
1999       const PetscInt face    = star[f];
2000       PetscInt      *closure = NULL;
2001       PetscInt       closureSize, c;
2002       PetscInt       faceLoc;
2003 
2004       ierr = DMLabelGetValue(subpointMap, face, &faceLoc);CHKERRQ(ierr);
2005       if (faceLoc == dim-1) continue;
2006       if (faceLoc >= 0) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Face %d has dimension %d in the surface label", face, faceLoc);
2007       ierr = DMPlexGetTransitiveClosure(dm, face, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
2008       for (c = 0; c < closureSize*2; c += 2) {
2009         const PetscInt point = closure[c];
2010         PetscInt       vertexLoc;
2011 
2012         if ((point >= pStart[0]) && (point < pEnd[0])) {
2013           ierr = DMLabelGetValue(vertexLabel, point, &vertexLoc);CHKERRQ(ierr);
2014           if (vertexLoc != value) break;
2015         }
2016       }
2017       if (c == closureSize*2) {
2018         const PetscInt *support;
2019         PetscInt        supportSize, s;
2020 
2021         for (c = 0; c < closureSize*2; c += 2) {
2022           const PetscInt point = closure[c];
2023 
2024           for (d = 0; d < dim; ++d) {
2025             if ((point >= pStart[d]) && (point < pEnd[d])) {
2026               ierr = DMLabelSetValue(subpointMap, point, d);CHKERRQ(ierr);
2027               break;
2028             }
2029           }
2030         }
2031         ierr = DMPlexGetSupportSize(dm, face, &supportSize);CHKERRQ(ierr);
2032         ierr = DMPlexGetSupport(dm, face, &support);CHKERRQ(ierr);
2033         for (s = 0; s < supportSize; ++s) {
2034           ierr = DMLabelSetValue(subpointMap, support[s], dim);CHKERRQ(ierr);
2035         }
2036       }
2037       ierr = DMPlexRestoreTransitiveClosure(dm, face, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
2038     }
2039     ierr = DMPlexRestoreTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star);CHKERRQ(ierr);
2040   }
2041   if (subvertexIS) {ierr = ISRestoreIndices(subvertexIS, &subvertices);CHKERRQ(ierr);}
2042   ierr = ISDestroy(&subvertexIS);CHKERRQ(ierr);
2043   ierr = PetscFree3(pStart,pEnd,pMax);CHKERRQ(ierr);
2044   PetscFunctionReturn(0);
2045 }
2046 
2047 static PetscErrorCode DMPlexMarkCohesiveSubmesh_Uninterpolated(DM dm, PetscBool hasLagrange, const char labelname[], PetscInt value, DMLabel subpointMap, PetscInt *numFaces, PetscInt *nFV, PetscInt *subCells[], DM subdm)
2048 {
2049   DMLabel         label = NULL;
2050   const PetscInt *cone;
2051   PetscInt        dim, cMax, cEnd, c, subc = 0, p, coneSize = -1;
2052   PetscErrorCode  ierr;
2053 
2054   PetscFunctionBegin;
2055   *numFaces = 0;
2056   *nFV = 0;
2057   if (labelname) {ierr = DMGetLabel(dm, labelname, &label);CHKERRQ(ierr);}
2058   *subCells = NULL;
2059   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
2060   ierr = DMPlexGetHeightStratum(dm, 0, NULL, &cEnd);CHKERRQ(ierr);
2061   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
2062   if (cMax < 0) PetscFunctionReturn(0);
2063   if (label) {
2064     for (c = cMax; c < cEnd; ++c) {
2065       PetscInt val;
2066 
2067       ierr = DMLabelGetValue(label, c, &val);CHKERRQ(ierr);
2068       if (val == value) {
2069         ++(*numFaces);
2070         ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
2071       }
2072     }
2073   } else {
2074     *numFaces = cEnd - cMax;
2075     ierr = DMPlexGetConeSize(dm, cMax, &coneSize);CHKERRQ(ierr);
2076   }
2077   ierr = PetscMalloc1(*numFaces *2, subCells);CHKERRQ(ierr);
2078   if (!(*numFaces)) PetscFunctionReturn(0);
2079   *nFV = hasLagrange ? coneSize/3 : coneSize/2;
2080   for (c = cMax; c < cEnd; ++c) {
2081     const PetscInt *cells;
2082     PetscInt        numCells;
2083 
2084     if (label) {
2085       PetscInt val;
2086 
2087       ierr = DMLabelGetValue(label, c, &val);CHKERRQ(ierr);
2088       if (val != value) continue;
2089     }
2090     ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
2091     for (p = 0; p < *nFV; ++p) {
2092       ierr = DMLabelSetValue(subpointMap, cone[p], 0);CHKERRQ(ierr);
2093     }
2094     /* Negative face */
2095     ierr = DMPlexGetJoin(dm, *nFV, cone, &numCells, &cells);CHKERRQ(ierr);
2096     /* Not true in parallel
2097     if (numCells != 2) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive cells should separate two cells"); */
2098     for (p = 0; p < numCells; ++p) {
2099       ierr = DMLabelSetValue(subpointMap, cells[p], 2);CHKERRQ(ierr);
2100       (*subCells)[subc++] = cells[p];
2101     }
2102     ierr = DMPlexRestoreJoin(dm, *nFV, cone, &numCells, &cells);CHKERRQ(ierr);
2103     /* Positive face is not included */
2104   }
2105   PetscFunctionReturn(0);
2106 }
2107 
2108 static PetscErrorCode DMPlexMarkCohesiveSubmesh_Interpolated(DM dm, DMLabel label, PetscInt value, DMLabel subpointMap, DM subdm)
2109 {
2110   PetscInt      *pStart, *pEnd;
2111   PetscInt       dim, cMax, cEnd, c, d;
2112   PetscErrorCode ierr;
2113 
2114   PetscFunctionBegin;
2115   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
2116   ierr = DMPlexGetHeightStratum(dm, 0, NULL, &cEnd);CHKERRQ(ierr);
2117   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
2118   if (cMax < 0) PetscFunctionReturn(0);
2119   ierr = PetscMalloc2(dim+1,&pStart,dim+1,&pEnd);CHKERRQ(ierr);
2120   for (d = 0; d <= dim; ++d) {ierr = DMPlexGetDepthStratum(dm, d, &pStart[d], &pEnd[d]);CHKERRQ(ierr);}
2121   for (c = cMax; c < cEnd; ++c) {
2122     const PetscInt *cone;
2123     PetscInt       *closure = NULL;
2124     PetscInt        fconeSize, coneSize, closureSize, cl, val;
2125 
2126     if (label) {
2127       ierr = DMLabelGetValue(label, c, &val);CHKERRQ(ierr);
2128       if (val != value) continue;
2129     }
2130     ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
2131     ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
2132     ierr = DMPlexGetConeSize(dm, cone[0], &fconeSize);CHKERRQ(ierr);
2133     if (coneSize != (fconeSize ? fconeSize : 1) + 2) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive cells should separate two cells");
2134     /* Negative face */
2135     ierr = DMPlexGetTransitiveClosure(dm, cone[0], PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
2136     for (cl = 0; cl < closureSize*2; cl += 2) {
2137       const PetscInt point = closure[cl];
2138 
2139       for (d = 0; d <= dim; ++d) {
2140         if ((point >= pStart[d]) && (point < pEnd[d])) {
2141           ierr = DMLabelSetValue(subpointMap, point, d);CHKERRQ(ierr);
2142           break;
2143         }
2144       }
2145     }
2146     ierr = DMPlexRestoreTransitiveClosure(dm, cone[0], PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
2147     /* Cells -- positive face is not included */
2148     for (cl = 0; cl < 1; ++cl) {
2149       const PetscInt *support;
2150       PetscInt        supportSize, s;
2151 
2152       ierr = DMPlexGetSupportSize(dm, cone[cl], &supportSize);CHKERRQ(ierr);
2153       /* if (supportSize != 2) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive faces should separate two cells"); */
2154       ierr = DMPlexGetSupport(dm, cone[cl], &support);CHKERRQ(ierr);
2155       for (s = 0; s < supportSize; ++s) {
2156         ierr = DMLabelSetValue(subpointMap, support[s], dim);CHKERRQ(ierr);
2157       }
2158     }
2159   }
2160   ierr = PetscFree2(pStart, pEnd);CHKERRQ(ierr);
2161   PetscFunctionReturn(0);
2162 }
2163 
2164 static PetscErrorCode DMPlexGetFaceOrientation(DM dm, PetscInt cell, PetscInt numCorners, PetscInt indices[], PetscInt oppositeVertex, PetscInt origVertices[], PetscInt faceVertices[], PetscBool *posOriented)
2165 {
2166   MPI_Comm       comm;
2167   PetscBool      posOrient = PETSC_FALSE;
2168   const PetscInt debug     = 0;
2169   PetscInt       cellDim, faceSize, f;
2170   PetscErrorCode ierr;
2171 
2172   PetscFunctionBegin;
2173   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
2174   ierr = DMGetDimension(dm, &cellDim);CHKERRQ(ierr);
2175   if (debug) {PetscPrintf(comm, "cellDim: %d numCorners: %d\n", cellDim, numCorners);CHKERRQ(ierr);}
2176 
2177   if (cellDim == 1 && numCorners == 2) {
2178     /* Triangle */
2179     faceSize  = numCorners-1;
2180     posOrient = !(oppositeVertex%2) ? PETSC_TRUE : PETSC_FALSE;
2181   } else if (cellDim == 2 && numCorners == 3) {
2182     /* Triangle */
2183     faceSize  = numCorners-1;
2184     posOrient = !(oppositeVertex%2) ? PETSC_TRUE : PETSC_FALSE;
2185   } else if (cellDim == 3 && numCorners == 4) {
2186     /* Tetrahedron */
2187     faceSize  = numCorners-1;
2188     posOrient = (oppositeVertex%2) ? PETSC_TRUE : PETSC_FALSE;
2189   } else if (cellDim == 1 && numCorners == 3) {
2190     /* Quadratic line */
2191     faceSize  = 1;
2192     posOrient = PETSC_TRUE;
2193   } else if (cellDim == 2 && numCorners == 4) {
2194     /* Quads */
2195     faceSize = 2;
2196     if ((indices[1] > indices[0]) && (indices[1] - indices[0] == 1)) {
2197       posOrient = PETSC_TRUE;
2198     } else if ((indices[0] == 3) && (indices[1] == 0)) {
2199       posOrient = PETSC_TRUE;
2200     } else {
2201       if (((indices[0] > indices[1]) && (indices[0] - indices[1] == 1)) || ((indices[0] == 0) && (indices[1] == 3))) {
2202         posOrient = PETSC_FALSE;
2203       } else SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid quad crossedge");
2204     }
2205   } else if (cellDim == 2 && numCorners == 6) {
2206     /* Quadratic triangle (I hate this) */
2207     /* Edges are determined by the first 2 vertices (corners of edges) */
2208     const PetscInt faceSizeTri = 3;
2209     PetscInt       sortedIndices[3], i, iFace;
2210     PetscBool      found                    = PETSC_FALSE;
2211     PetscInt       faceVerticesTriSorted[9] = {
2212       0, 3,  4, /* bottom */
2213       1, 4,  5, /* right */
2214       2, 3,  5, /* left */
2215     };
2216     PetscInt       faceVerticesTri[9] = {
2217       0, 3,  4, /* bottom */
2218       1, 4,  5, /* right */
2219       2, 5,  3, /* left */
2220     };
2221 
2222     for (i = 0; i < faceSizeTri; ++i) sortedIndices[i] = indices[i];
2223     ierr = PetscSortInt(faceSizeTri, sortedIndices);CHKERRQ(ierr);
2224     for (iFace = 0; iFace < 3; ++iFace) {
2225       const PetscInt ii = iFace*faceSizeTri;
2226       PetscInt       fVertex, cVertex;
2227 
2228       if ((sortedIndices[0] == faceVerticesTriSorted[ii+0]) &&
2229           (sortedIndices[1] == faceVerticesTriSorted[ii+1])) {
2230         for (fVertex = 0; fVertex < faceSizeTri; ++fVertex) {
2231           for (cVertex = 0; cVertex < faceSizeTri; ++cVertex) {
2232             if (indices[cVertex] == faceVerticesTri[ii+fVertex]) {
2233               faceVertices[fVertex] = origVertices[cVertex];
2234               break;
2235             }
2236           }
2237         }
2238         found = PETSC_TRUE;
2239         break;
2240       }
2241     }
2242     if (!found) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid tri crossface");
2243     if (posOriented) *posOriented = PETSC_TRUE;
2244     PetscFunctionReturn(0);
2245   } else if (cellDim == 2 && numCorners == 9) {
2246     /* Quadratic quad (I hate this) */
2247     /* Edges are determined by the first 2 vertices (corners of edges) */
2248     const PetscInt faceSizeQuad = 3;
2249     PetscInt       sortedIndices[3], i, iFace;
2250     PetscBool      found                      = PETSC_FALSE;
2251     PetscInt       faceVerticesQuadSorted[12] = {
2252       0, 1,  4, /* bottom */
2253       1, 2,  5, /* right */
2254       2, 3,  6, /* top */
2255       0, 3,  7, /* left */
2256     };
2257     PetscInt       faceVerticesQuad[12] = {
2258       0, 1,  4, /* bottom */
2259       1, 2,  5, /* right */
2260       2, 3,  6, /* top */
2261       3, 0,  7, /* left */
2262     };
2263 
2264     for (i = 0; i < faceSizeQuad; ++i) sortedIndices[i] = indices[i];
2265     ierr = PetscSortInt(faceSizeQuad, sortedIndices);CHKERRQ(ierr);
2266     for (iFace = 0; iFace < 4; ++iFace) {
2267       const PetscInt ii = iFace*faceSizeQuad;
2268       PetscInt       fVertex, cVertex;
2269 
2270       if ((sortedIndices[0] == faceVerticesQuadSorted[ii+0]) &&
2271           (sortedIndices[1] == faceVerticesQuadSorted[ii+1])) {
2272         for (fVertex = 0; fVertex < faceSizeQuad; ++fVertex) {
2273           for (cVertex = 0; cVertex < faceSizeQuad; ++cVertex) {
2274             if (indices[cVertex] == faceVerticesQuad[ii+fVertex]) {
2275               faceVertices[fVertex] = origVertices[cVertex];
2276               break;
2277             }
2278           }
2279         }
2280         found = PETSC_TRUE;
2281         break;
2282       }
2283     }
2284     if (!found) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid quad crossface");
2285     if (posOriented) *posOriented = PETSC_TRUE;
2286     PetscFunctionReturn(0);
2287   } else if (cellDim == 3 && numCorners == 8) {
2288     /* Hexes
2289        A hex is two oriented quads with the normal of the first
2290        pointing up at the second.
2291 
2292           7---6
2293          /|  /|
2294         4---5 |
2295         | 1-|-2
2296         |/  |/
2297         0---3
2298 
2299         Faces are determined by the first 4 vertices (corners of faces) */
2300     const PetscInt faceSizeHex = 4;
2301     PetscInt       sortedIndices[4], i, iFace;
2302     PetscBool      found                     = PETSC_FALSE;
2303     PetscInt       faceVerticesHexSorted[24] = {
2304       0, 1, 2, 3,  /* bottom */
2305       4, 5, 6, 7,  /* top */
2306       0, 3, 4, 5,  /* front */
2307       2, 3, 5, 6,  /* right */
2308       1, 2, 6, 7,  /* back */
2309       0, 1, 4, 7,  /* left */
2310     };
2311     PetscInt       faceVerticesHex[24] = {
2312       1, 2, 3, 0,  /* bottom */
2313       4, 5, 6, 7,  /* top */
2314       0, 3, 5, 4,  /* front */
2315       3, 2, 6, 5,  /* right */
2316       2, 1, 7, 6,  /* back */
2317       1, 0, 4, 7,  /* left */
2318     };
2319 
2320     for (i = 0; i < faceSizeHex; ++i) sortedIndices[i] = indices[i];
2321     ierr = PetscSortInt(faceSizeHex, sortedIndices);CHKERRQ(ierr);
2322     for (iFace = 0; iFace < 6; ++iFace) {
2323       const PetscInt ii = iFace*faceSizeHex;
2324       PetscInt       fVertex, cVertex;
2325 
2326       if ((sortedIndices[0] == faceVerticesHexSorted[ii+0]) &&
2327           (sortedIndices[1] == faceVerticesHexSorted[ii+1]) &&
2328           (sortedIndices[2] == faceVerticesHexSorted[ii+2]) &&
2329           (sortedIndices[3] == faceVerticesHexSorted[ii+3])) {
2330         for (fVertex = 0; fVertex < faceSizeHex; ++fVertex) {
2331           for (cVertex = 0; cVertex < faceSizeHex; ++cVertex) {
2332             if (indices[cVertex] == faceVerticesHex[ii+fVertex]) {
2333               faceVertices[fVertex] = origVertices[cVertex];
2334               break;
2335             }
2336           }
2337         }
2338         found = PETSC_TRUE;
2339         break;
2340       }
2341     }
2342     if (!found) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid hex crossface");
2343     if (posOriented) *posOriented = PETSC_TRUE;
2344     PetscFunctionReturn(0);
2345   } else if (cellDim == 3 && numCorners == 10) {
2346     /* Quadratic tet */
2347     /* Faces are determined by the first 3 vertices (corners of faces) */
2348     const PetscInt faceSizeTet = 6;
2349     PetscInt       sortedIndices[6], i, iFace;
2350     PetscBool      found                     = PETSC_FALSE;
2351     PetscInt       faceVerticesTetSorted[24] = {
2352       0, 1, 2,  6, 7, 8, /* bottom */
2353       0, 3, 4,  6, 7, 9,  /* front */
2354       1, 4, 5,  7, 8, 9,  /* right */
2355       2, 3, 5,  6, 8, 9,  /* left */
2356     };
2357     PetscInt       faceVerticesTet[24] = {
2358       0, 1, 2,  6, 7, 8, /* bottom */
2359       0, 4, 3,  6, 7, 9,  /* front */
2360       1, 5, 4,  7, 8, 9,  /* right */
2361       2, 3, 5,  8, 6, 9,  /* left */
2362     };
2363 
2364     for (i = 0; i < faceSizeTet; ++i) sortedIndices[i] = indices[i];
2365     ierr = PetscSortInt(faceSizeTet, sortedIndices);CHKERRQ(ierr);
2366     for (iFace=0; iFace < 4; ++iFace) {
2367       const PetscInt ii = iFace*faceSizeTet;
2368       PetscInt       fVertex, cVertex;
2369 
2370       if ((sortedIndices[0] == faceVerticesTetSorted[ii+0]) &&
2371           (sortedIndices[1] == faceVerticesTetSorted[ii+1]) &&
2372           (sortedIndices[2] == faceVerticesTetSorted[ii+2]) &&
2373           (sortedIndices[3] == faceVerticesTetSorted[ii+3])) {
2374         for (fVertex = 0; fVertex < faceSizeTet; ++fVertex) {
2375           for (cVertex = 0; cVertex < faceSizeTet; ++cVertex) {
2376             if (indices[cVertex] == faceVerticesTet[ii+fVertex]) {
2377               faceVertices[fVertex] = origVertices[cVertex];
2378               break;
2379             }
2380           }
2381         }
2382         found = PETSC_TRUE;
2383         break;
2384       }
2385     }
2386     if (!found) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid tet crossface");
2387     if (posOriented) *posOriented = PETSC_TRUE;
2388     PetscFunctionReturn(0);
2389   } else if (cellDim == 3 && numCorners == 27) {
2390     /* Quadratic hexes (I hate this)
2391        A hex is two oriented quads with the normal of the first
2392        pointing up at the second.
2393 
2394          7---6
2395         /|  /|
2396        4---5 |
2397        | 3-|-2
2398        |/  |/
2399        0---1
2400 
2401        Faces are determined by the first 4 vertices (corners of faces) */
2402     const PetscInt faceSizeQuadHex = 9;
2403     PetscInt       sortedIndices[9], i, iFace;
2404     PetscBool      found                         = PETSC_FALSE;
2405     PetscInt       faceVerticesQuadHexSorted[54] = {
2406       0, 1, 2, 3,  8, 9, 10, 11,  24, /* bottom */
2407       4, 5, 6, 7,  12, 13, 14, 15,  25, /* top */
2408       0, 1, 4, 5,  8, 12, 16, 17,  22, /* front */
2409       1, 2, 5, 6,  9, 13, 17, 18,  21, /* right */
2410       2, 3, 6, 7,  10, 14, 18, 19,  23, /* back */
2411       0, 3, 4, 7,  11, 15, 16, 19,  20, /* left */
2412     };
2413     PetscInt       faceVerticesQuadHex[54] = {
2414       3, 2, 1, 0,  10, 9, 8, 11,  24, /* bottom */
2415       4, 5, 6, 7,  12, 13, 14, 15,  25, /* top */
2416       0, 1, 5, 4,  8, 17, 12, 16,  22, /* front */
2417       1, 2, 6, 5,  9, 18, 13, 17,  21, /* right */
2418       2, 3, 7, 6,  10, 19, 14, 18,  23, /* back */
2419       3, 0, 4, 7,  11, 16, 15, 19,  20 /* left */
2420     };
2421 
2422     for (i = 0; i < faceSizeQuadHex; ++i) sortedIndices[i] = indices[i];
2423     ierr = PetscSortInt(faceSizeQuadHex, sortedIndices);CHKERRQ(ierr);
2424     for (iFace = 0; iFace < 6; ++iFace) {
2425       const PetscInt ii = iFace*faceSizeQuadHex;
2426       PetscInt       fVertex, cVertex;
2427 
2428       if ((sortedIndices[0] == faceVerticesQuadHexSorted[ii+0]) &&
2429           (sortedIndices[1] == faceVerticesQuadHexSorted[ii+1]) &&
2430           (sortedIndices[2] == faceVerticesQuadHexSorted[ii+2]) &&
2431           (sortedIndices[3] == faceVerticesQuadHexSorted[ii+3])) {
2432         for (fVertex = 0; fVertex < faceSizeQuadHex; ++fVertex) {
2433           for (cVertex = 0; cVertex < faceSizeQuadHex; ++cVertex) {
2434             if (indices[cVertex] == faceVerticesQuadHex[ii+fVertex]) {
2435               faceVertices[fVertex] = origVertices[cVertex];
2436               break;
2437             }
2438           }
2439         }
2440         found = PETSC_TRUE;
2441         break;
2442       }
2443     }
2444     if (!found) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid hex crossface");
2445     if (posOriented) *posOriented = PETSC_TRUE;
2446     PetscFunctionReturn(0);
2447   } else SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Unknown cell type for faceOrientation().");
2448   if (!posOrient) {
2449     if (debug) {ierr = PetscPrintf(comm, "  Reversing initial face orientation\n");CHKERRQ(ierr);}
2450     for (f = 0; f < faceSize; ++f) faceVertices[f] = origVertices[faceSize-1 - f];
2451   } else {
2452     if (debug) {ierr = PetscPrintf(comm, "  Keeping initial face orientation\n");CHKERRQ(ierr);}
2453     for (f = 0; f < faceSize; ++f) faceVertices[f] = origVertices[f];
2454   }
2455   if (posOriented) *posOriented = posOrient;
2456   PetscFunctionReturn(0);
2457 }
2458 
2459 /*@
2460   DMPlexGetOrientedFace - Given a cell and a face, as a set of vertices, return the oriented face, as a set of vertices,
2461   in faceVertices. The orientation is such that the face normal points out of the cell
2462 
2463   Not collective
2464 
2465   Input Parameters:
2466 + dm           - The original mesh
2467 . cell         - The cell mesh point
2468 . faceSize     - The number of vertices on the face
2469 . face         - The face vertices
2470 . numCorners   - The number of vertices on the cell
2471 . indices      - Local numbering of face vertices in cell cone
2472 - origVertices - Original face vertices
2473 
2474   Output Parameter:
2475 + faceVertices - The face vertices properly oriented
2476 - posOriented  - PETSC_TRUE if the face was oriented with outward normal
2477 
2478   Level: developer
2479 
2480 .seealso: DMPlexGetCone()
2481 @*/
2482 PetscErrorCode DMPlexGetOrientedFace(DM dm, PetscInt cell, PetscInt faceSize, const PetscInt face[], PetscInt numCorners, PetscInt indices[], PetscInt origVertices[], PetscInt faceVertices[], PetscBool *posOriented)
2483 {
2484   const PetscInt *cone = NULL;
2485   PetscInt        coneSize, v, f, v2;
2486   PetscInt        oppositeVertex = -1;
2487   PetscErrorCode  ierr;
2488 
2489   PetscFunctionBegin;
2490   ierr = DMPlexGetConeSize(dm, cell, &coneSize);CHKERRQ(ierr);
2491   ierr = DMPlexGetCone(dm, cell, &cone);CHKERRQ(ierr);
2492   for (v = 0, v2 = 0; v < coneSize; ++v) {
2493     PetscBool found = PETSC_FALSE;
2494 
2495     for (f = 0; f < faceSize; ++f) {
2496       if (face[f] == cone[v]) {
2497         found = PETSC_TRUE; break;
2498       }
2499     }
2500     if (found) {
2501       indices[v2]      = v;
2502       origVertices[v2] = cone[v];
2503       ++v2;
2504     } else {
2505       oppositeVertex = v;
2506     }
2507   }
2508   ierr = DMPlexGetFaceOrientation(dm, cell, numCorners, indices, oppositeVertex, origVertices, faceVertices, posOriented);CHKERRQ(ierr);
2509   PetscFunctionReturn(0);
2510 }
2511 
2512 /*
2513   DMPlexInsertFace_Internal - Puts a face into the mesh
2514 
2515   Not collective
2516 
2517   Input Parameters:
2518   + dm              - The DMPlex
2519   . numFaceVertex   - The number of vertices in the face
2520   . faceVertices    - The vertices in the face for dm
2521   . subfaceVertices - The vertices in the face for subdm
2522   . numCorners      - The number of vertices in the cell
2523   . cell            - A cell in dm containing the face
2524   . subcell         - A cell in subdm containing the face
2525   . firstFace       - First face in the mesh
2526   - newFacePoint    - Next face in the mesh
2527 
2528   Output Parameters:
2529   . newFacePoint - Contains next face point number on input, updated on output
2530 
2531   Level: developer
2532 */
2533 static PetscErrorCode DMPlexInsertFace_Internal(DM dm, DM subdm, PetscInt numFaceVertices, const PetscInt faceVertices[], const PetscInt subfaceVertices[], PetscInt numCorners, PetscInt cell, PetscInt subcell, PetscInt firstFace, PetscInt *newFacePoint)
2534 {
2535   MPI_Comm        comm;
2536   DM_Plex        *submesh = (DM_Plex*) subdm->data;
2537   const PetscInt *faces;
2538   PetscInt        numFaces, coneSize;
2539   PetscErrorCode  ierr;
2540 
2541   PetscFunctionBegin;
2542   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
2543   ierr = DMPlexGetConeSize(subdm, subcell, &coneSize);CHKERRQ(ierr);
2544   if (coneSize != 1) SETERRQ2(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cone size of cell %d is %d != 1", cell, coneSize);
2545 #if 0
2546   /* Cannot use this because support() has not been constructed yet */
2547   ierr = DMPlexGetJoin(subdm, numFaceVertices, subfaceVertices, &numFaces, &faces);CHKERRQ(ierr);
2548 #else
2549   {
2550     PetscInt f;
2551 
2552     numFaces = 0;
2553     ierr     = DMGetWorkArray(subdm, 1, PETSC_INT, (void **) &faces);CHKERRQ(ierr);
2554     for (f = firstFace; f < *newFacePoint; ++f) {
2555       PetscInt dof, off, d;
2556 
2557       ierr = PetscSectionGetDof(submesh->coneSection, f, &dof);CHKERRQ(ierr);
2558       ierr = PetscSectionGetOffset(submesh->coneSection, f, &off);CHKERRQ(ierr);
2559       /* Yes, I know this is quadratic, but I expect the sizes to be <5 */
2560       for (d = 0; d < dof; ++d) {
2561         const PetscInt p = submesh->cones[off+d];
2562         PetscInt       v;
2563 
2564         for (v = 0; v < numFaceVertices; ++v) {
2565           if (subfaceVertices[v] == p) break;
2566         }
2567         if (v == numFaceVertices) break;
2568       }
2569       if (d == dof) {
2570         numFaces               = 1;
2571         ((PetscInt*) faces)[0] = f;
2572       }
2573     }
2574   }
2575 #endif
2576   if (numFaces > 1) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Vertex set had %d faces, not one", numFaces);
2577   else if (numFaces == 1) {
2578     /* Add the other cell neighbor for this face */
2579     ierr = DMPlexSetCone(subdm, subcell, faces);CHKERRQ(ierr);
2580   } else {
2581     PetscInt *indices, *origVertices, *orientedVertices, *orientedSubVertices, v, ov;
2582     PetscBool posOriented;
2583 
2584     ierr                = DMGetWorkArray(subdm, 4*numFaceVertices * sizeof(PetscInt), PETSC_INT, &orientedVertices);CHKERRQ(ierr);
2585     origVertices        = &orientedVertices[numFaceVertices];
2586     indices             = &orientedVertices[numFaceVertices*2];
2587     orientedSubVertices = &orientedVertices[numFaceVertices*3];
2588     ierr                = DMPlexGetOrientedFace(dm, cell, numFaceVertices, faceVertices, numCorners, indices, origVertices, orientedVertices, &posOriented);CHKERRQ(ierr);
2589     /* TODO: I know that routine should return a permutation, not the indices */
2590     for (v = 0; v < numFaceVertices; ++v) {
2591       const PetscInt vertex = faceVertices[v], subvertex = subfaceVertices[v];
2592       for (ov = 0; ov < numFaceVertices; ++ov) {
2593         if (orientedVertices[ov] == vertex) {
2594           orientedSubVertices[ov] = subvertex;
2595           break;
2596         }
2597       }
2598       if (ov == numFaceVertices) SETERRQ1(comm, PETSC_ERR_PLIB, "Could not find face vertex %d in orientated set", vertex);
2599     }
2600     ierr = DMPlexSetCone(subdm, *newFacePoint, orientedSubVertices);CHKERRQ(ierr);
2601     ierr = DMPlexSetCone(subdm, subcell, newFacePoint);CHKERRQ(ierr);
2602     ierr = DMRestoreWorkArray(subdm, 4*numFaceVertices * sizeof(PetscInt), PETSC_INT, &orientedVertices);CHKERRQ(ierr);
2603     ++(*newFacePoint);
2604   }
2605 #if 0
2606   ierr = DMPlexRestoreJoin(subdm, numFaceVertices, subfaceVertices, &numFaces, &faces);CHKERRQ(ierr);
2607 #else
2608   ierr = DMRestoreWorkArray(subdm, 1, PETSC_INT, (void **) &faces);CHKERRQ(ierr);
2609 #endif
2610   PetscFunctionReturn(0);
2611 }
2612 
2613 static PetscErrorCode DMPlexCreateSubmesh_Uninterpolated(DM dm, DMLabel vertexLabel, PetscInt value, DM subdm)
2614 {
2615   MPI_Comm        comm;
2616   DMLabel         subpointMap;
2617   IS              subvertexIS,  subcellIS;
2618   const PetscInt *subVertices, *subCells;
2619   PetscInt        numSubVertices, firstSubVertex, numSubCells;
2620   PetscInt       *subface, maxConeSize, numSubFaces = 0, firstSubFace, newFacePoint, nFV = 0;
2621   PetscInt        vStart, vEnd, c, f;
2622   PetscErrorCode  ierr;
2623 
2624   PetscFunctionBegin;
2625   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
2626   /* Create subpointMap which marks the submesh */
2627   ierr = DMLabelCreate("subpoint_map", &subpointMap);CHKERRQ(ierr);
2628   ierr = DMPlexSetSubpointMap(subdm, subpointMap);CHKERRQ(ierr);
2629   ierr = DMLabelDestroy(&subpointMap);CHKERRQ(ierr);
2630   if (vertexLabel) {ierr = DMPlexMarkSubmesh_Uninterpolated(dm, vertexLabel, value, subpointMap, &numSubFaces, &nFV, subdm);CHKERRQ(ierr);}
2631   /* Setup chart */
2632   ierr = DMLabelGetStratumSize(subpointMap, 0, &numSubVertices);CHKERRQ(ierr);
2633   ierr = DMLabelGetStratumSize(subpointMap, 2, &numSubCells);CHKERRQ(ierr);
2634   ierr = DMPlexSetChart(subdm, 0, numSubCells+numSubFaces+numSubVertices);CHKERRQ(ierr);
2635   ierr = DMPlexSetVTKCellHeight(subdm, 1);CHKERRQ(ierr);
2636   /* Set cone sizes */
2637   firstSubVertex = numSubCells;
2638   firstSubFace   = numSubCells+numSubVertices;
2639   newFacePoint   = firstSubFace;
2640   ierr = DMLabelGetStratumIS(subpointMap, 0, &subvertexIS);CHKERRQ(ierr);
2641   if (subvertexIS) {ierr = ISGetIndices(subvertexIS, &subVertices);CHKERRQ(ierr);}
2642   ierr = DMLabelGetStratumIS(subpointMap, 2, &subcellIS);CHKERRQ(ierr);
2643   if (subcellIS) {ierr = ISGetIndices(subcellIS, &subCells);CHKERRQ(ierr);}
2644   for (c = 0; c < numSubCells; ++c) {
2645     ierr = DMPlexSetConeSize(subdm, c, 1);CHKERRQ(ierr);
2646   }
2647   for (f = firstSubFace; f < firstSubFace+numSubFaces; ++f) {
2648     ierr = DMPlexSetConeSize(subdm, f, nFV);CHKERRQ(ierr);
2649   }
2650   ierr = DMSetUp(subdm);CHKERRQ(ierr);
2651   /* Create face cones */
2652   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
2653   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, NULL);CHKERRQ(ierr);
2654   ierr = DMGetWorkArray(subdm, maxConeSize, PETSC_INT, (void**) &subface);CHKERRQ(ierr);
2655   for (c = 0; c < numSubCells; ++c) {
2656     const PetscInt cell    = subCells[c];
2657     const PetscInt subcell = c;
2658     PetscInt      *closure = NULL;
2659     PetscInt       closureSize, cl, numCorners = 0, faceSize = 0;
2660 
2661     ierr = DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
2662     for (cl = 0; cl < closureSize*2; cl += 2) {
2663       const PetscInt point = closure[cl];
2664       PetscInt       subVertex;
2665 
2666       if ((point >= vStart) && (point < vEnd)) {
2667         ++numCorners;
2668         ierr = PetscFindInt(point, numSubVertices, subVertices, &subVertex);CHKERRQ(ierr);
2669         if (subVertex >= 0) {
2670           closure[faceSize] = point;
2671           subface[faceSize] = firstSubVertex+subVertex;
2672           ++faceSize;
2673         }
2674       }
2675     }
2676     if (faceSize > nFV) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Invalid submesh: Too many vertices %d of an element on the surface", faceSize);
2677     if (faceSize == nFV) {
2678       ierr = DMPlexInsertFace_Internal(dm, subdm, faceSize, closure, subface, numCorners, cell, subcell, firstSubFace, &newFacePoint);CHKERRQ(ierr);
2679     }
2680     ierr = DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
2681   }
2682   ierr = DMRestoreWorkArray(subdm, maxConeSize, PETSC_INT, (void**) &subface);CHKERRQ(ierr);
2683   ierr = DMPlexSymmetrize(subdm);CHKERRQ(ierr);
2684   ierr = DMPlexStratify(subdm);CHKERRQ(ierr);
2685   /* Build coordinates */
2686   {
2687     PetscSection coordSection, subCoordSection;
2688     Vec          coordinates, subCoordinates;
2689     PetscScalar *coords, *subCoords;
2690     PetscInt     numComp, coordSize, v;
2691     const char  *name;
2692 
2693     ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
2694     ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
2695     ierr = DMGetCoordinateSection(subdm, &subCoordSection);CHKERRQ(ierr);
2696     ierr = PetscSectionSetNumFields(subCoordSection, 1);CHKERRQ(ierr);
2697     ierr = PetscSectionGetFieldComponents(coordSection, 0, &numComp);CHKERRQ(ierr);
2698     ierr = PetscSectionSetFieldComponents(subCoordSection, 0, numComp);CHKERRQ(ierr);
2699     ierr = PetscSectionSetChart(subCoordSection, firstSubVertex, firstSubVertex+numSubVertices);CHKERRQ(ierr);
2700     for (v = 0; v < numSubVertices; ++v) {
2701       const PetscInt vertex    = subVertices[v];
2702       const PetscInt subvertex = firstSubVertex+v;
2703       PetscInt       dof;
2704 
2705       ierr = PetscSectionGetDof(coordSection, vertex, &dof);CHKERRQ(ierr);
2706       ierr = PetscSectionSetDof(subCoordSection, subvertex, dof);CHKERRQ(ierr);
2707       ierr = PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof);CHKERRQ(ierr);
2708     }
2709     ierr = PetscSectionSetUp(subCoordSection);CHKERRQ(ierr);
2710     ierr = PetscSectionGetStorageSize(subCoordSection, &coordSize);CHKERRQ(ierr);
2711     ierr = VecCreate(PETSC_COMM_SELF, &subCoordinates);CHKERRQ(ierr);
2712     ierr = PetscObjectGetName((PetscObject)coordinates,&name);CHKERRQ(ierr);
2713     ierr = PetscObjectSetName((PetscObject)subCoordinates,name);CHKERRQ(ierr);
2714     ierr = VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
2715     ierr = VecSetType(subCoordinates,VECSTANDARD);CHKERRQ(ierr);
2716     if (coordSize) {
2717       ierr = VecGetArray(coordinates,    &coords);CHKERRQ(ierr);
2718       ierr = VecGetArray(subCoordinates, &subCoords);CHKERRQ(ierr);
2719       for (v = 0; v < numSubVertices; ++v) {
2720         const PetscInt vertex    = subVertices[v];
2721         const PetscInt subvertex = firstSubVertex+v;
2722         PetscInt       dof, off, sdof, soff, d;
2723 
2724         ierr = PetscSectionGetDof(coordSection, vertex, &dof);CHKERRQ(ierr);
2725         ierr = PetscSectionGetOffset(coordSection, vertex, &off);CHKERRQ(ierr);
2726         ierr = PetscSectionGetDof(subCoordSection, subvertex, &sdof);CHKERRQ(ierr);
2727         ierr = PetscSectionGetOffset(subCoordSection, subvertex, &soff);CHKERRQ(ierr);
2728         if (dof != sdof) SETERRQ4(comm, PETSC_ERR_PLIB, "Coordinate dimension %d on subvertex %d, vertex %d should be %d", sdof, subvertex, vertex, dof);
2729         for (d = 0; d < dof; ++d) subCoords[soff+d] = coords[off+d];
2730       }
2731       ierr = VecRestoreArray(coordinates,    &coords);CHKERRQ(ierr);
2732       ierr = VecRestoreArray(subCoordinates, &subCoords);CHKERRQ(ierr);
2733     }
2734     ierr = DMSetCoordinatesLocal(subdm, subCoordinates);CHKERRQ(ierr);
2735     ierr = VecDestroy(&subCoordinates);CHKERRQ(ierr);
2736   }
2737   /* Cleanup */
2738   if (subvertexIS) {ierr = ISRestoreIndices(subvertexIS, &subVertices);CHKERRQ(ierr);}
2739   ierr = ISDestroy(&subvertexIS);CHKERRQ(ierr);
2740   if (subcellIS) {ierr = ISRestoreIndices(subcellIS, &subCells);CHKERRQ(ierr);}
2741   ierr = ISDestroy(&subcellIS);CHKERRQ(ierr);
2742   PetscFunctionReturn(0);
2743 }
2744 
2745 PETSC_STATIC_INLINE PetscInt DMPlexFilterPoint_Internal(PetscInt point, PetscInt firstSubPoint, PetscInt numSubPoints, const PetscInt subPoints[])
2746 {
2747   PetscInt       subPoint;
2748   PetscErrorCode ierr;
2749 
2750   ierr = PetscFindInt(point, numSubPoints, subPoints, &subPoint); if (ierr < 0) return ierr;
2751   return subPoint < 0 ? subPoint : firstSubPoint+subPoint;
2752 }
2753 
2754 static PetscErrorCode DMPlexCreateSubmeshGeneric_Interpolated(DM dm, DMLabel label, PetscInt value, PetscBool isCohesive, PetscInt cellHeight, DM subdm)
2755 {
2756   MPI_Comm         comm;
2757   DMLabel          subpointMap;
2758   IS              *subpointIS;
2759   const PetscInt **subpoints;
2760   PetscInt        *numSubPoints, *firstSubPoint, *coneNew, *orntNew;
2761   PetscInt         totSubPoints = 0, maxConeSize, cMax, cEnd, dim, p, d, v;
2762   PetscMPIInt      rank;
2763   PetscErrorCode   ierr;
2764 
2765   PetscFunctionBegin;
2766   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
2767   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
2768   /* Create subpointMap which marks the submesh */
2769   ierr = DMLabelCreate("subpoint_map", &subpointMap);CHKERRQ(ierr);
2770   ierr = DMPlexSetSubpointMap(subdm, subpointMap);CHKERRQ(ierr);
2771   if (cellHeight) {
2772     if (isCohesive) {ierr = DMPlexMarkCohesiveSubmesh_Interpolated(dm, label, value, subpointMap, subdm);CHKERRQ(ierr);}
2773     else            {ierr = DMPlexMarkSubmesh_Interpolated(dm, label, value, subpointMap, subdm);CHKERRQ(ierr);}
2774   } else {
2775     DMLabel         depth;
2776     IS              pointIS;
2777     const PetscInt *points;
2778     PetscInt        numPoints;
2779 
2780     ierr = DMPlexGetDepthLabel(dm, &depth);CHKERRQ(ierr);
2781     ierr = DMLabelGetStratumSize(label, value, &numPoints);CHKERRQ(ierr);
2782     ierr = DMLabelGetStratumIS(label, value, &pointIS);CHKERRQ(ierr);
2783     ierr = ISGetIndices(pointIS, &points);CHKERRQ(ierr);
2784     for (p = 0; p < numPoints; ++p) {
2785       PetscInt *closure = NULL;
2786       PetscInt  closureSize, c, pdim;
2787 
2788       ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
2789       for (c = 0; c < closureSize*2; c += 2) {
2790         ierr = DMLabelGetValue(depth, closure[c], &pdim);CHKERRQ(ierr);
2791         ierr = DMLabelSetValue(subpointMap, closure[c], pdim);CHKERRQ(ierr);
2792       }
2793       ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
2794     }
2795     ierr = ISRestoreIndices(pointIS, &points);CHKERRQ(ierr);
2796     ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
2797   }
2798   ierr = DMLabelDestroy(&subpointMap);CHKERRQ(ierr);
2799   ierr = DMPlexGetHeightStratum(dm, 0, NULL, &cEnd);CHKERRQ(ierr);
2800   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
2801   cMax = (cMax < 0) ? cEnd : cMax;
2802   /* Setup chart */
2803   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
2804   ierr = PetscMalloc4(dim+1,&numSubPoints,dim+1,&firstSubPoint,dim+1,&subpointIS,dim+1,&subpoints);CHKERRQ(ierr);
2805   for (d = 0; d <= dim; ++d) {
2806     ierr = DMLabelGetStratumSize(subpointMap, d, &numSubPoints[d]);CHKERRQ(ierr);
2807     totSubPoints += numSubPoints[d];
2808   }
2809   ierr = DMPlexSetChart(subdm, 0, totSubPoints);CHKERRQ(ierr);
2810   ierr = DMPlexSetVTKCellHeight(subdm, cellHeight);CHKERRQ(ierr);
2811   /* Set cone sizes */
2812   firstSubPoint[dim] = 0;
2813   firstSubPoint[0]   = firstSubPoint[dim] + numSubPoints[dim];
2814   if (dim > 1) {firstSubPoint[dim-1] = firstSubPoint[0]     + numSubPoints[0];}
2815   if (dim > 2) {firstSubPoint[dim-2] = firstSubPoint[dim-1] + numSubPoints[dim-1];}
2816   for (d = 0; d <= dim; ++d) {
2817     ierr = DMLabelGetStratumIS(subpointMap, d, &subpointIS[d]);CHKERRQ(ierr);
2818     if (subpointIS[d]) {ierr = ISGetIndices(subpointIS[d], &subpoints[d]);CHKERRQ(ierr);}
2819   }
2820   for (d = 0; d <= dim; ++d) {
2821     for (p = 0; p < numSubPoints[d]; ++p) {
2822       const PetscInt  point    = subpoints[d][p];
2823       const PetscInt  subpoint = firstSubPoint[d] + p;
2824       const PetscInt *cone;
2825       PetscInt        coneSize, coneSizeNew, c, val;
2826 
2827       ierr = DMPlexGetConeSize(dm, point, &coneSize);CHKERRQ(ierr);
2828       ierr = DMPlexSetConeSize(subdm, subpoint, coneSize);CHKERRQ(ierr);
2829       if (cellHeight && (d == dim)) {
2830         ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
2831         for (c = 0, coneSizeNew = 0; c < coneSize; ++c) {
2832           ierr = DMLabelGetValue(subpointMap, cone[c], &val);CHKERRQ(ierr);
2833           if (val >= 0) coneSizeNew++;
2834         }
2835         ierr = DMPlexSetConeSize(subdm, subpoint, coneSizeNew);CHKERRQ(ierr);
2836       }
2837     }
2838   }
2839   ierr = DMSetUp(subdm);CHKERRQ(ierr);
2840   /* Set cones */
2841   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, NULL);CHKERRQ(ierr);
2842   ierr = PetscMalloc2(maxConeSize,&coneNew,maxConeSize,&orntNew);CHKERRQ(ierr);
2843   for (d = 0; d <= dim; ++d) {
2844     for (p = 0; p < numSubPoints[d]; ++p) {
2845       const PetscInt  point    = subpoints[d][p];
2846       const PetscInt  subpoint = firstSubPoint[d] + p;
2847       const PetscInt *cone, *ornt;
2848       PetscInt        coneSize, subconeSize, coneSizeNew, c, subc, fornt = 0;
2849 
2850       if (d == dim-1) {
2851         const PetscInt *support, *cone, *ornt;
2852         PetscInt        supportSize, coneSize, s, subc;
2853 
2854         ierr = DMPlexGetSupport(dm, point, &support);CHKERRQ(ierr);
2855         ierr = DMPlexGetSupportSize(dm, point, &supportSize);CHKERRQ(ierr);
2856         for (s = 0; s < supportSize; ++s) {
2857           if ((support[s] < cMax) || (support[s] >= cEnd)) continue;
2858           ierr = PetscFindInt(support[s], numSubPoints[d+1], subpoints[d+1], &subc);CHKERRQ(ierr);
2859           if (subc >= 0) {
2860             const PetscInt ccell = subpoints[d+1][subc];
2861 
2862             ierr = DMPlexGetCone(dm, ccell, &cone);
2863             ierr = DMPlexGetConeSize(dm, ccell, &coneSize);
2864             ierr = DMPlexGetConeOrientation(dm, ccell, &ornt);
2865             for (c = 0; c < coneSize; ++c) {
2866               if (cone[c] == point) {
2867                 fornt = ornt[c];
2868                 break;
2869               }
2870             }
2871             break;
2872           }
2873         }
2874       }
2875       ierr = DMPlexGetConeSize(dm, point, &coneSize);CHKERRQ(ierr);
2876       ierr = DMPlexGetConeSize(subdm, subpoint, &subconeSize);CHKERRQ(ierr);
2877       ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
2878       ierr = DMPlexGetConeOrientation(dm, point, &ornt);CHKERRQ(ierr);
2879       for (c = 0, coneSizeNew = 0; c < coneSize; ++c) {
2880         ierr = PetscFindInt(cone[c], numSubPoints[d-1], subpoints[d-1], &subc);CHKERRQ(ierr);
2881         if (subc >= 0) {
2882           coneNew[coneSizeNew] = firstSubPoint[d-1] + subc;
2883           orntNew[coneSizeNew] = ornt[c];
2884           ++coneSizeNew;
2885         }
2886       }
2887       if (coneSizeNew != subconeSize) SETERRQ2(comm, PETSC_ERR_PLIB, "Number of cone points located %d does not match subcone size %d", coneSizeNew, subconeSize);
2888       if (fornt < 0) {
2889         /* This should be replaced by a call to DMPlexReverseCell() */
2890 #if 0
2891         ierr = DMPlexReverseCell(subdm, subpoint);CHKERRQ(ierr);
2892 #else
2893         for (c = 0; c < coneSizeNew/2 + coneSizeNew%2; ++c) {
2894           PetscInt faceSize, tmp;
2895 
2896           tmp        = coneNew[c];
2897           coneNew[c] = coneNew[coneSizeNew-1-c];
2898           coneNew[coneSizeNew-1-c] = tmp;
2899           ierr = DMPlexGetConeSize(dm, cone[c], &faceSize);CHKERRQ(ierr);
2900           tmp        = orntNew[c] >= 0 ? -(faceSize-orntNew[c]) : faceSize+orntNew[c];
2901           orntNew[c] = orntNew[coneSizeNew-1-c] >= 0 ? -(faceSize-orntNew[coneSizeNew-1-c]) : faceSize+orntNew[coneSizeNew-1-c];
2902           orntNew[coneSizeNew-1-c] = tmp;
2903         }
2904       }
2905       ierr = DMPlexSetCone(subdm, subpoint, coneNew);CHKERRQ(ierr);
2906       ierr = DMPlexSetConeOrientation(subdm, subpoint, orntNew);CHKERRQ(ierr);
2907 #endif
2908     }
2909   }
2910   ierr = PetscFree2(coneNew,orntNew);CHKERRQ(ierr);
2911   ierr = DMPlexSymmetrize(subdm);CHKERRQ(ierr);
2912   ierr = DMPlexStratify(subdm);CHKERRQ(ierr);
2913   /* Build coordinates */
2914   {
2915     PetscSection coordSection, subCoordSection;
2916     Vec          coordinates, subCoordinates;
2917     PetscScalar *coords, *subCoords;
2918     PetscInt     cdim, numComp, coordSize;
2919     const char  *name;
2920 
2921     ierr = DMGetCoordinateDim(dm, &cdim);CHKERRQ(ierr);
2922     ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
2923     ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
2924     ierr = DMGetCoordinateSection(subdm, &subCoordSection);CHKERRQ(ierr);
2925     ierr = PetscSectionSetNumFields(subCoordSection, 1);CHKERRQ(ierr);
2926     ierr = PetscSectionGetFieldComponents(coordSection, 0, &numComp);CHKERRQ(ierr);
2927     ierr = PetscSectionSetFieldComponents(subCoordSection, 0, numComp);CHKERRQ(ierr);
2928     ierr = PetscSectionSetChart(subCoordSection, firstSubPoint[0], firstSubPoint[0]+numSubPoints[0]);CHKERRQ(ierr);
2929     for (v = 0; v < numSubPoints[0]; ++v) {
2930       const PetscInt vertex    = subpoints[0][v];
2931       const PetscInt subvertex = firstSubPoint[0]+v;
2932       PetscInt       dof;
2933 
2934       ierr = PetscSectionGetDof(coordSection, vertex, &dof);CHKERRQ(ierr);
2935       ierr = PetscSectionSetDof(subCoordSection, subvertex, dof);CHKERRQ(ierr);
2936       ierr = PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof);CHKERRQ(ierr);
2937     }
2938     ierr = PetscSectionSetUp(subCoordSection);CHKERRQ(ierr);
2939     ierr = PetscSectionGetStorageSize(subCoordSection, &coordSize);CHKERRQ(ierr);
2940     ierr = VecCreate(PETSC_COMM_SELF, &subCoordinates);CHKERRQ(ierr);
2941     ierr = PetscObjectGetName((PetscObject)coordinates,&name);CHKERRQ(ierr);
2942     ierr = PetscObjectSetName((PetscObject)subCoordinates,name);CHKERRQ(ierr);
2943     ierr = VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
2944     ierr = VecSetBlockSize(subCoordinates, cdim);CHKERRQ(ierr);
2945     ierr = VecSetType(subCoordinates,VECSTANDARD);CHKERRQ(ierr);
2946     ierr = VecGetArray(coordinates,    &coords);CHKERRQ(ierr);
2947     ierr = VecGetArray(subCoordinates, &subCoords);CHKERRQ(ierr);
2948     for (v = 0; v < numSubPoints[0]; ++v) {
2949       const PetscInt vertex    = subpoints[0][v];
2950       const PetscInt subvertex = firstSubPoint[0]+v;
2951       PetscInt dof, off, sdof, soff, d;
2952 
2953       ierr = PetscSectionGetDof(coordSection, vertex, &dof);CHKERRQ(ierr);
2954       ierr = PetscSectionGetOffset(coordSection, vertex, &off);CHKERRQ(ierr);
2955       ierr = PetscSectionGetDof(subCoordSection, subvertex, &sdof);CHKERRQ(ierr);
2956       ierr = PetscSectionGetOffset(subCoordSection, subvertex, &soff);CHKERRQ(ierr);
2957       if (dof != sdof) SETERRQ4(comm, PETSC_ERR_PLIB, "Coordinate dimension %d on subvertex %d, vertex %d should be %d", sdof, subvertex, vertex, dof);
2958       for (d = 0; d < dof; ++d) subCoords[soff+d] = coords[off+d];
2959     }
2960     ierr = VecRestoreArray(coordinates,    &coords);CHKERRQ(ierr);
2961     ierr = VecRestoreArray(subCoordinates, &subCoords);CHKERRQ(ierr);
2962     ierr = DMSetCoordinatesLocal(subdm, subCoordinates);CHKERRQ(ierr);
2963     ierr = VecDestroy(&subCoordinates);CHKERRQ(ierr);
2964   }
2965   /* Build SF: We need this complexity because subpoints might not be selected on the owning process */
2966   {
2967     PetscSF            sfPoint, sfPointSub;
2968     IS                 subpIS;
2969     const PetscSFNode *remotePoints;
2970     PetscSFNode       *sremotePoints, *newLocalPoints, *newOwners;
2971     const PetscInt    *localPoints, *subpoints;
2972     PetscInt          *slocalPoints;
2973     PetscInt           numRoots, numLeaves, numSubpoints = 0, numSubroots, numSubleaves = 0, l, sl, ll, pStart, pEnd, p;
2974     PetscMPIInt        rank;
2975 
2976     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRQ(ierr);
2977     ierr = DMGetPointSF(dm, &sfPoint);CHKERRQ(ierr);
2978     ierr = DMGetPointSF(subdm, &sfPointSub);CHKERRQ(ierr);
2979     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
2980     ierr = DMPlexGetChart(subdm, NULL, &numSubroots);CHKERRQ(ierr);
2981     ierr = DMPlexCreateSubpointIS(subdm, &subpIS);CHKERRQ(ierr);
2982     if (subpIS) {
2983       ierr = ISGetIndices(subpIS, &subpoints);CHKERRQ(ierr);
2984       ierr = ISGetLocalSize(subpIS, &numSubpoints);CHKERRQ(ierr);
2985     }
2986     ierr = PetscSFGetGraph(sfPoint, &numRoots, &numLeaves, &localPoints, &remotePoints);CHKERRQ(ierr);
2987     if (numRoots >= 0) {
2988       ierr = PetscMalloc2(pEnd-pStart,&newLocalPoints,numRoots,&newOwners);CHKERRQ(ierr);
2989       for (p = 0; p < pEnd-pStart; ++p) {
2990         newLocalPoints[p].rank  = -2;
2991         newLocalPoints[p].index = -2;
2992       }
2993       /* Set subleaves */
2994       for (l = 0; l < numLeaves; ++l) {
2995         const PetscInt point    = localPoints[l];
2996         const PetscInt subpoint = DMPlexFilterPoint_Internal(point, 0, numSubpoints, subpoints);
2997 
2998         if (subpoint < 0) continue;
2999         newLocalPoints[point-pStart].rank  = rank;
3000         newLocalPoints[point-pStart].index = subpoint;
3001         ++numSubleaves;
3002       }
3003       /* Must put in owned subpoints */
3004       for (p = pStart; p < pEnd; ++p) {
3005         const PetscInt subpoint = DMPlexFilterPoint_Internal(p, 0, numSubpoints, subpoints);
3006 
3007         if (subpoint < 0) {
3008           newOwners[p-pStart].rank  = -3;
3009           newOwners[p-pStart].index = -3;
3010         } else {
3011           newOwners[p-pStart].rank  = rank;
3012           newOwners[p-pStart].index = subpoint;
3013         }
3014       }
3015       ierr = PetscSFReduceBegin(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC);CHKERRQ(ierr);
3016       ierr = PetscSFReduceEnd(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC);CHKERRQ(ierr);
3017       ierr = PetscSFBcastBegin(sfPoint, MPIU_2INT, newOwners, newLocalPoints);CHKERRQ(ierr);
3018       ierr = PetscSFBcastEnd(sfPoint, MPIU_2INT, newOwners, newLocalPoints);CHKERRQ(ierr);
3019       ierr = PetscMalloc1(numSubleaves, &slocalPoints);CHKERRQ(ierr);
3020       ierr = PetscMalloc1(numSubleaves, &sremotePoints);CHKERRQ(ierr);
3021       for (l = 0, sl = 0, ll = 0; l < numLeaves; ++l) {
3022         const PetscInt point    = localPoints[l];
3023         const PetscInt subpoint = DMPlexFilterPoint_Internal(point, 0, numSubpoints, subpoints);
3024 
3025         if (subpoint < 0) continue;
3026         if (newLocalPoints[point].rank == rank) {++ll; continue;}
3027         slocalPoints[sl]        = subpoint;
3028         sremotePoints[sl].rank  = newLocalPoints[point].rank;
3029         sremotePoints[sl].index = newLocalPoints[point].index;
3030         if (sremotePoints[sl].rank  < 0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote rank for local point %d", point);
3031         if (sremotePoints[sl].index < 0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote subpoint for local point %d", point);
3032         ++sl;
3033       }
3034       if (sl + ll != numSubleaves) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatch in number of subleaves %d + %d != %d", sl, ll, numSubleaves);
3035       ierr = PetscFree2(newLocalPoints,newOwners);CHKERRQ(ierr);
3036       ierr = PetscSFSetGraph(sfPointSub, numSubroots, sl, slocalPoints, PETSC_OWN_POINTER, sremotePoints, PETSC_OWN_POINTER);CHKERRQ(ierr);
3037     }
3038     if (subpIS) {
3039       ierr = ISRestoreIndices(subpIS, &subpoints);CHKERRQ(ierr);
3040       ierr = ISDestroy(&subpIS);CHKERRQ(ierr);
3041     }
3042   }
3043   /* Cleanup */
3044   for (d = 0; d <= dim; ++d) {
3045     if (subpointIS[d]) {ierr = ISRestoreIndices(subpointIS[d], &subpoints[d]);CHKERRQ(ierr);}
3046     ierr = ISDestroy(&subpointIS[d]);CHKERRQ(ierr);
3047   }
3048   ierr = PetscFree4(numSubPoints,firstSubPoint,subpointIS,subpoints);CHKERRQ(ierr);
3049   PetscFunctionReturn(0);
3050 }
3051 
3052 static PetscErrorCode DMPlexCreateSubmesh_Interpolated(DM dm, DMLabel vertexLabel, PetscInt value, DM subdm)
3053 {
3054   PetscErrorCode ierr;
3055 
3056   PetscFunctionBegin;
3057   ierr = DMPlexCreateSubmeshGeneric_Interpolated(dm, vertexLabel, value, PETSC_FALSE, 1, subdm);CHKERRQ(ierr);
3058   PetscFunctionReturn(0);
3059 }
3060 
3061 /*@
3062   DMPlexCreateSubmesh - Extract a hypersurface from the mesh using vertices defined by a label
3063 
3064   Input Parameters:
3065 + dm           - The original mesh
3066 . vertexLabel  - The DMLabel marking vertices contained in the surface
3067 - value        - The label value to use
3068 
3069   Output Parameter:
3070 . subdm - The surface mesh
3071 
3072   Note: This function produces a DMLabel mapping original points in the submesh to their depth. This can be obtained using DMPlexGetSubpointMap().
3073 
3074   Level: developer
3075 
3076 .seealso: DMPlexGetSubpointMap(), DMGetLabel(), DMLabelSetValue()
3077 @*/
3078 PetscErrorCode DMPlexCreateSubmesh(DM dm, DMLabel vertexLabel, PetscInt value, DM *subdm)
3079 {
3080   PetscInt       dim, cdim, depth;
3081   PetscErrorCode ierr;
3082 
3083   PetscFunctionBegin;
3084   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3085   PetscValidPointer(subdm, 3);
3086   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
3087   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3088   ierr = DMCreate(PetscObjectComm((PetscObject)dm), subdm);CHKERRQ(ierr);
3089   ierr = DMSetType(*subdm, DMPLEX);CHKERRQ(ierr);
3090   ierr = DMSetDimension(*subdm, dim-1);CHKERRQ(ierr);
3091   ierr = DMGetCoordinateDim(dm, &cdim);CHKERRQ(ierr);
3092   ierr = DMSetCoordinateDim(*subdm, cdim);CHKERRQ(ierr);
3093   if (depth == dim) {
3094     ierr = DMPlexCreateSubmesh_Interpolated(dm, vertexLabel, value, *subdm);CHKERRQ(ierr);
3095   } else {
3096     ierr = DMPlexCreateSubmesh_Uninterpolated(dm, vertexLabel, value, *subdm);CHKERRQ(ierr);
3097   }
3098   PetscFunctionReturn(0);
3099 }
3100 
3101 static PetscErrorCode DMPlexCreateCohesiveSubmesh_Uninterpolated(DM dm, PetscBool hasLagrange, const char label[], PetscInt value, DM subdm)
3102 {
3103   MPI_Comm        comm;
3104   DMLabel         subpointMap;
3105   IS              subvertexIS;
3106   const PetscInt *subVertices;
3107   PetscInt        numSubVertices, firstSubVertex, numSubCells, *subCells = NULL;
3108   PetscInt       *subface, maxConeSize, numSubFaces, firstSubFace, newFacePoint, nFV;
3109   PetscInt        cMax, c, f;
3110   PetscErrorCode  ierr;
3111 
3112   PetscFunctionBegin;
3113   ierr = PetscObjectGetComm((PetscObject)dm, &comm);CHKERRQ(ierr);
3114   /* Create subpointMap which marks the submesh */
3115   ierr = DMLabelCreate("subpoint_map", &subpointMap);CHKERRQ(ierr);
3116   ierr = DMPlexSetSubpointMap(subdm, subpointMap);CHKERRQ(ierr);
3117   ierr = DMLabelDestroy(&subpointMap);CHKERRQ(ierr);
3118   ierr = DMPlexMarkCohesiveSubmesh_Uninterpolated(dm, hasLagrange, label, value, subpointMap, &numSubFaces, &nFV, &subCells, subdm);CHKERRQ(ierr);
3119   /* Setup chart */
3120   ierr = DMLabelGetStratumSize(subpointMap, 0, &numSubVertices);CHKERRQ(ierr);
3121   ierr = DMLabelGetStratumSize(subpointMap, 2, &numSubCells);CHKERRQ(ierr);
3122   ierr = DMPlexSetChart(subdm, 0, numSubCells+numSubFaces+numSubVertices);CHKERRQ(ierr);
3123   ierr = DMPlexSetVTKCellHeight(subdm, 1);CHKERRQ(ierr);
3124   /* Set cone sizes */
3125   firstSubVertex = numSubCells;
3126   firstSubFace   = numSubCells+numSubVertices;
3127   newFacePoint   = firstSubFace;
3128   ierr = DMLabelGetStratumIS(subpointMap, 0, &subvertexIS);CHKERRQ(ierr);
3129   if (subvertexIS) {ierr = ISGetIndices(subvertexIS, &subVertices);CHKERRQ(ierr);}
3130   for (c = 0; c < numSubCells; ++c) {
3131     ierr = DMPlexSetConeSize(subdm, c, 1);CHKERRQ(ierr);
3132   }
3133   for (f = firstSubFace; f < firstSubFace+numSubFaces; ++f) {
3134     ierr = DMPlexSetConeSize(subdm, f, nFV);CHKERRQ(ierr);
3135   }
3136   ierr = DMSetUp(subdm);CHKERRQ(ierr);
3137   /* Create face cones */
3138   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, NULL);CHKERRQ(ierr);
3139   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
3140   ierr = DMGetWorkArray(subdm, maxConeSize, PETSC_INT, (void**) &subface);CHKERRQ(ierr);
3141   for (c = 0; c < numSubCells; ++c) {
3142     const PetscInt  cell    = subCells[c];
3143     const PetscInt  subcell = c;
3144     const PetscInt *cone, *cells;
3145     PetscInt        numCells, subVertex, p, v;
3146 
3147     if (cell < cMax) continue;
3148     ierr = DMPlexGetCone(dm, cell, &cone);CHKERRQ(ierr);
3149     for (v = 0; v < nFV; ++v) {
3150       ierr = PetscFindInt(cone[v], numSubVertices, subVertices, &subVertex);CHKERRQ(ierr);
3151       subface[v] = firstSubVertex+subVertex;
3152     }
3153     ierr = DMPlexSetCone(subdm, newFacePoint, subface);CHKERRQ(ierr);
3154     ierr = DMPlexSetCone(subdm, subcell, &newFacePoint);CHKERRQ(ierr);
3155     ierr = DMPlexGetJoin(dm, nFV, cone, &numCells, &cells);CHKERRQ(ierr);
3156     /* Not true in parallel
3157     if (numCells != 2) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive cells should separate two cells"); */
3158     for (p = 0; p < numCells; ++p) {
3159       PetscInt negsubcell;
3160 
3161       if (cells[p] >= cMax) continue;
3162       /* I know this is a crap search */
3163       for (negsubcell = 0; negsubcell < numSubCells; ++negsubcell) {
3164         if (subCells[negsubcell] == cells[p]) break;
3165       }
3166       if (negsubcell == numSubCells) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not find negative face neighbor for cohesive cell %d", cell);
3167       ierr = DMPlexSetCone(subdm, negsubcell, &newFacePoint);CHKERRQ(ierr);
3168     }
3169     ierr = DMPlexRestoreJoin(dm, nFV, cone, &numCells, &cells);CHKERRQ(ierr);
3170     ++newFacePoint;
3171   }
3172   ierr = DMRestoreWorkArray(subdm, maxConeSize, PETSC_INT, (void**) &subface);CHKERRQ(ierr);
3173   ierr = DMPlexSymmetrize(subdm);CHKERRQ(ierr);
3174   ierr = DMPlexStratify(subdm);CHKERRQ(ierr);
3175   /* Build coordinates */
3176   {
3177     PetscSection coordSection, subCoordSection;
3178     Vec          coordinates, subCoordinates;
3179     PetscScalar *coords, *subCoords;
3180     PetscInt     cdim, numComp, coordSize, v;
3181     const char  *name;
3182 
3183     ierr = DMGetCoordinateDim(dm, &cdim);CHKERRQ(ierr);
3184     ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
3185     ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
3186     ierr = DMGetCoordinateSection(subdm, &subCoordSection);CHKERRQ(ierr);
3187     ierr = PetscSectionSetNumFields(subCoordSection, 1);CHKERRQ(ierr);
3188     ierr = PetscSectionGetFieldComponents(coordSection, 0, &numComp);CHKERRQ(ierr);
3189     ierr = PetscSectionSetFieldComponents(subCoordSection, 0, numComp);CHKERRQ(ierr);
3190     ierr = PetscSectionSetChart(subCoordSection, firstSubVertex, firstSubVertex+numSubVertices);CHKERRQ(ierr);
3191     for (v = 0; v < numSubVertices; ++v) {
3192       const PetscInt vertex    = subVertices[v];
3193       const PetscInt subvertex = firstSubVertex+v;
3194       PetscInt       dof;
3195 
3196       ierr = PetscSectionGetDof(coordSection, vertex, &dof);CHKERRQ(ierr);
3197       ierr = PetscSectionSetDof(subCoordSection, subvertex, dof);CHKERRQ(ierr);
3198       ierr = PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof);CHKERRQ(ierr);
3199     }
3200     ierr = PetscSectionSetUp(subCoordSection);CHKERRQ(ierr);
3201     ierr = PetscSectionGetStorageSize(subCoordSection, &coordSize);CHKERRQ(ierr);
3202     ierr = VecCreate(PETSC_COMM_SELF, &subCoordinates);CHKERRQ(ierr);
3203     ierr = PetscObjectGetName((PetscObject)coordinates,&name);CHKERRQ(ierr);
3204     ierr = PetscObjectSetName((PetscObject)subCoordinates,name);CHKERRQ(ierr);
3205     ierr = VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
3206     ierr = VecSetBlockSize(subCoordinates, cdim);CHKERRQ(ierr);
3207     ierr = VecSetType(subCoordinates,VECSTANDARD);CHKERRQ(ierr);
3208     ierr = VecGetArray(coordinates,    &coords);CHKERRQ(ierr);
3209     ierr = VecGetArray(subCoordinates, &subCoords);CHKERRQ(ierr);
3210     for (v = 0; v < numSubVertices; ++v) {
3211       const PetscInt vertex    = subVertices[v];
3212       const PetscInt subvertex = firstSubVertex+v;
3213       PetscInt       dof, off, sdof, soff, d;
3214 
3215       ierr = PetscSectionGetDof(coordSection, vertex, &dof);CHKERRQ(ierr);
3216       ierr = PetscSectionGetOffset(coordSection, vertex, &off);CHKERRQ(ierr);
3217       ierr = PetscSectionGetDof(subCoordSection, subvertex, &sdof);CHKERRQ(ierr);
3218       ierr = PetscSectionGetOffset(subCoordSection, subvertex, &soff);CHKERRQ(ierr);
3219       if (dof != sdof) SETERRQ4(comm, PETSC_ERR_PLIB, "Coordinate dimension %d on subvertex %d, vertex %d should be %d", sdof, subvertex, vertex, dof);
3220       for (d = 0; d < dof; ++d) subCoords[soff+d] = coords[off+d];
3221     }
3222     ierr = VecRestoreArray(coordinates,    &coords);CHKERRQ(ierr);
3223     ierr = VecRestoreArray(subCoordinates, &subCoords);CHKERRQ(ierr);
3224     ierr = DMSetCoordinatesLocal(subdm, subCoordinates);CHKERRQ(ierr);
3225     ierr = VecDestroy(&subCoordinates);CHKERRQ(ierr);
3226   }
3227   /* Build SF */
3228   CHKMEMQ;
3229   {
3230     PetscSF            sfPoint, sfPointSub;
3231     const PetscSFNode *remotePoints;
3232     PetscSFNode       *sremotePoints, *newLocalPoints, *newOwners;
3233     const PetscInt    *localPoints;
3234     PetscInt          *slocalPoints;
3235     PetscInt           numRoots, numLeaves, numSubRoots = numSubCells+numSubFaces+numSubVertices, numSubLeaves = 0, l, sl, ll, pStart, pEnd, p, vStart, vEnd;
3236     PetscMPIInt        rank;
3237 
3238     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRQ(ierr);
3239     ierr = DMGetPointSF(dm, &sfPoint);CHKERRQ(ierr);
3240     ierr = DMGetPointSF(subdm, &sfPointSub);CHKERRQ(ierr);
3241     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
3242     ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
3243     ierr = PetscSFGetGraph(sfPoint, &numRoots, &numLeaves, &localPoints, &remotePoints);CHKERRQ(ierr);
3244     if (numRoots >= 0) {
3245       /* Only vertices should be shared */
3246       ierr = PetscMalloc2(pEnd-pStart,&newLocalPoints,numRoots,&newOwners);CHKERRQ(ierr);
3247       for (p = 0; p < pEnd-pStart; ++p) {
3248         newLocalPoints[p].rank  = -2;
3249         newLocalPoints[p].index = -2;
3250       }
3251       /* Set subleaves */
3252       for (l = 0; l < numLeaves; ++l) {
3253         const PetscInt point    = localPoints[l];
3254         const PetscInt subPoint = DMPlexFilterPoint_Internal(point, firstSubVertex, numSubVertices, subVertices);
3255 
3256         if ((point < vStart) && (point >= vEnd)) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Should not be mapping anything but vertices, %d", point);
3257         if (subPoint < 0) continue;
3258         newLocalPoints[point-pStart].rank  = rank;
3259         newLocalPoints[point-pStart].index = subPoint;
3260         ++numSubLeaves;
3261       }
3262       /* Must put in owned subpoints */
3263       for (p = pStart; p < pEnd; ++p) {
3264         const PetscInt subPoint = DMPlexFilterPoint_Internal(p, firstSubVertex, numSubVertices, subVertices);
3265 
3266         if (subPoint < 0) {
3267           newOwners[p-pStart].rank  = -3;
3268           newOwners[p-pStart].index = -3;
3269         } else {
3270           newOwners[p-pStart].rank  = rank;
3271           newOwners[p-pStart].index = subPoint;
3272         }
3273       }
3274       ierr = PetscSFReduceBegin(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC);CHKERRQ(ierr);
3275       ierr = PetscSFReduceEnd(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC);CHKERRQ(ierr);
3276       ierr = PetscSFBcastBegin(sfPoint, MPIU_2INT, newOwners, newLocalPoints);CHKERRQ(ierr);
3277       ierr = PetscSFBcastEnd(sfPoint, MPIU_2INT, newOwners, newLocalPoints);CHKERRQ(ierr);
3278       ierr = PetscMalloc1(numSubLeaves,    &slocalPoints);CHKERRQ(ierr);
3279       ierr = PetscMalloc1(numSubLeaves, &sremotePoints);CHKERRQ(ierr);
3280       for (l = 0, sl = 0, ll = 0; l < numLeaves; ++l) {
3281         const PetscInt point    = localPoints[l];
3282         const PetscInt subPoint = DMPlexFilterPoint_Internal(point, firstSubVertex, numSubVertices, subVertices);
3283 
3284         if (subPoint < 0) continue;
3285         if (newLocalPoints[point].rank == rank) {++ll; continue;}
3286         slocalPoints[sl]        = subPoint;
3287         sremotePoints[sl].rank  = newLocalPoints[point].rank;
3288         sremotePoints[sl].index = newLocalPoints[point].index;
3289         if (sremotePoints[sl].rank  < 0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote rank for local point %d", point);
3290         if (sremotePoints[sl].index < 0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote subpoint for local point %d", point);
3291         ++sl;
3292       }
3293       ierr = PetscFree2(newLocalPoints,newOwners);CHKERRQ(ierr);
3294       if (sl + ll != numSubLeaves) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatch in number of subleaves %d + %d != %d", sl, ll, numSubLeaves);
3295       ierr = PetscSFSetGraph(sfPointSub, numSubRoots, sl, slocalPoints, PETSC_OWN_POINTER, sremotePoints, PETSC_OWN_POINTER);CHKERRQ(ierr);
3296     }
3297   }
3298   CHKMEMQ;
3299   /* Cleanup */
3300   if (subvertexIS) {ierr = ISRestoreIndices(subvertexIS, &subVertices);CHKERRQ(ierr);}
3301   ierr = ISDestroy(&subvertexIS);CHKERRQ(ierr);
3302   ierr = PetscFree(subCells);CHKERRQ(ierr);
3303   PetscFunctionReturn(0);
3304 }
3305 
3306 static PetscErrorCode DMPlexCreateCohesiveSubmesh_Interpolated(DM dm, const char labelname[], PetscInt value, DM subdm)
3307 {
3308   DMLabel        label = NULL;
3309   PetscErrorCode ierr;
3310 
3311   PetscFunctionBegin;
3312   if (labelname) {ierr = DMGetLabel(dm, labelname, &label);CHKERRQ(ierr);}
3313   ierr = DMPlexCreateSubmeshGeneric_Interpolated(dm, label, value, PETSC_TRUE, 1, subdm);CHKERRQ(ierr);
3314   PetscFunctionReturn(0);
3315 }
3316 
3317 /*@C
3318   DMPlexCreateCohesiveSubmesh - Extract from a mesh with cohesive cells the hypersurface defined by one face of the cells. Optionally, a Label an be given to restrict the cells.
3319 
3320   Input Parameters:
3321 + dm          - The original mesh
3322 . hasLagrange - The mesh has Lagrange unknowns in the cohesive cells
3323 . label       - A label name, or NULL
3324 - value  - A label value
3325 
3326   Output Parameter:
3327 . subdm - The surface mesh
3328 
3329   Note: This function produces a DMLabel mapping original points in the submesh to their depth. This can be obtained using DMPlexGetSubpointMap().
3330 
3331   Level: developer
3332 
3333 .seealso: DMPlexGetSubpointMap(), DMPlexCreateSubmesh()
3334 @*/
3335 PetscErrorCode DMPlexCreateCohesiveSubmesh(DM dm, PetscBool hasLagrange, const char label[], PetscInt value, DM *subdm)
3336 {
3337   PetscInt       dim, cdim, depth;
3338   PetscErrorCode ierr;
3339 
3340   PetscFunctionBegin;
3341   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3342   PetscValidPointer(subdm, 5);
3343   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
3344   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3345   ierr = DMCreate(PetscObjectComm((PetscObject)dm), subdm);CHKERRQ(ierr);
3346   ierr = DMSetType(*subdm, DMPLEX);CHKERRQ(ierr);
3347   ierr = DMSetDimension(*subdm, dim-1);CHKERRQ(ierr);
3348   ierr = DMGetCoordinateDim(dm, &cdim);CHKERRQ(ierr);
3349   ierr = DMSetCoordinateDim(*subdm, cdim);CHKERRQ(ierr);
3350   if (depth == dim) {
3351     ierr = DMPlexCreateCohesiveSubmesh_Interpolated(dm, label, value, *subdm);CHKERRQ(ierr);
3352   } else {
3353     ierr = DMPlexCreateCohesiveSubmesh_Uninterpolated(dm, hasLagrange, label, value, *subdm);CHKERRQ(ierr);
3354   }
3355   PetscFunctionReturn(0);
3356 }
3357 
3358 /*@
3359   DMPlexFilter - Extract a subset of mesh cells defined by a label as a separate mesh
3360 
3361   Input Parameters:
3362 + dm        - The original mesh
3363 . cellLabel - The DMLabel marking cells contained in the new mesh
3364 - value     - The label value to use
3365 
3366   Output Parameter:
3367 . subdm - The new mesh
3368 
3369   Note: This function produces a DMLabel mapping original points in the submesh to their depth. This can be obtained using DMPlexGetSubpointMap().
3370 
3371   Level: developer
3372 
3373 .seealso: DMPlexGetSubpointMap(), DMGetLabel(), DMLabelSetValue()
3374 @*/
3375 PetscErrorCode DMPlexFilter(DM dm, DMLabel cellLabel, PetscInt value, DM *subdm)
3376 {
3377   PetscInt       dim;
3378   PetscErrorCode ierr;
3379 
3380   PetscFunctionBegin;
3381   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3382   PetscValidPointer(subdm, 3);
3383   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
3384   ierr = DMCreate(PetscObjectComm((PetscObject) dm), subdm);CHKERRQ(ierr);
3385   ierr = DMSetType(*subdm, DMPLEX);CHKERRQ(ierr);
3386   ierr = DMSetDimension(*subdm, dim);CHKERRQ(ierr);
3387   /* Extract submesh in place, could be empty on some procs, could have inconsistency if procs do not both extract a shared cell */
3388   ierr = DMPlexCreateSubmeshGeneric_Interpolated(dm, cellLabel, value, PETSC_FALSE, 0, *subdm);CHKERRQ(ierr);
3389   PetscFunctionReturn(0);
3390 }
3391 
3392 /*@
3393   DMPlexGetSubpointMap - Returns a DMLabel with point dimension as values
3394 
3395   Input Parameter:
3396 . dm - The submesh DM
3397 
3398   Output Parameter:
3399 . subpointMap - The DMLabel of all the points from the original mesh in this submesh, or NULL if this is not a submesh
3400 
3401   Level: developer
3402 
3403 .seealso: DMPlexCreateSubmesh(), DMPlexCreateSubpointIS()
3404 @*/
3405 PetscErrorCode DMPlexGetSubpointMap(DM dm, DMLabel *subpointMap)
3406 {
3407   PetscFunctionBegin;
3408   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3409   PetscValidPointer(subpointMap, 2);
3410   *subpointMap = ((DM_Plex*) dm->data)->subpointMap;
3411   PetscFunctionReturn(0);
3412 }
3413 
3414 /*@
3415   DMPlexSetSubpointMap - Sets the DMLabel with point dimension as values
3416 
3417   Input Parameters:
3418 + dm - The submesh DM
3419 - subpointMap - The DMLabel of all the points from the original mesh in this submesh
3420 
3421   Note: Should normally not be called by the user, since it is set in DMPlexCreateSubmesh()
3422 
3423   Level: developer
3424 
3425 .seealso: DMPlexCreateSubmesh(), DMPlexCreateSubpointIS()
3426 @*/
3427 PetscErrorCode DMPlexSetSubpointMap(DM dm, DMLabel subpointMap)
3428 {
3429   DM_Plex       *mesh = (DM_Plex *) dm->data;
3430   DMLabel        tmp;
3431   PetscErrorCode ierr;
3432 
3433   PetscFunctionBegin;
3434   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3435   tmp  = mesh->subpointMap;
3436   mesh->subpointMap = subpointMap;
3437   ++mesh->subpointMap->refct;
3438   ierr = DMLabelDestroy(&tmp);CHKERRQ(ierr);
3439   PetscFunctionReturn(0);
3440 }
3441 
3442 /*@
3443   DMPlexCreateSubpointIS - Creates an IS covering the entire subdm chart with the original points as data
3444 
3445   Input Parameter:
3446 . dm - The submesh DM
3447 
3448   Output Parameter:
3449 . subpointIS - The IS of all the points from the original mesh in this submesh, or NULL if this is not a submesh
3450 
3451   Note: This IS is guaranteed to be sorted by the construction of the submesh
3452 
3453   Level: developer
3454 
3455 .seealso: DMPlexCreateSubmesh(), DMPlexGetSubpointMap()
3456 @*/
3457 PetscErrorCode DMPlexCreateSubpointIS(DM dm, IS *subpointIS)
3458 {
3459   MPI_Comm        comm;
3460   DMLabel         subpointMap;
3461   IS              is;
3462   const PetscInt *opoints;
3463   PetscInt       *points, *depths;
3464   PetscInt        depth, depStart, depEnd, d, pStart, pEnd, p, n, off;
3465   PetscErrorCode  ierr;
3466 
3467   PetscFunctionBegin;
3468   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3469   PetscValidPointer(subpointIS, 2);
3470   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
3471   *subpointIS = NULL;
3472   ierr = DMPlexGetSubpointMap(dm, &subpointMap);CHKERRQ(ierr);
3473   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3474   if (subpointMap && depth >= 0) {
3475     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
3476     if (pStart) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Submeshes must start the point numbering at 0, not %d", pStart);
3477     ierr = DMGetWorkArray(dm, depth+1, PETSC_INT, &depths);CHKERRQ(ierr);
3478     depths[0] = depth;
3479     depths[1] = 0;
3480     for(d = 2; d <= depth; ++d) {depths[d] = depth+1 - d;}
3481     ierr = PetscMalloc1(pEnd, &points);CHKERRQ(ierr);
3482     for(d = 0, off = 0; d <= depth; ++d) {
3483       const PetscInt dep = depths[d];
3484 
3485       ierr = DMPlexGetDepthStratum(dm, dep, &depStart, &depEnd);CHKERRQ(ierr);
3486       ierr = DMLabelGetStratumSize(subpointMap, dep, &n);CHKERRQ(ierr);
3487       if (((d < 2) && (depth > 1)) || (d == 1)) { /* Only check vertices and cells for now since the map is broken for others */
3488         if (n != depEnd-depStart) SETERRQ3(comm, PETSC_ERR_ARG_WRONG, "The number of mapped submesh points %d at depth %d should be %d", n, dep, depEnd-depStart);
3489       } else {
3490         if (!n) {
3491           if (d == 0) {
3492             /* Missing cells */
3493             for(p = 0; p < depEnd-depStart; ++p, ++off) points[off] = -1;
3494           } else {
3495             /* Missing faces */
3496             for(p = 0; p < depEnd-depStart; ++p, ++off) points[off] = PETSC_MAX_INT;
3497           }
3498         }
3499       }
3500       if (n) {
3501         ierr = DMLabelGetStratumIS(subpointMap, dep, &is);CHKERRQ(ierr);
3502         ierr = ISGetIndices(is, &opoints);CHKERRQ(ierr);
3503         for(p = 0; p < n; ++p, ++off) points[off] = opoints[p];
3504         ierr = ISRestoreIndices(is, &opoints);CHKERRQ(ierr);
3505         ierr = ISDestroy(&is);CHKERRQ(ierr);
3506       }
3507     }
3508     ierr = DMRestoreWorkArray(dm, depth+1, PETSC_INT, &depths);CHKERRQ(ierr);
3509     if (off != pEnd) SETERRQ2(comm, PETSC_ERR_ARG_WRONG, "The number of mapped submesh points %d should be %d", off, pEnd);
3510     ierr = ISCreateGeneral(PETSC_COMM_SELF, pEnd, points, PETSC_OWN_POINTER, subpointIS);CHKERRQ(ierr);
3511   }
3512   PetscFunctionReturn(0);
3513 }
3514