xref: /petsc/src/dm/impls/plex/plexsubmesh.c (revision dd169d643cbf4b5f50ca99c97eb8f9235c6ca078)
1 #include <petsc/private/dmpleximpl.h>    /*I      "petscdmplex.h"    I*/
2 #include <petsc/private/dmlabelimpl.h>   /*I      "petscdmlabel.h"   I*/
3 #include <petscsf.h>
4 
5 static PetscErrorCode DMPlexMarkBoundaryFaces_Internal(DM dm, PetscInt cellHeight, DMLabel label)
6 {
7   PetscInt       fStart, fEnd, f;
8   PetscErrorCode ierr;
9 
10   PetscFunctionBegin;
11   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
12   ierr = DMPlexGetHeightStratum(dm, cellHeight+1, &fStart, &fEnd);CHKERRQ(ierr);
13   for (f = fStart; f < fEnd; ++f) {
14     PetscInt supportSize;
15 
16     ierr = DMPlexGetSupportSize(dm, f, &supportSize);CHKERRQ(ierr);
17     if (supportSize == 1) {ierr = DMLabelSetValue(label, f, 1);CHKERRQ(ierr);}
18   }
19   PetscFunctionReturn(0);
20 }
21 
22 /*@
23   DMPlexMarkBoundaryFaces - Mark all faces on the boundary
24 
25   Not Collective
26 
27   Input Parameter:
28 . dm - The original DM
29 
30   Output Parameter:
31 . label - The DMLabel marking boundary faces with value 1
32 
33   Level: developer
34 
35 .seealso: DMLabelCreate(), DMCreateLabel()
36 @*/
37 PetscErrorCode DMPlexMarkBoundaryFaces(DM dm, DMLabel label)
38 {
39   PetscErrorCode ierr;
40 
41   PetscFunctionBegin;
42   ierr = DMPlexMarkBoundaryFaces_Internal(dm, 0, label);CHKERRQ(ierr);
43   PetscFunctionReturn(0);
44 }
45 
46 static PetscErrorCode DMPlexLabelComplete_Internal(DM dm, DMLabel label, PetscBool completeCells)
47 {
48   IS              valueIS;
49   const PetscInt *values;
50   PetscInt        numValues, v, cStart, cEnd;
51   PetscErrorCode  ierr;
52 
53   PetscFunctionBegin;
54   ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
55   ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
56   ierr = DMPlexGetHeightStratum(dm,0,&cStart,&cEnd);CHKERRQ(ierr);
57   ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
58   for (v = 0; v < numValues; ++v) {
59     IS              pointIS;
60     const PetscInt *points;
61     PetscInt        numPoints, p;
62 
63     ierr = DMLabelGetStratumSize(label, values[v], &numPoints);CHKERRQ(ierr);
64     ierr = DMLabelGetStratumIS(label, values[v], &pointIS);CHKERRQ(ierr);
65     ierr = ISGetIndices(pointIS, &points);CHKERRQ(ierr);
66     for (p = 0; p < numPoints; ++p) {
67       PetscInt  q = points[p];
68       PetscInt *closure = NULL;
69       PetscInt  closureSize, c;
70 
71       if (cStart <= q && q < cEnd && !completeCells) { /* skip cells */
72         continue;
73       }
74       ierr = DMPlexGetTransitiveClosure(dm, q, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
75       for (c = 0; c < closureSize*2; c += 2) {
76         ierr = DMLabelSetValue(label, closure[c], values[v]);CHKERRQ(ierr);
77       }
78       ierr = DMPlexRestoreTransitiveClosure(dm, q, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
79     }
80     ierr = ISRestoreIndices(pointIS, &points);CHKERRQ(ierr);
81     ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
82   }
83   ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
84   ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
85   PetscFunctionReturn(0);
86 }
87 
88 /*@
89   DMPlexLabelComplete - Starting with a label marking points on a surface, we add the transitive closure to the surface
90 
91   Input Parameters:
92 + dm - The DM
93 - label - A DMLabel marking the surface points
94 
95   Output Parameter:
96 . label - A DMLabel marking all surface points in the transitive closure
97 
98   Level: developer
99 
100 .seealso: DMPlexLabelCohesiveComplete()
101 @*/
102 PetscErrorCode DMPlexLabelComplete(DM dm, DMLabel label)
103 {
104   PetscErrorCode ierr;
105 
106   PetscFunctionBegin;
107   ierr = DMPlexLabelComplete_Internal(dm, label, PETSC_TRUE);CHKERRQ(ierr);
108   PetscFunctionReturn(0);
109 }
110 
111 /*@
112   DMPlexLabelAddCells - Starting with a label marking faces on a surface, we add a cell for each face
113 
114   Input Parameters:
115 + dm - The DM
116 - label - A DMLabel marking the surface points
117 
118   Output Parameter:
119 . label - A DMLabel incorporating cells
120 
121   Level: developer
122 
123   Note: The cells allow FEM boundary conditions to be applied using the cell geometry
124 
125 .seealso: DMPlexLabelComplete(), DMPlexLabelCohesiveComplete()
126 @*/
127 PetscErrorCode DMPlexLabelAddCells(DM dm, DMLabel label)
128 {
129   IS              valueIS;
130   const PetscInt *values;
131   PetscInt        numValues, v, cStart, cEnd, cEndInterior;
132   PetscErrorCode  ierr;
133 
134   PetscFunctionBegin;
135   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
136   ierr = DMPlexGetHybridBounds(dm, &cEndInterior, NULL, NULL, NULL);CHKERRQ(ierr);
137   cEnd = cEndInterior < 0 ? cEnd : cEndInterior;
138   ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
139   ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
140   ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
141   for (v = 0; v < numValues; ++v) {
142     IS              pointIS;
143     const PetscInt *points;
144     PetscInt        numPoints, p;
145 
146     ierr = DMLabelGetStratumSize(label, values[v], &numPoints);CHKERRQ(ierr);
147     ierr = DMLabelGetStratumIS(label, values[v], &pointIS);CHKERRQ(ierr);
148     ierr = ISGetIndices(pointIS, &points);CHKERRQ(ierr);
149     for (p = 0; p < numPoints; ++p) {
150       PetscInt *closure = NULL;
151       PetscInt  closureSize, point, cl;
152 
153       ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_FALSE, &closureSize, &closure);CHKERRQ(ierr);
154       for (cl = closureSize-1; cl > 0; --cl) {
155         point = closure[cl*2];
156         if ((point >= cStart) && (point < cEnd)) {ierr = DMLabelSetValue(label, point, values[v]);CHKERRQ(ierr); break;}
157       }
158       ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_FALSE, &closureSize, &closure);CHKERRQ(ierr);
159     }
160     ierr = ISRestoreIndices(pointIS, &points);CHKERRQ(ierr);
161     ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
162   }
163   ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
164   ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
165   PetscFunctionReturn(0);
166 }
167 
168 /*@
169   DMPlexLabelClearCells - Remove cells from a label
170 
171   Input Parameters:
172 + dm - The DM
173 - label - A DMLabel marking surface points and their adjacent cells
174 
175   Output Parameter:
176 . label - A DMLabel without cells
177 
178   Level: developer
179 
180   Note: This undoes DMPlexLabelAddCells()
181 
182 .seealso: DMPlexLabelComplete(), DMPlexLabelCohesiveComplete(), DMPlexLabelAddCells()
183 @*/
184 PetscErrorCode DMPlexLabelClearCells(DM dm, DMLabel label)
185 {
186   IS              valueIS;
187   const PetscInt *values;
188   PetscInt        numValues, v, cStart, cEnd, cEndInterior;
189   PetscErrorCode  ierr;
190 
191   PetscFunctionBegin;
192   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
193   ierr = DMPlexGetHybridBounds(dm, &cEndInterior, NULL, NULL, NULL);CHKERRQ(ierr);
194   cEnd = cEndInterior < 0 ? cEnd : cEndInterior;
195   ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
196   ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
197   ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
198   for (v = 0; v < numValues; ++v) {
199     IS              pointIS;
200     const PetscInt *points;
201     PetscInt        numPoints, p;
202 
203     ierr = DMLabelGetStratumSize(label, values[v], &numPoints);CHKERRQ(ierr);
204     ierr = DMLabelGetStratumIS(label, values[v], &pointIS);CHKERRQ(ierr);
205     ierr = ISGetIndices(pointIS, &points);CHKERRQ(ierr);
206     for (p = 0; p < numPoints; ++p) {
207       PetscInt point = points[p];
208 
209       if (point >= cStart && point < cEnd) {
210         ierr = DMLabelClearValue(label,point,values[v]);CHKERRQ(ierr);
211       }
212     }
213     ierr = ISRestoreIndices(pointIS, &points);CHKERRQ(ierr);
214     ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
215   }
216   ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
217   ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
218   PetscFunctionReturn(0);
219 }
220 
221 /* take (oldEnd, added) pairs, ordered by height and convert them to (oldstart, newstart) pairs, ordered by ascending
222  * index (skipping first, which is (0,0)) */
223 PETSC_STATIC_INLINE PetscErrorCode DMPlexShiftPointSetUp_Internal(PetscInt depth, PetscInt depthShift[])
224 {
225   PetscInt d, off = 0;
226 
227   PetscFunctionBegin;
228   /* sort by (oldend): yes this is an O(n^2) sort, we expect depth <= 3 */
229   for (d = 0; d < depth; d++) {
230     PetscInt firstd = d;
231     PetscInt firstStart = depthShift[2*d];
232     PetscInt e;
233 
234     for (e = d+1; e <= depth; e++) {
235       if (depthShift[2*e] < firstStart) {
236         firstd = e;
237         firstStart = depthShift[2*d];
238       }
239     }
240     if (firstd != d) {
241       PetscInt swap[2];
242 
243       e = firstd;
244       swap[0] = depthShift[2*d];
245       swap[1] = depthShift[2*d+1];
246       depthShift[2*d]   = depthShift[2*e];
247       depthShift[2*d+1] = depthShift[2*e+1];
248       depthShift[2*e]   = swap[0];
249       depthShift[2*e+1] = swap[1];
250     }
251   }
252   /* convert (oldstart, added) to (oldstart, newstart) */
253   for (d = 0; d <= depth; d++) {
254     off += depthShift[2*d+1];
255     depthShift[2*d+1] = depthShift[2*d] + off;
256   }
257   PetscFunctionReturn(0);
258 }
259 
260 /* depthShift is a list of (old, new) pairs */
261 PETSC_STATIC_INLINE PetscInt DMPlexShiftPoint_Internal(PetscInt p, PetscInt depth, PetscInt depthShift[])
262 {
263   PetscInt d;
264   PetscInt newOff = 0;
265 
266   for (d = 0; d <= depth; d++) {
267     if (p < depthShift[2*d]) return p + newOff;
268     else newOff = depthShift[2*d+1] - depthShift[2*d];
269   }
270   return p + newOff;
271 }
272 
273 /* depthShift is a list of (old, new) pairs */
274 PETSC_STATIC_INLINE PetscInt DMPlexShiftPointInverse_Internal(PetscInt p, PetscInt depth, PetscInt depthShift[])
275 {
276   PetscInt d;
277   PetscInt newOff = 0;
278 
279   for (d = 0; d <= depth; d++) {
280     if (p < depthShift[2*d+1]) return p + newOff;
281     else newOff = depthShift[2*d] - depthShift[2*d+1];
282   }
283   return p + newOff;
284 }
285 
286 static PetscErrorCode DMPlexShiftSizes_Internal(DM dm, PetscInt depthShift[], DM dmNew)
287 {
288   PetscInt       depth = 0, d, pStart, pEnd, p;
289   PetscErrorCode ierr;
290 
291   PetscFunctionBegin;
292   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
293   if (depth < 0) PetscFunctionReturn(0);
294   /* Step 1: Expand chart */
295   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
296   pEnd = DMPlexShiftPoint_Internal(pEnd,depth,depthShift);
297   ierr = DMPlexSetChart(dmNew, pStart, pEnd);CHKERRQ(ierr);
298   /* Step 2: Set cone and support sizes */
299   for (d = 0; d <= depth; ++d) {
300     ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
301     for (p = pStart; p < pEnd; ++p) {
302       PetscInt newp = DMPlexShiftPoint_Internal(p, depth, depthShift);
303       PetscInt size;
304 
305       ierr = DMPlexGetConeSize(dm, p, &size);CHKERRQ(ierr);
306       ierr = DMPlexSetConeSize(dmNew, newp, size);CHKERRQ(ierr);
307       ierr = DMPlexGetSupportSize(dm, p, &size);CHKERRQ(ierr);
308       ierr = DMPlexSetSupportSize(dmNew, newp, size);CHKERRQ(ierr);
309     }
310   }
311   PetscFunctionReturn(0);
312 }
313 
314 static PetscErrorCode DMPlexShiftPoints_Internal(DM dm, PetscInt depthShift[], DM dmNew)
315 {
316   PetscInt      *newpoints;
317   PetscInt       depth = 0, maxConeSize, maxSupportSize, maxConeSizeNew, maxSupportSizeNew, pStart, pEnd, p;
318   PetscErrorCode ierr;
319 
320   PetscFunctionBegin;
321   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
322   if (depth < 0) PetscFunctionReturn(0);
323   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);CHKERRQ(ierr);
324   ierr = DMPlexGetMaxSizes(dmNew, &maxConeSizeNew, &maxSupportSizeNew);CHKERRQ(ierr);
325   ierr = PetscMalloc1(PetscMax(PetscMax(maxConeSize, maxSupportSize), PetscMax(maxConeSizeNew, maxSupportSizeNew)),&newpoints);CHKERRQ(ierr);
326   /* Step 5: Set cones and supports */
327   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
328   for (p = pStart; p < pEnd; ++p) {
329     const PetscInt *points = NULL, *orientations = NULL;
330     PetscInt        size,sizeNew, i, newp = DMPlexShiftPoint_Internal(p, depth, depthShift);
331 
332     ierr = DMPlexGetConeSize(dm, p, &size);CHKERRQ(ierr);
333     ierr = DMPlexGetCone(dm, p, &points);CHKERRQ(ierr);
334     ierr = DMPlexGetConeOrientation(dm, p, &orientations);CHKERRQ(ierr);
335     for (i = 0; i < size; ++i) {
336       newpoints[i] = DMPlexShiftPoint_Internal(points[i], depth, depthShift);
337     }
338     ierr = DMPlexSetCone(dmNew, newp, newpoints);CHKERRQ(ierr);
339     ierr = DMPlexSetConeOrientation(dmNew, newp, orientations);CHKERRQ(ierr);
340     ierr = DMPlexGetSupportSize(dm, p, &size);CHKERRQ(ierr);
341     ierr = DMPlexGetSupportSize(dmNew, newp, &sizeNew);CHKERRQ(ierr);
342     ierr = DMPlexGetSupport(dm, p, &points);CHKERRQ(ierr);
343     for (i = 0; i < size; ++i) {
344       newpoints[i] = DMPlexShiftPoint_Internal(points[i], depth, depthShift);
345     }
346     for (i = size; i < sizeNew; ++i) newpoints[i] = 0;
347     ierr = DMPlexSetSupport(dmNew, newp, newpoints);CHKERRQ(ierr);
348   }
349   ierr = PetscFree(newpoints);CHKERRQ(ierr);
350   PetscFunctionReturn(0);
351 }
352 
353 static PetscErrorCode DMPlexShiftCoordinates_Internal(DM dm, PetscInt depthShift[], DM dmNew)
354 {
355   PetscSection   coordSection, newCoordSection;
356   Vec            coordinates, newCoordinates;
357   PetscScalar   *coords, *newCoords;
358   PetscInt       coordSize, sStart, sEnd;
359   PetscInt       dim, depth = 0, cStart, cEnd, cStartNew, cEndNew, c, vStart, vEnd, vStartNew, vEndNew, v;
360   PetscBool      hasCells;
361   PetscErrorCode ierr;
362 
363   PetscFunctionBegin;
364   ierr = DMGetCoordinateDim(dm, &dim);CHKERRQ(ierr);
365   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
366   /* Step 8: Convert coordinates */
367   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
368   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
369   ierr = DMPlexGetDepthStratum(dmNew, 0, &vStartNew, &vEndNew);CHKERRQ(ierr);
370   ierr = DMPlexGetHeightStratum(dmNew, 0, &cStartNew, &cEndNew);CHKERRQ(ierr);
371   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
372   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &newCoordSection);CHKERRQ(ierr);
373   ierr = PetscSectionSetNumFields(newCoordSection, 1);CHKERRQ(ierr);
374   ierr = PetscSectionSetFieldComponents(newCoordSection, 0, dim);CHKERRQ(ierr);
375   ierr = PetscSectionGetChart(coordSection, &sStart, &sEnd);CHKERRQ(ierr);
376   hasCells = sStart == cStart ? PETSC_TRUE : PETSC_FALSE;
377   ierr = PetscSectionSetChart(newCoordSection, hasCells ? cStartNew : vStartNew, vEndNew);CHKERRQ(ierr);
378   if (hasCells) {
379     for (c = cStart; c < cEnd; ++c) {
380       PetscInt cNew = DMPlexShiftPoint_Internal(c, depth, depthShift), dof;
381 
382       ierr = PetscSectionGetDof(coordSection, c, &dof);CHKERRQ(ierr);
383       ierr = PetscSectionSetDof(newCoordSection, cNew, dof);CHKERRQ(ierr);
384       ierr = PetscSectionSetFieldDof(newCoordSection, cNew, 0, dof);CHKERRQ(ierr);
385     }
386   }
387   for (v = vStartNew; v < vEndNew; ++v) {
388     ierr = PetscSectionSetDof(newCoordSection, v, dim);CHKERRQ(ierr);
389     ierr = PetscSectionSetFieldDof(newCoordSection, v, 0, dim);CHKERRQ(ierr);
390   }
391   ierr = PetscSectionSetUp(newCoordSection);CHKERRQ(ierr);
392   ierr = DMSetCoordinateSection(dmNew, PETSC_DETERMINE, newCoordSection);CHKERRQ(ierr);
393   ierr = PetscSectionGetStorageSize(newCoordSection, &coordSize);CHKERRQ(ierr);
394   ierr = VecCreate(PETSC_COMM_SELF, &newCoordinates);CHKERRQ(ierr);
395   ierr = PetscObjectSetName((PetscObject) newCoordinates, "coordinates");CHKERRQ(ierr);
396   ierr = VecSetSizes(newCoordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
397   ierr = VecSetBlockSize(newCoordinates, dim);CHKERRQ(ierr);
398   ierr = VecSetType(newCoordinates,VECSTANDARD);CHKERRQ(ierr);
399   ierr = DMSetCoordinatesLocal(dmNew, newCoordinates);CHKERRQ(ierr);
400   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
401   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
402   ierr = VecGetArray(newCoordinates, &newCoords);CHKERRQ(ierr);
403   if (hasCells) {
404     for (c = cStart; c < cEnd; ++c) {
405       PetscInt cNew = DMPlexShiftPoint_Internal(c, depth, depthShift), dof, off, noff, d;
406 
407       ierr = PetscSectionGetDof(coordSection, c, &dof);CHKERRQ(ierr);
408       ierr = PetscSectionGetOffset(coordSection, c, &off);CHKERRQ(ierr);
409       ierr = PetscSectionGetOffset(newCoordSection, cNew, &noff);CHKERRQ(ierr);
410       for (d = 0; d < dof; ++d) newCoords[noff+d] = coords[off+d];
411     }
412   }
413   for (v = vStart; v < vEnd; ++v) {
414     PetscInt dof, off, noff, d;
415 
416     ierr = PetscSectionGetDof(coordSection, v, &dof);CHKERRQ(ierr);
417     ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
418     ierr = PetscSectionGetOffset(newCoordSection, DMPlexShiftPoint_Internal(v, depth, depthShift), &noff);CHKERRQ(ierr);
419     for (d = 0; d < dof; ++d) newCoords[noff+d] = coords[off+d];
420   }
421   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
422   ierr = VecRestoreArray(newCoordinates, &newCoords);CHKERRQ(ierr);
423   ierr = VecDestroy(&newCoordinates);CHKERRQ(ierr);
424   ierr = PetscSectionDestroy(&newCoordSection);CHKERRQ(ierr);
425   PetscFunctionReturn(0);
426 }
427 
428 static PetscErrorCode DMPlexShiftSF_Internal(DM dm, PetscInt depthShift[], DM dmNew)
429 {
430   PetscInt           depth = 0;
431   PetscSF            sfPoint, sfPointNew;
432   const PetscSFNode *remotePoints;
433   PetscSFNode       *gremotePoints;
434   const PetscInt    *localPoints;
435   PetscInt          *glocalPoints, *newLocation, *newRemoteLocation;
436   PetscInt           numRoots, numLeaves, l, pStart, pEnd, totShift = 0;
437   PetscErrorCode     ierr;
438 
439   PetscFunctionBegin;
440   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
441   /* Step 9: Convert pointSF */
442   ierr = DMGetPointSF(dm, &sfPoint);CHKERRQ(ierr);
443   ierr = DMGetPointSF(dmNew, &sfPointNew);CHKERRQ(ierr);
444   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
445   ierr = PetscSFGetGraph(sfPoint, &numRoots, &numLeaves, &localPoints, &remotePoints);CHKERRQ(ierr);
446   totShift = DMPlexShiftPoint_Internal(pEnd,depth,depthShift) - pEnd;
447   if (numRoots >= 0) {
448     ierr = PetscMalloc2(numRoots,&newLocation,pEnd-pStart,&newRemoteLocation);CHKERRQ(ierr);
449     for (l=0; l<numRoots; l++) newLocation[l] = DMPlexShiftPoint_Internal(l, depth, depthShift);
450     ierr = PetscSFBcastBegin(sfPoint, MPIU_INT, newLocation, newRemoteLocation);CHKERRQ(ierr);
451     ierr = PetscSFBcastEnd(sfPoint, MPIU_INT, newLocation, newRemoteLocation);CHKERRQ(ierr);
452     ierr = PetscMalloc1(numLeaves,    &glocalPoints);CHKERRQ(ierr);
453     ierr = PetscMalloc1(numLeaves, &gremotePoints);CHKERRQ(ierr);
454     for (l = 0; l < numLeaves; ++l) {
455       glocalPoints[l]        = DMPlexShiftPoint_Internal(localPoints[l], depth, depthShift);
456       gremotePoints[l].rank  = remotePoints[l].rank;
457       gremotePoints[l].index = newRemoteLocation[localPoints[l]];
458     }
459     ierr = PetscFree2(newLocation,newRemoteLocation);CHKERRQ(ierr);
460     ierr = PetscSFSetGraph(sfPointNew, numRoots + totShift, numLeaves, glocalPoints, PETSC_OWN_POINTER, gremotePoints, PETSC_OWN_POINTER);CHKERRQ(ierr);
461   }
462   PetscFunctionReturn(0);
463 }
464 
465 static PetscErrorCode DMPlexShiftLabels_Internal(DM dm, PetscInt depthShift[], DM dmNew)
466 {
467   PetscSF            sfPoint;
468   DMLabel            vtkLabel, ghostLabel;
469   const PetscSFNode *leafRemote;
470   const PetscInt    *leafLocal;
471   PetscInt           depth = 0, numLeaves, numLabels, l, cStart, cEnd, c, fStart, fEnd, f;
472   PetscMPIInt        rank;
473   PetscErrorCode     ierr;
474 
475   PetscFunctionBegin;
476   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
477   /* Step 10: Convert labels */
478   ierr = DMGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
479   for (l = 0; l < numLabels; ++l) {
480     DMLabel         label, newlabel;
481     const char     *lname;
482     PetscBool       isDepth;
483     IS              valueIS;
484     const PetscInt *values;
485     PetscInt        numValues, val;
486 
487     ierr = DMGetLabelName(dm, l, &lname);CHKERRQ(ierr);
488     ierr = PetscStrcmp(lname, "depth", &isDepth);CHKERRQ(ierr);
489     if (isDepth) continue;
490     ierr = DMCreateLabel(dmNew, lname);CHKERRQ(ierr);
491     ierr = DMGetLabel(dm, lname, &label);CHKERRQ(ierr);
492     ierr = DMGetLabel(dmNew, lname, &newlabel);CHKERRQ(ierr);
493     ierr = DMLabelGetDefaultValue(label,&val);CHKERRQ(ierr);
494     ierr = DMLabelSetDefaultValue(newlabel,val);CHKERRQ(ierr);
495     ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
496     ierr = ISGetLocalSize(valueIS, &numValues);CHKERRQ(ierr);
497     ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
498     for (val = 0; val < numValues; ++val) {
499       IS              pointIS;
500       const PetscInt *points;
501       PetscInt        numPoints, p;
502 
503       ierr = DMLabelGetStratumIS(label, values[val], &pointIS);CHKERRQ(ierr);
504       ierr = ISGetLocalSize(pointIS, &numPoints);CHKERRQ(ierr);
505       ierr = ISGetIndices(pointIS, &points);CHKERRQ(ierr);
506       for (p = 0; p < numPoints; ++p) {
507         const PetscInt newpoint = DMPlexShiftPoint_Internal(points[p], depth, depthShift);
508 
509         ierr = DMLabelSetValue(newlabel, newpoint, values[val]);CHKERRQ(ierr);
510       }
511       ierr = ISRestoreIndices(pointIS, &points);CHKERRQ(ierr);
512       ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
513     }
514     ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
515     ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
516   }
517   /* Step 11: Make label for output (vtk) and to mark ghost points (ghost) */
518   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
519   ierr = DMGetPointSF(dm, &sfPoint);CHKERRQ(ierr);
520   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
521   ierr = PetscSFGetGraph(sfPoint, NULL, &numLeaves, &leafLocal, &leafRemote);CHKERRQ(ierr);
522   ierr = DMCreateLabel(dmNew, "vtk");CHKERRQ(ierr);
523   ierr = DMCreateLabel(dmNew, "ghost");CHKERRQ(ierr);
524   ierr = DMGetLabel(dmNew, "vtk", &vtkLabel);CHKERRQ(ierr);
525   ierr = DMGetLabel(dmNew, "ghost", &ghostLabel);CHKERRQ(ierr);
526   for (l = 0, c = cStart; l < numLeaves && c < cEnd; ++l, ++c) {
527     for (; c < leafLocal[l] && c < cEnd; ++c) {
528       ierr = DMLabelSetValue(vtkLabel, c, 1);CHKERRQ(ierr);
529     }
530     if (leafLocal[l] >= cEnd) break;
531     if (leafRemote[l].rank == rank) {
532       ierr = DMLabelSetValue(vtkLabel, c, 1);CHKERRQ(ierr);
533     } else {
534       ierr = DMLabelSetValue(ghostLabel, c, 2);CHKERRQ(ierr);
535     }
536   }
537   for (; c < cEnd; ++c) {
538     ierr = DMLabelSetValue(vtkLabel, c, 1);CHKERRQ(ierr);
539   }
540   if (0) {
541     ierr = DMLabelView(vtkLabel, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
542   }
543   ierr = DMPlexGetHeightStratum(dmNew, 1, &fStart, &fEnd);CHKERRQ(ierr);
544   for (f = fStart; f < fEnd; ++f) {
545     PetscInt numCells;
546 
547     ierr = DMPlexGetSupportSize(dmNew, f, &numCells);CHKERRQ(ierr);
548     if (numCells < 2) {
549       ierr = DMLabelSetValue(ghostLabel, f, 1);CHKERRQ(ierr);
550     } else {
551       const PetscInt *cells = NULL;
552       PetscInt        vA, vB;
553 
554       ierr = DMPlexGetSupport(dmNew, f, &cells);CHKERRQ(ierr);
555       ierr = DMLabelGetValue(vtkLabel, cells[0], &vA);CHKERRQ(ierr);
556       ierr = DMLabelGetValue(vtkLabel, cells[1], &vB);CHKERRQ(ierr);
557       if (vA != 1 && vB != 1) {ierr = DMLabelSetValue(ghostLabel, f, 1);CHKERRQ(ierr);}
558     }
559   }
560   if (0) {
561     ierr = DMLabelView(ghostLabel, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
562   }
563   PetscFunctionReturn(0);
564 }
565 
566 static PetscErrorCode DMPlexShiftTree_Internal(DM dm, PetscInt depthShift[], DM dmNew)
567 {
568   DM             refTree;
569   PetscSection   pSec;
570   PetscInt       *parents, *childIDs;
571   PetscErrorCode ierr;
572 
573   PetscFunctionBegin;
574   ierr = DMPlexGetReferenceTree(dm,&refTree);CHKERRQ(ierr);
575   ierr = DMPlexSetReferenceTree(dmNew,refTree);CHKERRQ(ierr);
576   ierr = DMPlexGetTree(dm,&pSec,&parents,&childIDs,NULL,NULL);CHKERRQ(ierr);
577   if (pSec) {
578     PetscInt p, pStart, pEnd, *parentsShifted, pStartShifted, pEndShifted, depth;
579     PetscInt *childIDsShifted;
580     PetscSection pSecShifted;
581 
582     ierr = PetscSectionGetChart(pSec,&pStart,&pEnd);CHKERRQ(ierr);
583     ierr = DMPlexGetDepth(dm,&depth);CHKERRQ(ierr);
584     pStartShifted = DMPlexShiftPoint_Internal(pStart,depth,depthShift);
585     pEndShifted   = DMPlexShiftPoint_Internal(pEnd,depth,depthShift);
586     ierr = PetscMalloc2(pEndShifted - pStartShifted,&parentsShifted,pEndShifted-pStartShifted,&childIDsShifted);CHKERRQ(ierr);
587     ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dmNew),&pSecShifted);CHKERRQ(ierr);
588     ierr = PetscSectionSetChart(pSecShifted,pStartShifted,pEndShifted);CHKERRQ(ierr);
589     for (p = pStartShifted; p < pEndShifted; p++) {
590       /* start off assuming no children */
591       ierr = PetscSectionSetDof(pSecShifted,p,0);CHKERRQ(ierr);
592     }
593     for (p = pStart; p < pEnd; p++) {
594       PetscInt dof;
595       PetscInt pNew = DMPlexShiftPoint_Internal(p,depth,depthShift);
596 
597       ierr = PetscSectionGetDof(pSec,p,&dof);CHKERRQ(ierr);
598       ierr = PetscSectionSetDof(pSecShifted,pNew,dof);CHKERRQ(ierr);
599     }
600     ierr = PetscSectionSetUp(pSecShifted);CHKERRQ(ierr);
601     for (p = pStart; p < pEnd; p++) {
602       PetscInt dof;
603       PetscInt pNew = DMPlexShiftPoint_Internal(p,depth,depthShift);
604 
605       ierr = PetscSectionGetDof(pSec,p,&dof);CHKERRQ(ierr);
606       if (dof) {
607         PetscInt off, offNew;
608 
609         ierr = PetscSectionGetOffset(pSec,p,&off);CHKERRQ(ierr);
610         ierr = PetscSectionGetOffset(pSecShifted,pNew,&offNew);CHKERRQ(ierr);
611         parentsShifted[offNew] = DMPlexShiftPoint_Internal(parents[off],depth,depthShift);
612         childIDsShifted[offNew] = childIDs[off];
613       }
614     }
615     ierr = DMPlexSetTree(dmNew,pSecShifted,parentsShifted,childIDsShifted);CHKERRQ(ierr);
616     ierr = PetscFree2(parentsShifted,childIDsShifted);CHKERRQ(ierr);
617     ierr = PetscSectionDestroy(&pSecShifted);CHKERRQ(ierr);
618   }
619   PetscFunctionReturn(0);
620 }
621 
622 static PetscErrorCode DMPlexConstructGhostCells_Internal(DM dm, DMLabel label, PetscInt *numGhostCells, DM gdm)
623 {
624   PetscSF               sf;
625   IS                    valueIS;
626   const PetscInt       *values, *leaves;
627   PetscInt             *depthShift;
628   PetscInt              d, depth = 0, nleaves, loc, Ng, numFS, fs, fStart, fEnd, ghostCell, cEnd, c;
629   PetscBool             isper;
630   const PetscReal      *maxCell, *L;
631   const DMBoundaryType *bd;
632   PetscErrorCode        ierr;
633 
634   PetscFunctionBegin;
635   ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
636   ierr = PetscSFGetGraph(sf, NULL, &nleaves, &leaves, NULL);CHKERRQ(ierr);
637   nleaves = PetscMax(0, nleaves);
638   ierr = DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd);CHKERRQ(ierr);
639   /* Count ghost cells */
640   ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
641   ierr = ISGetLocalSize(valueIS, &numFS);CHKERRQ(ierr);
642   ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
643   Ng   = 0;
644   for (fs = 0; fs < numFS; ++fs) {
645     IS              faceIS;
646     const PetscInt *faces;
647     PetscInt        numFaces, f, numBdFaces = 0;
648 
649     ierr = DMLabelGetStratumIS(label, values[fs], &faceIS);CHKERRQ(ierr);
650     ierr = ISGetLocalSize(faceIS, &numFaces);CHKERRQ(ierr);
651     ierr = ISGetIndices(faceIS, &faces);CHKERRQ(ierr);
652     for (f = 0; f < numFaces; ++f) {
653       PetscInt numChildren;
654 
655       ierr = PetscFindInt(faces[f], nleaves, leaves, &loc);CHKERRQ(ierr);
656       ierr = DMPlexGetTreeChildren(dm,faces[f],&numChildren,NULL);CHKERRQ(ierr);
657       /* non-local and ancestors points don't get to register ghosts */
658       if (loc >= 0 || numChildren) continue;
659       if ((faces[f] >= fStart) && (faces[f] < fEnd)) ++numBdFaces;
660     }
661     Ng += numBdFaces;
662     ierr = ISDestroy(&faceIS);CHKERRQ(ierr);
663   }
664   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
665   ierr = PetscMalloc1(2*(depth+1), &depthShift);CHKERRQ(ierr);
666   for (d = 0; d <= depth; d++) {
667     PetscInt dEnd;
668 
669     ierr = DMPlexGetDepthStratum(dm,d,NULL,&dEnd);CHKERRQ(ierr);
670     depthShift[2*d]   = dEnd;
671     depthShift[2*d+1] = 0;
672   }
673   if (depth >= 0) depthShift[2*depth+1] = Ng;
674   ierr = DMPlexShiftPointSetUp_Internal(depth,depthShift);CHKERRQ(ierr);
675   ierr = DMPlexShiftSizes_Internal(dm, depthShift, gdm);CHKERRQ(ierr);
676   /* Step 3: Set cone/support sizes for new points */
677   ierr = DMPlexGetHeightStratum(dm, 0, NULL, &cEnd);CHKERRQ(ierr);
678   ierr = DMPlexSetHybridBounds(gdm, cEnd, PETSC_DETERMINE, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
679   for (c = cEnd; c < cEnd + Ng; ++c) {
680     ierr = DMPlexSetConeSize(gdm, c, 1);CHKERRQ(ierr);
681   }
682   for (fs = 0; fs < numFS; ++fs) {
683     IS              faceIS;
684     const PetscInt *faces;
685     PetscInt        numFaces, f;
686 
687     ierr = DMLabelGetStratumIS(label, values[fs], &faceIS);CHKERRQ(ierr);
688     ierr = ISGetLocalSize(faceIS, &numFaces);CHKERRQ(ierr);
689     ierr = ISGetIndices(faceIS, &faces);CHKERRQ(ierr);
690     for (f = 0; f < numFaces; ++f) {
691       PetscInt size, numChildren;
692 
693       ierr = PetscFindInt(faces[f], nleaves, leaves, &loc);CHKERRQ(ierr);
694       ierr = DMPlexGetTreeChildren(dm,faces[f],&numChildren,NULL);CHKERRQ(ierr);
695       if (loc >= 0 || numChildren) continue;
696       if ((faces[f] < fStart) || (faces[f] >= fEnd)) continue;
697       ierr = DMPlexGetSupportSize(dm, faces[f], &size);CHKERRQ(ierr);
698       if (size != 1) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM has boundary face %d with %d support cells", faces[f], size);
699       ierr = DMPlexSetSupportSize(gdm, faces[f] + Ng, 2);CHKERRQ(ierr);
700     }
701     ierr = ISRestoreIndices(faceIS, &faces);CHKERRQ(ierr);
702     ierr = ISDestroy(&faceIS);CHKERRQ(ierr);
703   }
704   /* Step 4: Setup ghosted DM */
705   ierr = DMSetUp(gdm);CHKERRQ(ierr);
706   ierr = DMPlexShiftPoints_Internal(dm, depthShift, gdm);CHKERRQ(ierr);
707   /* Step 6: Set cones and supports for new points */
708   ghostCell = cEnd;
709   for (fs = 0; fs < numFS; ++fs) {
710     IS              faceIS;
711     const PetscInt *faces;
712     PetscInt        numFaces, f;
713 
714     ierr = DMLabelGetStratumIS(label, values[fs], &faceIS);CHKERRQ(ierr);
715     ierr = ISGetLocalSize(faceIS, &numFaces);CHKERRQ(ierr);
716     ierr = ISGetIndices(faceIS, &faces);CHKERRQ(ierr);
717     for (f = 0; f < numFaces; ++f) {
718       PetscInt newFace = faces[f] + Ng, numChildren;
719 
720       ierr = PetscFindInt(faces[f], nleaves, leaves, &loc);CHKERRQ(ierr);
721       ierr = DMPlexGetTreeChildren(dm,faces[f],&numChildren,NULL);CHKERRQ(ierr);
722       if (loc >= 0 || numChildren) continue;
723       if ((faces[f] < fStart) || (faces[f] >= fEnd)) continue;
724       ierr = DMPlexSetCone(gdm, ghostCell, &newFace);CHKERRQ(ierr);
725       ierr = DMPlexInsertSupport(gdm, newFace, 1, ghostCell);CHKERRQ(ierr);
726       ++ghostCell;
727     }
728     ierr = ISRestoreIndices(faceIS, &faces);CHKERRQ(ierr);
729     ierr = ISDestroy(&faceIS);CHKERRQ(ierr);
730   }
731   ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
732   ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
733   /* Step 7: Stratify */
734   ierr = DMPlexStratify(gdm);CHKERRQ(ierr);
735   ierr = DMPlexShiftCoordinates_Internal(dm, depthShift, gdm);CHKERRQ(ierr);
736   ierr = DMPlexShiftSF_Internal(dm, depthShift, gdm);CHKERRQ(ierr);
737   ierr = DMPlexShiftLabels_Internal(dm, depthShift, gdm);CHKERRQ(ierr);
738   ierr = DMPlexShiftTree_Internal(dm, depthShift, gdm);CHKERRQ(ierr);
739   ierr = PetscFree(depthShift);CHKERRQ(ierr);
740   /* Step 7: Periodicity */
741   ierr = DMGetPeriodicity(dm, &isper, &maxCell, &L, &bd);CHKERRQ(ierr);
742   ierr = DMSetPeriodicity(gdm, isper, maxCell,  L,  bd);CHKERRQ(ierr);
743   if (numGhostCells) *numGhostCells = Ng;
744   PetscFunctionReturn(0);
745 }
746 
747 /*@C
748   DMPlexConstructGhostCells - Construct ghost cells which connect to every boundary face
749 
750   Collective on dm
751 
752   Input Parameters:
753 + dm - The original DM
754 - labelName - The label specifying the boundary faces, or "Face Sets" if this is NULL
755 
756   Output Parameters:
757 + numGhostCells - The number of ghost cells added to the DM
758 - dmGhosted - The new DM
759 
760   Note: If no label exists of that name, one will be created marking all boundary faces
761 
762   Level: developer
763 
764 .seealso: DMCreate()
765 @*/
766 PetscErrorCode DMPlexConstructGhostCells(DM dm, const char labelName[], PetscInt *numGhostCells, DM *dmGhosted)
767 {
768   DM             gdm;
769   DMLabel        label;
770   const char    *name = labelName ? labelName : "Face Sets";
771   PetscInt       dim;
772   PetscBool      flag;
773   PetscErrorCode ierr;
774 
775   PetscFunctionBegin;
776   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
777   if (numGhostCells) PetscValidPointer(numGhostCells, 3);
778   PetscValidPointer(dmGhosted, 4);
779   ierr = DMCreate(PetscObjectComm((PetscObject)dm), &gdm);CHKERRQ(ierr);
780   ierr = DMSetType(gdm, DMPLEX);CHKERRQ(ierr);
781   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
782   ierr = DMSetDimension(gdm, dim);CHKERRQ(ierr);
783   ierr = DMPlexGetAdjacencyUseCone(dm, &flag);CHKERRQ(ierr);
784   ierr = DMPlexSetAdjacencyUseCone(gdm, flag);CHKERRQ(ierr);
785   ierr = DMPlexGetAdjacencyUseClosure(dm, &flag);CHKERRQ(ierr);
786   ierr = DMPlexSetAdjacencyUseClosure(gdm, flag);CHKERRQ(ierr);
787   ierr = DMGetLabel(dm, name, &label);CHKERRQ(ierr);
788   if (!label) {
789     /* Get label for boundary faces */
790     ierr = DMCreateLabel(dm, name);CHKERRQ(ierr);
791     ierr = DMGetLabel(dm, name, &label);CHKERRQ(ierr);
792     ierr = DMPlexMarkBoundaryFaces(dm, label);CHKERRQ(ierr);
793   }
794   ierr = DMPlexConstructGhostCells_Internal(dm, label, numGhostCells, gdm);CHKERRQ(ierr);
795   ierr = DMCopyBoundary(dm, gdm);CHKERRQ(ierr);
796   *dmGhosted = gdm;
797   PetscFunctionReturn(0);
798 }
799 
800 /*
801   We are adding three kinds of points here:
802     Replicated:     Copies of points which exist in the mesh, such as vertices identified across a fault
803     Non-replicated: Points which exist on the fault, but are not replicated
804     Hybrid:         Entirely new points, such as cohesive cells
805 
806   When creating subsequent cohesive cells, we shift the old hybrid cells to the end of the numbering at
807   each depth so that the new split/hybrid points can be inserted as a block.
808 */
809 static PetscErrorCode DMPlexConstructCohesiveCells_Internal(DM dm, DMLabel label, DM sdm)
810 {
811   MPI_Comm         comm;
812   IS               valueIS;
813   PetscInt         numSP = 0;       /* The number of depths for which we have replicated points */
814   const PetscInt  *values;          /* List of depths for which we have replicated points */
815   IS              *splitIS;
816   IS              *unsplitIS;
817   PetscInt        *numSplitPoints;     /* The number of replicated points at each depth */
818   PetscInt        *numUnsplitPoints;   /* The number of non-replicated points at each depth which still give rise to hybrid points */
819   PetscInt        *numHybridPoints;    /* The number of new hybrid points at each depth */
820   PetscInt        *numHybridPointsOld; /* The number of existing hybrid points at each depth */
821   const PetscInt **splitPoints;        /* Replicated points for each depth */
822   const PetscInt **unsplitPoints;      /* Non-replicated points for each depth */
823   PetscSection     coordSection;
824   Vec              coordinates;
825   PetscScalar     *coords;
826   PetscInt        *depthMax;           /* The first hybrid point at each depth in the original mesh */
827   PetscInt        *depthEnd;           /* The point limit at each depth in the original mesh */
828   PetscInt        *depthShift;         /* Number of replicated+hybrid points at each depth */
829   PetscInt        *pMaxNew;            /* The first replicated point at each depth in the new mesh, hybrids come after this */
830   PetscInt        *coneNew, *coneONew, *supportNew;
831   PetscInt         shift = 100, shift2 = 200, depth = 0, dep, dim, d, sp, maxConeSize, maxSupportSize, maxConeSizeNew, maxSupportSizeNew, numLabels, vStart, vEnd, pEnd, p, v;
832   PetscErrorCode   ierr;
833 
834   PetscFunctionBegin;
835   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
836   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
837   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
838   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
839   /* Count split points and add cohesive cells */
840   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);CHKERRQ(ierr);
841   ierr = PetscMalloc5(depth+1,&depthMax,depth+1,&depthEnd,2*(depth+1),&depthShift,depth+1,&pMaxNew,depth+1,&numHybridPointsOld);CHKERRQ(ierr);
842   ierr = PetscMalloc7(depth+1,&splitIS,depth+1,&unsplitIS,depth+1,&numSplitPoints,depth+1,&numUnsplitPoints,depth+1,&numHybridPoints,depth+1,&splitPoints,depth+1,&unsplitPoints);CHKERRQ(ierr);
843   ierr = DMPlexGetHybridBounds(dm, depth >= 0 ? &depthMax[depth] : NULL, depth>1 ? &depthMax[depth-1] : NULL, depth>2 ? &depthMax[1] : NULL, depth >= 0 ? &depthMax[0] : NULL);CHKERRQ(ierr);
844   for (d = 0; d <= depth; ++d) {
845     ierr = DMPlexGetDepthStratum(dm, d, NULL, &pMaxNew[d]);CHKERRQ(ierr);
846     depthEnd[d]           = pMaxNew[d];
847     depthMax[d]           = depthMax[d] < 0 ? depthEnd[d] : depthMax[d];
848     numSplitPoints[d]     = 0;
849     numUnsplitPoints[d]   = 0;
850     numHybridPoints[d]    = 0;
851     numHybridPointsOld[d] = depthMax[d] < 0 ? 0 : depthEnd[d] - depthMax[d];
852     splitPoints[d]        = NULL;
853     unsplitPoints[d]      = NULL;
854     splitIS[d]            = NULL;
855     unsplitIS[d]          = NULL;
856     /* we are shifting the existing hybrid points with the stratum behind them, so
857      * the split comes at the end of the normal points, i.e., at depthMax[d] */
858     depthShift[2*d]       = depthMax[d];
859     depthShift[2*d+1]     = 0;
860   }
861   if (label) {
862     ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
863     ierr = ISGetLocalSize(valueIS, &numSP);CHKERRQ(ierr);
864     ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
865   }
866   for (sp = 0; sp < numSP; ++sp) {
867     const PetscInt dep = values[sp];
868 
869     if ((dep < 0) || (dep > depth)) continue;
870     ierr = DMLabelGetStratumIS(label, dep, &splitIS[dep]);CHKERRQ(ierr);
871     if (splitIS[dep]) {
872       ierr = ISGetLocalSize(splitIS[dep], &numSplitPoints[dep]);CHKERRQ(ierr);
873       ierr = ISGetIndices(splitIS[dep], &splitPoints[dep]);CHKERRQ(ierr);
874     }
875     ierr = DMLabelGetStratumIS(label, shift2+dep, &unsplitIS[dep]);CHKERRQ(ierr);
876     if (unsplitIS[dep]) {
877       ierr = ISGetLocalSize(unsplitIS[dep], &numUnsplitPoints[dep]);CHKERRQ(ierr);
878       ierr = ISGetIndices(unsplitIS[dep], &unsplitPoints[dep]);CHKERRQ(ierr);
879     }
880   }
881   /* Calculate number of hybrid points */
882   for (d = 1; d <= depth; ++d) numHybridPoints[d]     = numSplitPoints[d-1] + numUnsplitPoints[d-1]; /* There is a hybrid cell/face/edge for every split face/edge/vertex   */
883   for (d = 0; d <= depth; ++d) depthShift[2*d+1]      = numSplitPoints[d] + numHybridPoints[d];
884   ierr = DMPlexShiftPointSetUp_Internal(depth,depthShift);CHKERRQ(ierr);
885   /* the end of the points in this stratum that come before the new points:
886    * shifting pMaxNew[d] gets the new start of the next stratum, then count back the old hybrid points and the newly
887    * added points */
888   for (d = 0; d <= depth; ++d) pMaxNew[d]             = DMPlexShiftPoint_Internal(pMaxNew[d],depth,depthShift) - (numHybridPointsOld[d] + numSplitPoints[d] + numHybridPoints[d]);
889   ierr = DMPlexShiftSizes_Internal(dm, depthShift, sdm);CHKERRQ(ierr);
890   /* Step 3: Set cone/support sizes for new points */
891   for (dep = 0; dep <= depth; ++dep) {
892     for (p = 0; p < numSplitPoints[dep]; ++p) {
893       const PetscInt  oldp   = splitPoints[dep][p];
894       const PetscInt  newp   = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*oldp + depthOffset[dep]*/;
895       const PetscInt  splitp = p    + pMaxNew[dep];
896       const PetscInt *support;
897       PetscInt        coneSize, supportSize, qf, qn, qp, e;
898 
899       ierr = DMPlexGetConeSize(dm, oldp, &coneSize);CHKERRQ(ierr);
900       ierr = DMPlexSetConeSize(sdm, splitp, coneSize);CHKERRQ(ierr);
901       ierr = DMPlexGetSupportSize(dm, oldp, &supportSize);CHKERRQ(ierr);
902       ierr = DMPlexSetSupportSize(sdm, splitp, supportSize);CHKERRQ(ierr);
903       if (dep == depth-1) {
904         const PetscInt hybcell = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
905 
906         /* Add cohesive cells, they are prisms */
907         ierr = DMPlexSetConeSize(sdm, hybcell, 2 + coneSize);CHKERRQ(ierr);
908       } else if (dep == 0) {
909         const PetscInt hybedge = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
910 
911         ierr = DMPlexGetSupport(dm, oldp, &support);CHKERRQ(ierr);
912         for (e = 0, qn = 0, qp = 0, qf = 0; e < supportSize; ++e) {
913           PetscInt val;
914 
915           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
916           if (val == 1) ++qf;
917           if ((val == 1) || (val ==  (shift + 1))) ++qn;
918           if ((val == 1) || (val == -(shift + 1))) ++qp;
919         }
920         /* Split old vertex: Edges into original vertex and new cohesive edge */
921         ierr = DMPlexSetSupportSize(sdm, newp, qn+1);CHKERRQ(ierr);
922         /* Split new vertex: Edges into split vertex and new cohesive edge */
923         ierr = DMPlexSetSupportSize(sdm, splitp, qp+1);CHKERRQ(ierr);
924         /* Add hybrid edge */
925         ierr = DMPlexSetConeSize(sdm, hybedge, 2);CHKERRQ(ierr);
926         ierr = DMPlexSetSupportSize(sdm, hybedge, qf);CHKERRQ(ierr);
927       } else if (dep == dim-2) {
928         const PetscInt hybface = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
929 
930         ierr = DMPlexGetSupport(dm, oldp, &support);CHKERRQ(ierr);
931         for (e = 0, qn = 0, qp = 0, qf = 0; e < supportSize; ++e) {
932           PetscInt val;
933 
934           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
935           if (val == dim-1) ++qf;
936           if ((val == dim-1) || (val ==  (shift + dim-1))) ++qn;
937           if ((val == dim-1) || (val == -(shift + dim-1))) ++qp;
938         }
939         /* Split old edge: Faces into original edge and cohesive face (positive side?) */
940         ierr = DMPlexSetSupportSize(sdm, newp, qn+1);CHKERRQ(ierr);
941         /* Split new edge: Faces into split edge and cohesive face (negative side?) */
942         ierr = DMPlexSetSupportSize(sdm, splitp, qp+1);CHKERRQ(ierr);
943         /* Add hybrid face */
944         ierr = DMPlexSetConeSize(sdm, hybface, 4);CHKERRQ(ierr);
945         ierr = DMPlexSetSupportSize(sdm, hybface, qf);CHKERRQ(ierr);
946       }
947     }
948   }
949   for (dep = 0; dep <= depth; ++dep) {
950     for (p = 0; p < numUnsplitPoints[dep]; ++p) {
951       const PetscInt  oldp   = unsplitPoints[dep][p];
952       const PetscInt  newp   = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*oldp + depthOffset[dep]*/;
953       const PetscInt *support;
954       PetscInt        coneSize, supportSize, qf, e, s;
955 
956       ierr = DMPlexGetConeSize(dm, oldp, &coneSize);CHKERRQ(ierr);
957       ierr = DMPlexGetSupportSize(dm, oldp, &supportSize);CHKERRQ(ierr);
958       ierr = DMPlexGetSupport(dm, oldp, &support);CHKERRQ(ierr);
959       if (dep == 0) {
960         const PetscInt hybedge = p + pMaxNew[dep+1] + numSplitPoints[dep+1] + numSplitPoints[dep];
961 
962         /* Unsplit vertex: Edges into original vertex, split edges, and new cohesive edge twice */
963         for (s = 0, qf = 0; s < supportSize; ++s, ++qf) {
964           ierr = PetscFindInt(support[s], numSplitPoints[dep+1], splitPoints[dep+1], &e);CHKERRQ(ierr);
965           if (e >= 0) ++qf;
966         }
967         ierr = DMPlexSetSupportSize(sdm, newp, qf+2);CHKERRQ(ierr);
968         /* Add hybrid edge */
969         ierr = DMPlexSetConeSize(sdm, hybedge, 2);CHKERRQ(ierr);
970         for (e = 0, qf = 0; e < supportSize; ++e) {
971           PetscInt val;
972 
973           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
974           /* Split and unsplit edges produce hybrid faces */
975           if (val == 1) ++qf;
976           if (val == (shift2 + 1)) ++qf;
977         }
978         ierr = DMPlexSetSupportSize(sdm, hybedge, qf);CHKERRQ(ierr);
979       } else if (dep == dim-2) {
980         const PetscInt hybface = p + pMaxNew[dep+1] + numSplitPoints[dep+1] + numSplitPoints[dep];
981         PetscInt       val;
982 
983         for (e = 0, qf = 0; e < supportSize; ++e) {
984           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
985           if (val == dim-1) qf += 2;
986           else              ++qf;
987         }
988         /* Unsplit edge: Faces into original edge, split face, and cohesive face twice */
989         ierr = DMPlexSetSupportSize(sdm, newp, qf+2);CHKERRQ(ierr);
990         /* Add hybrid face */
991         for (e = 0, qf = 0; e < supportSize; ++e) {
992           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
993           if (val == dim-1) ++qf;
994         }
995         ierr = DMPlexSetConeSize(sdm, hybface, 4);CHKERRQ(ierr);
996         ierr = DMPlexSetSupportSize(sdm, hybface, qf);CHKERRQ(ierr);
997       }
998     }
999   }
1000   /* Step 4: Setup split DM */
1001   ierr = DMSetUp(sdm);CHKERRQ(ierr);
1002   ierr = DMPlexShiftPoints_Internal(dm, depthShift, sdm);CHKERRQ(ierr);
1003   ierr = DMPlexGetMaxSizes(sdm, &maxConeSizeNew, &maxSupportSizeNew);CHKERRQ(ierr);
1004   ierr = PetscMalloc3(PetscMax(maxConeSize, maxConeSizeNew)*3,&coneNew,PetscMax(maxConeSize, maxConeSizeNew)*3,&coneONew,PetscMax(maxSupportSize, maxSupportSizeNew),&supportNew);CHKERRQ(ierr);
1005   /* Step 6: Set cones and supports for new points */
1006   for (dep = 0; dep <= depth; ++dep) {
1007     for (p = 0; p < numSplitPoints[dep]; ++p) {
1008       const PetscInt  oldp   = splitPoints[dep][p];
1009       const PetscInt  newp   = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*oldp + depthOffset[dep]*/;
1010       const PetscInt  splitp = p    + pMaxNew[dep];
1011       const PetscInt *cone, *support, *ornt;
1012       PetscInt        coneSize, supportSize, q, qf, qn, qp, v, e, s;
1013 
1014       ierr = DMPlexGetConeSize(dm, oldp, &coneSize);CHKERRQ(ierr);
1015       ierr = DMPlexGetCone(dm, oldp, &cone);CHKERRQ(ierr);
1016       ierr = DMPlexGetConeOrientation(dm, oldp, &ornt);CHKERRQ(ierr);
1017       ierr = DMPlexGetSupportSize(dm, oldp, &supportSize);CHKERRQ(ierr);
1018       ierr = DMPlexGetSupport(dm, oldp, &support);CHKERRQ(ierr);
1019       if (dep == depth-1) {
1020         PetscBool       hasUnsplit = PETSC_FALSE;
1021         const PetscInt  hybcell    = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1022         const PetscInt *supportF;
1023 
1024         /* Split face:       copy in old face to new face to start */
1025         ierr = DMPlexGetSupport(sdm, newp,  &supportF);CHKERRQ(ierr);
1026         ierr = DMPlexSetSupport(sdm, splitp, supportF);CHKERRQ(ierr);
1027         /* Split old face:   old vertices/edges in cone so no change */
1028         /* Split new face:   new vertices/edges in cone */
1029         for (q = 0; q < coneSize; ++q) {
1030           ierr = PetscFindInt(cone[q], numSplitPoints[dep-1], splitPoints[dep-1], &v);CHKERRQ(ierr);
1031           if (v < 0) {
1032             ierr = PetscFindInt(cone[q], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &v);CHKERRQ(ierr);
1033             if (v < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %d in split or unsplit points of depth %d", cone[q], dep-1);
1034             coneNew[2+q] = DMPlexShiftPoint_Internal(cone[q], depth, depthShift) /*cone[q] + depthOffset[dep-1]*/;
1035             hasUnsplit   = PETSC_TRUE;
1036           } else {
1037             coneNew[2+q] = v + pMaxNew[dep-1];
1038             if (dep > 1) {
1039               const PetscInt *econe;
1040               PetscInt        econeSize, r, vs, vu;
1041 
1042               ierr = DMPlexGetConeSize(dm, cone[q], &econeSize);CHKERRQ(ierr);
1043               ierr = DMPlexGetCone(dm, cone[q], &econe);CHKERRQ(ierr);
1044               for (r = 0; r < econeSize; ++r) {
1045                 ierr = PetscFindInt(econe[r], numSplitPoints[dep-2],   splitPoints[dep-2],   &vs);CHKERRQ(ierr);
1046                 ierr = PetscFindInt(econe[r], numUnsplitPoints[dep-2], unsplitPoints[dep-2], &vu);CHKERRQ(ierr);
1047                 if (vs >= 0) continue;
1048                 if (vu < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %d in split or unsplit points of depth %d", econe[r], dep-2);
1049                 hasUnsplit   = PETSC_TRUE;
1050               }
1051             }
1052           }
1053         }
1054         ierr = DMPlexSetCone(sdm, splitp, &coneNew[2]);CHKERRQ(ierr);
1055         ierr = DMPlexSetConeOrientation(sdm, splitp, ornt);CHKERRQ(ierr);
1056         /* Face support */
1057         for (s = 0; s < supportSize; ++s) {
1058           PetscInt val;
1059 
1060           ierr = DMLabelGetValue(label, support[s], &val);CHKERRQ(ierr);
1061           if (val < 0) {
1062             /* Split old face:   Replace negative side cell with cohesive cell */
1063              ierr = DMPlexInsertSupport(sdm, newp, s, hybcell);CHKERRQ(ierr);
1064           } else {
1065             /* Split new face:   Replace positive side cell with cohesive cell */
1066             ierr = DMPlexInsertSupport(sdm, splitp, s, hybcell);CHKERRQ(ierr);
1067             /* Get orientation for cohesive face */
1068             {
1069               const PetscInt *ncone, *nconeO;
1070               PetscInt        nconeSize, nc;
1071 
1072               ierr = DMPlexGetConeSize(dm, support[s], &nconeSize);CHKERRQ(ierr);
1073               ierr = DMPlexGetCone(dm, support[s], &ncone);CHKERRQ(ierr);
1074               ierr = DMPlexGetConeOrientation(dm, support[s], &nconeO);CHKERRQ(ierr);
1075               for (nc = 0; nc < nconeSize; ++nc) {
1076                 if (ncone[nc] == oldp) {
1077                   coneONew[0] = nconeO[nc];
1078                   break;
1079                 }
1080               }
1081               if (nc >= nconeSize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate face %d in neighboring cell %d", oldp, support[s]);
1082             }
1083           }
1084         }
1085         /* Cohesive cell:    Old and new split face, then new cohesive faces */
1086         coneNew[0]  = newp;   /* Extracted negative side orientation above */
1087         coneNew[1]  = splitp;
1088         coneONew[1] = coneONew[0];
1089         for (q = 0; q < coneSize; ++q) {
1090           ierr = PetscFindInt(cone[q], numSplitPoints[dep-1], splitPoints[dep-1], &v);CHKERRQ(ierr);
1091           if (v < 0) {
1092             ierr = PetscFindInt(cone[q], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &v);CHKERRQ(ierr);
1093             coneNew[2+q]  = v + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep-1];
1094             coneONew[2+q] = 0;
1095           } else {
1096             coneNew[2+q]  = v + pMaxNew[dep] + numSplitPoints[dep];
1097           }
1098           coneONew[2+q] = 0;
1099         }
1100         ierr = DMPlexSetCone(sdm, hybcell, coneNew);CHKERRQ(ierr);
1101         ierr = DMPlexSetConeOrientation(sdm, hybcell, coneONew);CHKERRQ(ierr);
1102         /* Label the hybrid cells on the boundary of the split */
1103         if (hasUnsplit) {ierr = DMLabelSetValue(label, -hybcell, dim);CHKERRQ(ierr);}
1104       } else if (dep == 0) {
1105         const PetscInt hybedge = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1106 
1107         /* Split old vertex: Edges in old split faces and new cohesive edge */
1108         for (e = 0, qn = 0; e < supportSize; ++e) {
1109           PetscInt val;
1110 
1111           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
1112           if ((val == 1) || (val == (shift + 1))) {
1113             supportNew[qn++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1114           }
1115         }
1116         supportNew[qn] = hybedge;
1117         ierr = DMPlexSetSupport(sdm, newp, supportNew);CHKERRQ(ierr);
1118         /* Split new vertex: Edges in new split faces and new cohesive edge */
1119         for (e = 0, qp = 0; e < supportSize; ++e) {
1120           PetscInt val, edge;
1121 
1122           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
1123           if (val == 1) {
1124             ierr = PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &edge);CHKERRQ(ierr);
1125             if (edge < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Edge %d is not a split edge", support[e]);
1126             supportNew[qp++] = edge + pMaxNew[dep+1];
1127           } else if (val == -(shift + 1)) {
1128             supportNew[qp++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1129           }
1130         }
1131         supportNew[qp] = hybedge;
1132         ierr = DMPlexSetSupport(sdm, splitp, supportNew);CHKERRQ(ierr);
1133         /* Hybrid edge:    Old and new split vertex */
1134         coneNew[0] = newp;
1135         coneNew[1] = splitp;
1136         ierr = DMPlexSetCone(sdm, hybedge, coneNew);CHKERRQ(ierr);
1137         for (e = 0, qf = 0; e < supportSize; ++e) {
1138           PetscInt val, edge;
1139 
1140           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
1141           if (val == 1) {
1142             ierr = PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &edge);CHKERRQ(ierr);
1143             if (edge < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Edge %d is not a split edge", support[e]);
1144             supportNew[qf++] = edge + pMaxNew[dep+2] + numSplitPoints[dep+2];
1145           }
1146         }
1147         ierr = DMPlexSetSupport(sdm, hybedge, supportNew);CHKERRQ(ierr);
1148       } else if (dep == dim-2) {
1149         const PetscInt hybface = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1150 
1151         /* Split old edge:   old vertices in cone so no change */
1152         /* Split new edge:   new vertices in cone */
1153         for (q = 0; q < coneSize; ++q) {
1154           ierr = PetscFindInt(cone[q], numSplitPoints[dep-1], splitPoints[dep-1], &v);CHKERRQ(ierr);
1155           if (v < 0) {
1156             ierr = PetscFindInt(cone[q], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &v);CHKERRQ(ierr);
1157             if (v < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %d in split or unsplit points of depth %d", cone[q], dep-1);
1158             coneNew[q] = DMPlexShiftPoint_Internal(cone[q], depth, depthShift) /*cone[q] + depthOffset[dep-1]*/;
1159           } else {
1160             coneNew[q] = v + pMaxNew[dep-1];
1161           }
1162         }
1163         ierr = DMPlexSetCone(sdm, splitp, coneNew);CHKERRQ(ierr);
1164         /* Split old edge: Faces in positive side cells and old split faces */
1165         for (e = 0, q = 0; e < supportSize; ++e) {
1166           PetscInt val;
1167 
1168           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
1169           if (val == dim-1) {
1170             supportNew[q++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1171           } else if (val == (shift + dim-1)) {
1172             supportNew[q++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1173           }
1174         }
1175         supportNew[q++] = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1176         ierr = DMPlexSetSupport(sdm, newp, supportNew);CHKERRQ(ierr);
1177         /* Split new edge: Faces in negative side cells and new split faces */
1178         for (e = 0, q = 0; e < supportSize; ++e) {
1179           PetscInt val, face;
1180 
1181           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
1182           if (val == dim-1) {
1183             ierr = PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &face);CHKERRQ(ierr);
1184             if (face < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Face %d is not a split face", support[e]);
1185             supportNew[q++] = face + pMaxNew[dep+1];
1186           } else if (val == -(shift + dim-1)) {
1187             supportNew[q++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1188           }
1189         }
1190         supportNew[q++] = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1191         ierr = DMPlexSetSupport(sdm, splitp, supportNew);CHKERRQ(ierr);
1192         /* Hybrid face */
1193         coneNew[0] = newp;
1194         coneNew[1] = splitp;
1195         for (v = 0; v < coneSize; ++v) {
1196           PetscInt vertex;
1197           ierr = PetscFindInt(cone[v], numSplitPoints[dep-1], splitPoints[dep-1], &vertex);CHKERRQ(ierr);
1198           if (vertex < 0) {
1199             ierr = PetscFindInt(cone[v], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &vertex);CHKERRQ(ierr);
1200             if (vertex < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %d in split or unsplit points of depth %d", cone[v], dep-1);
1201             coneNew[2+v] = vertex + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep-1];
1202           } else {
1203             coneNew[2+v] = vertex + pMaxNew[dep] + numSplitPoints[dep];
1204           }
1205         }
1206         ierr = DMPlexSetCone(sdm, hybface, coneNew);CHKERRQ(ierr);
1207         for (e = 0, qf = 0; e < supportSize; ++e) {
1208           PetscInt val, face;
1209 
1210           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
1211           if (val == dim-1) {
1212             ierr = PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &face);CHKERRQ(ierr);
1213             if (face < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Face %d is not a split face", support[e]);
1214             supportNew[qf++] = face + pMaxNew[dep+2] + numSplitPoints[dep+2];
1215           }
1216         }
1217         ierr = DMPlexSetSupport(sdm, hybface, supportNew);CHKERRQ(ierr);
1218       }
1219     }
1220   }
1221   for (dep = 0; dep <= depth; ++dep) {
1222     for (p = 0; p < numUnsplitPoints[dep]; ++p) {
1223       const PetscInt  oldp   = unsplitPoints[dep][p];
1224       const PetscInt  newp   = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*oldp + depthOffset[dep]*/;
1225       const PetscInt *cone, *support, *ornt;
1226       PetscInt        coneSize, supportSize, supportSizeNew, q, qf, e, f, s;
1227 
1228       ierr = DMPlexGetConeSize(dm, oldp, &coneSize);CHKERRQ(ierr);
1229       ierr = DMPlexGetCone(dm, oldp, &cone);CHKERRQ(ierr);
1230       ierr = DMPlexGetConeOrientation(dm, oldp, &ornt);CHKERRQ(ierr);
1231       ierr = DMPlexGetSupportSize(dm, oldp, &supportSize);CHKERRQ(ierr);
1232       ierr = DMPlexGetSupport(dm, oldp, &support);CHKERRQ(ierr);
1233       if (dep == 0) {
1234         const PetscInt hybedge = p + pMaxNew[dep+1] + numSplitPoints[dep+1] + numSplitPoints[dep];
1235 
1236         /* Unsplit vertex */
1237         ierr = DMPlexGetSupportSize(sdm, newp, &supportSizeNew);CHKERRQ(ierr);
1238         for (s = 0, q = 0; s < supportSize; ++s) {
1239           supportNew[q++] = DMPlexShiftPoint_Internal(support[s], depth, depthShift) /*support[s] + depthOffset[dep+1]*/;
1240           ierr = PetscFindInt(support[s], numSplitPoints[dep+1], splitPoints[dep+1], &e);CHKERRQ(ierr);
1241           if (e >= 0) {
1242             supportNew[q++] = e + pMaxNew[dep+1];
1243           }
1244         }
1245         supportNew[q++] = hybedge;
1246         supportNew[q++] = hybedge;
1247         if (q != supportSizeNew) SETERRQ3(comm, PETSC_ERR_ARG_WRONG, "Support size %d != %d for vertex %d", q, supportSizeNew, newp);
1248         ierr = DMPlexSetSupport(sdm, newp, supportNew);CHKERRQ(ierr);
1249         /* Hybrid edge */
1250         coneNew[0] = newp;
1251         coneNew[1] = newp;
1252         ierr = DMPlexSetCone(sdm, hybedge, coneNew);CHKERRQ(ierr);
1253         for (e = 0, qf = 0; e < supportSize; ++e) {
1254           PetscInt val, edge;
1255 
1256           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
1257           if (val == 1) {
1258             ierr = PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &edge);CHKERRQ(ierr);
1259             if (edge < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Edge %d is not a split edge", support[e]);
1260             supportNew[qf++] = edge + pMaxNew[dep+2] + numSplitPoints[dep+2];
1261           } else if  (val ==  (shift2 + 1)) {
1262             ierr = PetscFindInt(support[e], numUnsplitPoints[dep+1], unsplitPoints[dep+1], &edge);CHKERRQ(ierr);
1263             if (edge < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Edge %d is not a unsplit edge", support[e]);
1264             supportNew[qf++] = edge + pMaxNew[dep+2] + numSplitPoints[dep+2] + numSplitPoints[dep+1];
1265           }
1266         }
1267         ierr = DMPlexSetSupport(sdm, hybedge, supportNew);CHKERRQ(ierr);
1268       } else if (dep == dim-2) {
1269         const PetscInt hybface = p + pMaxNew[dep+1] + numSplitPoints[dep+1] + numSplitPoints[dep];
1270 
1271         /* Unsplit edge: Faces into original edge, split face, and hybrid face twice */
1272         for (f = 0, qf = 0; f < supportSize; ++f) {
1273           PetscInt val, face;
1274 
1275           ierr = DMLabelGetValue(label, support[f], &val);CHKERRQ(ierr);
1276           if (val == dim-1) {
1277             ierr = PetscFindInt(support[f], numSplitPoints[dep+1], splitPoints[dep+1], &face);CHKERRQ(ierr);
1278             if (face < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Face %d is not a split face", support[f]);
1279             supportNew[qf++] = DMPlexShiftPoint_Internal(support[f], depth, depthShift) /*support[f] + depthOffset[dep+1]*/;
1280             supportNew[qf++] = face + pMaxNew[dep+1];
1281           } else {
1282             supportNew[qf++] = DMPlexShiftPoint_Internal(support[f], depth, depthShift) /*support[f] + depthOffset[dep+1]*/;
1283           }
1284         }
1285         supportNew[qf++] = hybface;
1286         supportNew[qf++] = hybface;
1287         ierr = DMPlexGetSupportSize(sdm, newp, &supportSizeNew);CHKERRQ(ierr);
1288         if (qf != supportSizeNew) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Support size for unsplit edge %d is %d != %d\n", newp, qf, supportSizeNew);
1289         ierr = DMPlexSetSupport(sdm, newp, supportNew);CHKERRQ(ierr);
1290         /* Add hybrid face */
1291         coneNew[0] = newp;
1292         coneNew[1] = newp;
1293         ierr = PetscFindInt(cone[0], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &v);CHKERRQ(ierr);
1294         if (v < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Vertex %d is not an unsplit vertex", cone[0]);
1295         coneNew[2] = v + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep-1];
1296         ierr = PetscFindInt(cone[1], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &v);CHKERRQ(ierr);
1297         if (v < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Vertex %d is not an unsplit vertex", cone[1]);
1298         coneNew[3] = v + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep-1];
1299         ierr = DMPlexSetCone(sdm, hybface, coneNew);CHKERRQ(ierr);
1300         for (f = 0, qf = 0; f < supportSize; ++f) {
1301           PetscInt val, face;
1302 
1303           ierr = DMLabelGetValue(label, support[f], &val);CHKERRQ(ierr);
1304           if (val == dim-1) {
1305             ierr = PetscFindInt(support[f], numSplitPoints[dep+1], splitPoints[dep+1], &face);CHKERRQ(ierr);
1306             supportNew[qf++] = face + pMaxNew[dep+2] + numSplitPoints[dep+2];
1307           }
1308         }
1309         ierr = DMPlexGetSupportSize(sdm, hybface, &supportSizeNew);CHKERRQ(ierr);
1310         if (qf != supportSizeNew) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Support size for hybrid face %d is %d != %d\n", hybface, qf, supportSizeNew);
1311         ierr = DMPlexSetSupport(sdm, hybface, supportNew);CHKERRQ(ierr);
1312       }
1313     }
1314   }
1315   /* Step 6b: Replace split points in negative side cones */
1316   for (sp = 0; sp < numSP; ++sp) {
1317     PetscInt        dep = values[sp];
1318     IS              pIS;
1319     PetscInt        numPoints;
1320     const PetscInt *points;
1321 
1322     if (dep >= 0) continue;
1323     ierr = DMLabelGetStratumIS(label, dep, &pIS);CHKERRQ(ierr);
1324     if (!pIS) continue;
1325     dep  = -dep - shift;
1326     ierr = ISGetLocalSize(pIS, &numPoints);CHKERRQ(ierr);
1327     ierr = ISGetIndices(pIS, &points);CHKERRQ(ierr);
1328     for (p = 0; p < numPoints; ++p) {
1329       const PetscInt  oldp = points[p];
1330       const PetscInt  newp = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*depthOffset[dep] + oldp*/;
1331       const PetscInt *cone;
1332       PetscInt        coneSize, c;
1333       /* PetscBool       replaced = PETSC_FALSE; */
1334 
1335       /* Negative edge: replace split vertex */
1336       /* Negative cell: replace split face */
1337       ierr = DMPlexGetConeSize(sdm, newp, &coneSize);CHKERRQ(ierr);
1338       ierr = DMPlexGetCone(sdm, newp, &cone);CHKERRQ(ierr);
1339       for (c = 0; c < coneSize; ++c) {
1340         const PetscInt coldp = DMPlexShiftPointInverse_Internal(cone[c],depth,depthShift);
1341         PetscInt       csplitp, cp, val;
1342 
1343         ierr = DMLabelGetValue(label, coldp, &val);CHKERRQ(ierr);
1344         if (val == dep-1) {
1345           ierr = PetscFindInt(coldp, numSplitPoints[dep-1], splitPoints[dep-1], &cp);CHKERRQ(ierr);
1346           if (cp < 0) SETERRQ2(comm, PETSC_ERR_ARG_WRONG, "Point %d is not a split point of dimension %d", oldp, dep-1);
1347           csplitp  = pMaxNew[dep-1] + cp;
1348           ierr     = DMPlexInsertCone(sdm, newp, c, csplitp);CHKERRQ(ierr);
1349           /* replaced = PETSC_TRUE; */
1350         }
1351       }
1352       /* Cells with only a vertex or edge on the submesh have no replacement */
1353       /* if (!replaced) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "The cone of point %d does not contain split points", oldp); */
1354     }
1355     ierr = ISRestoreIndices(pIS, &points);CHKERRQ(ierr);
1356     ierr = ISDestroy(&pIS);CHKERRQ(ierr);
1357   }
1358   /* Step 7: Stratify */
1359   ierr = DMPlexStratify(sdm);CHKERRQ(ierr);
1360   /* Step 8: Coordinates */
1361   ierr = DMPlexShiftCoordinates_Internal(dm, depthShift, sdm);CHKERRQ(ierr);
1362   ierr = DMGetCoordinateSection(sdm, &coordSection);CHKERRQ(ierr);
1363   ierr = DMGetCoordinatesLocal(sdm, &coordinates);CHKERRQ(ierr);
1364   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
1365   for (v = 0; v < (numSplitPoints ? numSplitPoints[0] : 0); ++v) {
1366     const PetscInt newp   = DMPlexShiftPoint_Internal(splitPoints[0][v], depth, depthShift) /*depthOffset[0] + splitPoints[0][v]*/;
1367     const PetscInt splitp = pMaxNew[0] + v;
1368     PetscInt       dof, off, soff, d;
1369 
1370     ierr = PetscSectionGetDof(coordSection, newp, &dof);CHKERRQ(ierr);
1371     ierr = PetscSectionGetOffset(coordSection, newp, &off);CHKERRQ(ierr);
1372     ierr = PetscSectionGetOffset(coordSection, splitp, &soff);CHKERRQ(ierr);
1373     for (d = 0; d < dof; ++d) coords[soff+d] = coords[off+d];
1374   }
1375   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
1376   /* Step 9: SF, if I can figure this out we can split the mesh in parallel */
1377   ierr = DMPlexShiftSF_Internal(dm, depthShift, sdm);CHKERRQ(ierr);
1378   /* Step 10: Labels */
1379   ierr = DMPlexShiftLabels_Internal(dm, depthShift, sdm);CHKERRQ(ierr);
1380   ierr = DMGetNumLabels(sdm, &numLabels);CHKERRQ(ierr);
1381   for (dep = 0; dep <= depth; ++dep) {
1382     for (p = 0; p < numSplitPoints[dep]; ++p) {
1383       const PetscInt newp   = DMPlexShiftPoint_Internal(splitPoints[dep][p], depth, depthShift) /*depthOffset[dep] + splitPoints[dep][p]*/;
1384       const PetscInt splitp = pMaxNew[dep] + p;
1385       PetscInt       l;
1386 
1387       for (l = 0; l < numLabels; ++l) {
1388         DMLabel     mlabel;
1389         const char *lname;
1390         PetscInt    val;
1391         PetscBool   isDepth;
1392 
1393         ierr = DMGetLabelName(sdm, l, &lname);CHKERRQ(ierr);
1394         ierr = PetscStrcmp(lname, "depth", &isDepth);CHKERRQ(ierr);
1395         if (isDepth) continue;
1396         ierr = DMGetLabel(sdm, lname, &mlabel);CHKERRQ(ierr);
1397         ierr = DMLabelGetValue(mlabel, newp, &val);CHKERRQ(ierr);
1398         if (val >= 0) {
1399           ierr = DMLabelSetValue(mlabel, splitp, val);CHKERRQ(ierr);
1400         }
1401       }
1402     }
1403   }
1404   for (sp = 0; sp < numSP; ++sp) {
1405     const PetscInt dep = values[sp];
1406 
1407     if ((dep < 0) || (dep > depth)) continue;
1408     if (splitIS[dep]) {ierr = ISRestoreIndices(splitIS[dep], &splitPoints[dep]);CHKERRQ(ierr);}
1409     ierr = ISDestroy(&splitIS[dep]);CHKERRQ(ierr);
1410     if (unsplitIS[dep]) {ierr = ISRestoreIndices(unsplitIS[dep], &unsplitPoints[dep]);CHKERRQ(ierr);}
1411     ierr = ISDestroy(&unsplitIS[dep]);CHKERRQ(ierr);
1412   }
1413   if (label) {
1414     ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
1415     ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
1416   }
1417   for (d = 0; d <= depth; ++d) {
1418     ierr = DMPlexGetDepthStratum(sdm, d, NULL, &pEnd);CHKERRQ(ierr);
1419     pMaxNew[d] = pEnd - numHybridPoints[d] - numHybridPointsOld[d];
1420   }
1421   ierr = DMPlexSetHybridBounds(sdm, depth >= 0 ? pMaxNew[depth] : PETSC_DETERMINE, depth>1 ? pMaxNew[depth-1] : PETSC_DETERMINE, depth>2 ? pMaxNew[1] : PETSC_DETERMINE, depth >= 0 ? pMaxNew[0] : PETSC_DETERMINE);CHKERRQ(ierr);
1422   ierr = PetscFree3(coneNew, coneONew, supportNew);CHKERRQ(ierr);
1423   ierr = PetscFree5(depthMax, depthEnd, depthShift, pMaxNew, numHybridPointsOld);CHKERRQ(ierr);
1424   ierr = PetscFree7(splitIS, unsplitIS, numSplitPoints, numUnsplitPoints, numHybridPoints, splitPoints, unsplitPoints);CHKERRQ(ierr);
1425   PetscFunctionReturn(0);
1426 }
1427 
1428 /*@C
1429   DMPlexConstructCohesiveCells - Construct cohesive cells which split the face along an internal interface
1430 
1431   Collective on dm
1432 
1433   Input Parameters:
1434 + dm - The original DM
1435 - label - The label specifying the boundary faces (this could be auto-generated)
1436 
1437   Output Parameters:
1438 - dmSplit - The new DM
1439 
1440   Level: developer
1441 
1442 .seealso: DMCreate(), DMPlexLabelCohesiveComplete()
1443 @*/
1444 PetscErrorCode DMPlexConstructCohesiveCells(DM dm, DMLabel label, DM *dmSplit)
1445 {
1446   DM             sdm;
1447   PetscInt       dim;
1448   PetscErrorCode ierr;
1449 
1450   PetscFunctionBegin;
1451   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1452   PetscValidPointer(dmSplit, 3);
1453   ierr = DMCreate(PetscObjectComm((PetscObject)dm), &sdm);CHKERRQ(ierr);
1454   ierr = DMSetType(sdm, DMPLEX);CHKERRQ(ierr);
1455   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
1456   ierr = DMSetDimension(sdm, dim);CHKERRQ(ierr);
1457   switch (dim) {
1458   case 2:
1459   case 3:
1460     ierr = DMPlexConstructCohesiveCells_Internal(dm, label, sdm);CHKERRQ(ierr);
1461     break;
1462   default:
1463     SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cannot construct cohesive cells for dimension %d", dim);
1464   }
1465   *dmSplit = sdm;
1466   PetscFunctionReturn(0);
1467 }
1468 
1469 /* Returns the side of the surface for a given cell with a face on the surface */
1470 static PetscErrorCode GetSurfaceSide_Static(DM dm, DM subdm, PetscInt numSubpoints, const PetscInt *subpoints, PetscInt cell, PetscInt face, PetscBool *pos)
1471 {
1472   const PetscInt *cone, *ornt;
1473   PetscInt        dim, coneSize, c;
1474   PetscErrorCode  ierr;
1475 
1476   PetscFunctionBegin;
1477   *pos = PETSC_TRUE;
1478   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
1479   ierr = DMPlexGetConeSize(dm, cell, &coneSize);CHKERRQ(ierr);
1480   ierr = DMPlexGetCone(dm, cell, &cone);CHKERRQ(ierr);
1481   ierr = DMPlexGetConeOrientation(dm, cell, &ornt);CHKERRQ(ierr);
1482   for (c = 0; c < coneSize; ++c) {
1483     if (cone[c] == face) {
1484       PetscInt o = ornt[c];
1485 
1486       if (subdm) {
1487         const PetscInt *subcone, *subornt;
1488         PetscInt        subpoint, subface, subconeSize, sc;
1489 
1490         ierr = PetscFindInt(cell, numSubpoints, subpoints, &subpoint);CHKERRQ(ierr);
1491         ierr = PetscFindInt(face, numSubpoints, subpoints, &subface);CHKERRQ(ierr);
1492         ierr = DMPlexGetConeSize(subdm, subpoint, &subconeSize);CHKERRQ(ierr);
1493         ierr = DMPlexGetCone(subdm, subpoint, &subcone);CHKERRQ(ierr);
1494         ierr = DMPlexGetConeOrientation(subdm, subpoint, &subornt);CHKERRQ(ierr);
1495         for (sc = 0; sc < subconeSize; ++sc) {
1496           if (subcone[sc] == subface) {
1497             o = subornt[0];
1498             break;
1499           }
1500         }
1501         if (sc >= subconeSize) SETERRQ4(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not find subpoint %d (%d) in cone for subpoint %d (%d)", subface, face, subpoint, cell);
1502       }
1503       if (o >= 0) *pos = PETSC_TRUE;
1504       else        *pos = PETSC_FALSE;
1505       break;
1506     }
1507   }
1508   if (c == coneSize) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Cell %d in split face %d support does not have it in the cone", cell, face);
1509   PetscFunctionReturn(0);
1510 }
1511 
1512 /*@
1513   DMPlexLabelCohesiveComplete - Starting with a label marking points on an internal surface, we add all other mesh pieces
1514   to complete the surface
1515 
1516   Input Parameters:
1517 + dm     - The DM
1518 . label  - A DMLabel marking the surface
1519 . blabel - A DMLabel marking the vertices on the boundary which will not be duplicated, or NULL to find them automatically
1520 . flip   - Flag to flip the submesh normal and replace points on the other side
1521 - subdm  - The subDM associated with the label, or NULL
1522 
1523   Output Parameter:
1524 . label - A DMLabel marking all surface points
1525 
1526   Note: The vertices in blabel are called "unsplit" in the terminology from hybrid cell creation.
1527 
1528   Level: developer
1529 
1530 .seealso: DMPlexConstructCohesiveCells(), DMPlexLabelComplete()
1531 @*/
1532 PetscErrorCode DMPlexLabelCohesiveComplete(DM dm, DMLabel label, DMLabel blabel, PetscBool flip, DM subdm)
1533 {
1534   DMLabel         depthLabel;
1535   IS              dimIS, subpointIS = NULL, facePosIS, faceNegIS, crossEdgeIS = NULL;
1536   const PetscInt *points, *subpoints;
1537   const PetscInt  rev   = flip ? -1 : 1;
1538   PetscInt       *pMax;
1539   PetscInt        shift = 100, shift2 = 200, dim, depth, pSize, dep, cStart, cEnd, cMax, fStart, fEnd, vStart, vEnd, numPoints, numSubpoints, p, val;
1540   PetscErrorCode  ierr;
1541 
1542   PetscFunctionBegin;
1543   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
1544   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
1545   pSize = PetscMax(depth, dim) + 1;
1546   ierr = PetscMalloc1(pSize,&pMax);CHKERRQ(ierr);
1547   ierr = DMPlexGetHybridBounds(dm, depth >= 0 ? &pMax[depth] : NULL, depth>1 ? &pMax[depth-1] : NULL, depth>2 ? &pMax[1] : NULL, &pMax[0]);CHKERRQ(ierr);
1548   ierr = DMPlexGetDepthLabel(dm, &depthLabel);CHKERRQ(ierr);
1549   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
1550   if (subdm) {
1551     ierr = DMPlexCreateSubpointIS(subdm, &subpointIS);CHKERRQ(ierr);
1552     if (subpointIS) {
1553       ierr = ISGetLocalSize(subpointIS, &numSubpoints);CHKERRQ(ierr);
1554       ierr = ISGetIndices(subpointIS, &subpoints);CHKERRQ(ierr);
1555     }
1556   }
1557   /* Mark cell on the fault, and its faces which touch the fault: cell orientation for face gives the side of the fault */
1558   ierr = DMLabelGetStratumIS(label, dim-1, &dimIS);CHKERRQ(ierr);
1559   if (!dimIS) {
1560     ierr = PetscFree(pMax);CHKERRQ(ierr);
1561     ierr = ISDestroy(&subpointIS);CHKERRQ(ierr);
1562     PetscFunctionReturn(0);
1563   }
1564   ierr = ISGetLocalSize(dimIS, &numPoints);CHKERRQ(ierr);
1565   ierr = ISGetIndices(dimIS, &points);CHKERRQ(ierr);
1566   for (p = 0; p < numPoints; ++p) { /* Loop over fault faces */
1567     const PetscInt *support;
1568     PetscInt        supportSize, s;
1569 
1570     ierr = DMPlexGetSupportSize(dm, points[p], &supportSize);CHKERRQ(ierr);
1571     if (supportSize != 2) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Split face %d has %d != 2 supports", points[p], supportSize);
1572     ierr = DMPlexGetSupport(dm, points[p], &support);CHKERRQ(ierr);
1573     for (s = 0; s < supportSize; ++s) {
1574       const PetscInt *cone;
1575       PetscInt        coneSize, c;
1576       PetscBool       pos;
1577 
1578       ierr = GetSurfaceSide_Static(dm, subdm, numSubpoints, subpoints, support[s], points[p], &pos);CHKERRQ(ierr);
1579       if (pos) {ierr = DMLabelSetValue(label, support[s],  rev*(shift+dim));CHKERRQ(ierr);}
1580       else     {ierr = DMLabelSetValue(label, support[s], -rev*(shift+dim));CHKERRQ(ierr);}
1581       if (rev < 0) pos = !pos ? PETSC_TRUE : PETSC_FALSE;
1582       /* Put faces touching the fault in the label */
1583       ierr = DMPlexGetConeSize(dm, support[s], &coneSize);CHKERRQ(ierr);
1584       ierr = DMPlexGetCone(dm, support[s], &cone);CHKERRQ(ierr);
1585       for (c = 0; c < coneSize; ++c) {
1586         const PetscInt point = cone[c];
1587 
1588         ierr = DMLabelGetValue(label, point, &val);CHKERRQ(ierr);
1589         if (val == -1) {
1590           PetscInt *closure = NULL;
1591           PetscInt  closureSize, cl;
1592 
1593           ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1594           for (cl = 0; cl < closureSize*2; cl += 2) {
1595             const PetscInt clp  = closure[cl];
1596             PetscInt       bval = -1;
1597 
1598             ierr = DMLabelGetValue(label, clp, &val);CHKERRQ(ierr);
1599             if (blabel) {ierr = DMLabelGetValue(blabel, clp, &bval);CHKERRQ(ierr);}
1600             if ((val >= 0) && (val < dim-1) && (bval < 0)) {
1601               ierr = DMLabelSetValue(label, point, pos == PETSC_TRUE ? shift+dim-1 : -(shift+dim-1));CHKERRQ(ierr);
1602               break;
1603             }
1604           }
1605           ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1606         }
1607       }
1608     }
1609   }
1610   ierr = ISRestoreIndices(dimIS, &points);CHKERRQ(ierr);
1611   ierr = ISDestroy(&dimIS);CHKERRQ(ierr);
1612   if (subpointIS) {ierr = ISRestoreIndices(subpointIS, &subpoints);CHKERRQ(ierr);}
1613   ierr = ISDestroy(&subpointIS);CHKERRQ(ierr);
1614   /* Mark boundary points as unsplit */
1615   if (blabel) {
1616     ierr = DMLabelGetStratumIS(blabel, 1, &dimIS);CHKERRQ(ierr);
1617     ierr = ISGetLocalSize(dimIS, &numPoints);CHKERRQ(ierr);
1618     ierr = ISGetIndices(dimIS, &points);CHKERRQ(ierr);
1619     for (p = 0; p < numPoints; ++p) {
1620       const PetscInt point = points[p];
1621       PetscInt       val, bval;
1622 
1623       ierr = DMLabelGetValue(blabel, point, &bval);CHKERRQ(ierr);
1624       if (bval >= 0) {
1625         ierr = DMLabelGetValue(label, point, &val);CHKERRQ(ierr);
1626         if ((val < 0) || (val > dim)) {
1627           /* This could be a point added from splitting a vertex on an adjacent fault, otherwise its just wrong */
1628           ierr = DMLabelClearValue(blabel, point, bval);CHKERRQ(ierr);
1629         }
1630       }
1631     }
1632     for (p = 0; p < numPoints; ++p) {
1633       const PetscInt point = points[p];
1634       PetscInt       val, bval;
1635 
1636       ierr = DMLabelGetValue(blabel, point, &bval);CHKERRQ(ierr);
1637       if (bval >= 0) {
1638         const PetscInt *cone,    *support;
1639         PetscInt        coneSize, supportSize, s, valA, valB, valE;
1640 
1641         /* Mark as unsplit */
1642         ierr = DMLabelGetValue(label, point, &val);CHKERRQ(ierr);
1643         if ((val < 0) || (val > dim)) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %d has label value %d, should be part of the fault", point, val);
1644         ierr = DMLabelClearValue(label, point, val);CHKERRQ(ierr);
1645         ierr = DMLabelSetValue(label, point, shift2+val);CHKERRQ(ierr);
1646         /* Check for cross-edge
1647              A cross-edge has endpoints which are both on the boundary of the surface, but the edge itself is not. */
1648         if (val != 0) continue;
1649         ierr = DMPlexGetSupport(dm, point, &support);CHKERRQ(ierr);
1650         ierr = DMPlexGetSupportSize(dm, point, &supportSize);CHKERRQ(ierr);
1651         for (s = 0; s < supportSize; ++s) {
1652           ierr = DMPlexGetCone(dm, support[s], &cone);CHKERRQ(ierr);
1653           ierr = DMPlexGetConeSize(dm, support[s], &coneSize);CHKERRQ(ierr);
1654           if (coneSize != 2) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Edge %D has %D vertices != 2", support[s], coneSize);
1655           ierr = DMLabelGetValue(blabel, cone[0], &valA);CHKERRQ(ierr);
1656           ierr = DMLabelGetValue(blabel, cone[1], &valB);CHKERRQ(ierr);
1657           ierr = DMLabelGetValue(blabel, support[s], &valE);CHKERRQ(ierr);
1658           if ((valE < 0) && (valA >= 0) && (valB >= 0) && (cone[0] != cone[1])) {ierr = DMLabelSetValue(blabel, support[s], 2);CHKERRQ(ierr);}
1659         }
1660       }
1661     }
1662     ierr = ISRestoreIndices(dimIS, &points);CHKERRQ(ierr);
1663     ierr = ISDestroy(&dimIS);CHKERRQ(ierr);
1664   }
1665   /* Search for other cells/faces/edges connected to the fault by a vertex */
1666   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
1667   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
1668   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
1669   cMax = cMax < 0 ? cEnd : cMax;
1670   ierr = DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd);CHKERRQ(ierr);
1671   ierr = DMLabelGetStratumIS(label, 0, &dimIS);CHKERRQ(ierr);
1672   if (blabel) {ierr = DMLabelGetStratumIS(blabel, 2, &crossEdgeIS);CHKERRQ(ierr);}
1673   if (dimIS && crossEdgeIS) {
1674     IS vertIS = dimIS;
1675 
1676     ierr = ISExpand(vertIS, crossEdgeIS, &dimIS);CHKERRQ(ierr);
1677     ierr = ISDestroy(&crossEdgeIS);CHKERRQ(ierr);
1678     ierr = ISDestroy(&vertIS);CHKERRQ(ierr);
1679   }
1680   if (!dimIS) {
1681     ierr = PetscFree(pMax);CHKERRQ(ierr);
1682     PetscFunctionReturn(0);
1683   }
1684   ierr = ISGetLocalSize(dimIS, &numPoints);CHKERRQ(ierr);
1685   ierr = ISGetIndices(dimIS, &points);CHKERRQ(ierr);
1686   for (p = 0; p < numPoints; ++p) { /* Loop over fault vertices */
1687     PetscInt *star = NULL;
1688     PetscInt  starSize, s;
1689     PetscInt  again = 1;  /* 0: Finished 1: Keep iterating after a change 2: No change */
1690 
1691     /* All points connected to the fault are inside a cell, so at the top level we will only check cells */
1692     ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_FALSE, &starSize, &star);CHKERRQ(ierr);
1693     while (again) {
1694       if (again > 1) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Could not classify all cells connected to the fault");
1695       again = 0;
1696       for (s = 0; s < starSize*2; s += 2) {
1697         const PetscInt  point = star[s];
1698         const PetscInt *cone;
1699         PetscInt        coneSize, c;
1700 
1701         if ((point < cStart) || (point >= cMax)) continue;
1702         ierr = DMLabelGetValue(label, point, &val);CHKERRQ(ierr);
1703         if (val != -1) continue;
1704         again = again == 1 ? 1 : 2;
1705         ierr  = DMPlexGetConeSize(dm, point, &coneSize);CHKERRQ(ierr);
1706         ierr  = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
1707         for (c = 0; c < coneSize; ++c) {
1708           ierr = DMLabelGetValue(label, cone[c], &val);CHKERRQ(ierr);
1709           if (val != -1) {
1710             const PetscInt *ccone;
1711             PetscInt        cconeSize, cc, side;
1712 
1713             if (PetscAbs(val) < shift) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Face %d on cell %d has an invalid label %d", cone[c], point, val);
1714             if (val > 0) side =  1;
1715             else         side = -1;
1716             ierr = DMLabelSetValue(label, point, side*(shift+dim));CHKERRQ(ierr);
1717             /* Mark cell faces which touch the fault */
1718             ierr = DMPlexGetConeSize(dm, point, &cconeSize);CHKERRQ(ierr);
1719             ierr = DMPlexGetCone(dm, point, &ccone);CHKERRQ(ierr);
1720             for (cc = 0; cc < cconeSize; ++cc) {
1721               PetscInt *closure = NULL;
1722               PetscInt  closureSize, cl;
1723 
1724               ierr = DMLabelGetValue(label, ccone[cc], &val);CHKERRQ(ierr);
1725               if (val != -1) continue;
1726               ierr = DMPlexGetTransitiveClosure(dm, ccone[cc], PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1727               for (cl = 0; cl < closureSize*2; cl += 2) {
1728                 const PetscInt clp = closure[cl];
1729 
1730                 ierr = DMLabelGetValue(label, clp, &val);CHKERRQ(ierr);
1731                 if (val == -1) continue;
1732                 ierr = DMLabelSetValue(label, ccone[cc], side*(shift+dim-1));CHKERRQ(ierr);
1733                 break;
1734               }
1735               ierr = DMPlexRestoreTransitiveClosure(dm, ccone[cc], PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1736             }
1737             again = 1;
1738             break;
1739           }
1740         }
1741       }
1742     }
1743     /* Classify the rest by cell membership */
1744     for (s = 0; s < starSize*2; s += 2) {
1745       const PetscInt point = star[s];
1746 
1747       ierr = DMLabelGetValue(label, point, &val);CHKERRQ(ierr);
1748       if (val == -1) {
1749         PetscInt  *sstar = NULL;
1750         PetscInt   sstarSize, ss;
1751         PetscBool  marked = PETSC_FALSE;
1752 
1753         ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_FALSE, &sstarSize, &sstar);CHKERRQ(ierr);
1754         for (ss = 0; ss < sstarSize*2; ss += 2) {
1755           const PetscInt spoint = sstar[ss];
1756 
1757           if ((spoint < cStart) || (spoint >= cMax)) continue;
1758           ierr = DMLabelGetValue(label, spoint, &val);CHKERRQ(ierr);
1759           if (val == -1) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Cell %d in star of %d does not have a valid label", spoint, point);
1760           ierr = DMLabelGetValue(depthLabel, point, &dep);CHKERRQ(ierr);
1761           if (val > 0) {
1762             ierr = DMLabelSetValue(label, point,   shift+dep);CHKERRQ(ierr);
1763           } else {
1764             ierr = DMLabelSetValue(label, point, -(shift+dep));CHKERRQ(ierr);
1765           }
1766           marked = PETSC_TRUE;
1767           break;
1768         }
1769         ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_FALSE, &sstarSize, &sstar);CHKERRQ(ierr);
1770         ierr = DMLabelGetValue(depthLabel, point, &dep);CHKERRQ(ierr);
1771         if (point < pMax[dep] && !marked) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d could not be classified", point);
1772       }
1773     }
1774     ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_FALSE, &starSize, &star);CHKERRQ(ierr);
1775   }
1776   ierr = ISRestoreIndices(dimIS, &points);CHKERRQ(ierr);
1777   ierr = ISDestroy(&dimIS);CHKERRQ(ierr);
1778   /* If any faces touching the fault divide cells on either side, split them */
1779   ierr = DMLabelGetStratumIS(label,   shift+dim-1,  &facePosIS);CHKERRQ(ierr);
1780   ierr = DMLabelGetStratumIS(label, -(shift+dim-1), &faceNegIS);CHKERRQ(ierr);
1781   ierr = ISExpand(facePosIS, faceNegIS, &dimIS);CHKERRQ(ierr);
1782   ierr = ISDestroy(&facePosIS);CHKERRQ(ierr);
1783   ierr = ISDestroy(&faceNegIS);CHKERRQ(ierr);
1784   ierr = ISGetLocalSize(dimIS, &numPoints);CHKERRQ(ierr);
1785   ierr = ISGetIndices(dimIS, &points);CHKERRQ(ierr);
1786   for (p = 0; p < numPoints; ++p) {
1787     const PetscInt  point = points[p];
1788     const PetscInt *support;
1789     PetscInt        supportSize, valA, valB;
1790 
1791     ierr = DMPlexGetSupportSize(dm, point, &supportSize);CHKERRQ(ierr);
1792     if (supportSize != 2) continue;
1793     ierr = DMPlexGetSupport(dm, point, &support);CHKERRQ(ierr);
1794     ierr = DMLabelGetValue(label, support[0], &valA);CHKERRQ(ierr);
1795     ierr = DMLabelGetValue(label, support[1], &valB);CHKERRQ(ierr);
1796     if ((valA == -1) || (valB == -1)) continue;
1797     if (valA*valB > 0) continue;
1798     /* Split the face */
1799     ierr = DMLabelGetValue(label, point, &valA);CHKERRQ(ierr);
1800     ierr = DMLabelClearValue(label, point, valA);CHKERRQ(ierr);
1801     ierr = DMLabelSetValue(label, point, dim-1);CHKERRQ(ierr);
1802     /* Label its closure:
1803       unmarked: label as unsplit
1804       incident: relabel as split
1805       split:    do nothing
1806     */
1807     {
1808       PetscInt *closure = NULL;
1809       PetscInt  closureSize, cl;
1810 
1811       ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1812       for (cl = 0; cl < closureSize*2; cl += 2) {
1813         ierr = DMLabelGetValue(label, closure[cl], &valA);CHKERRQ(ierr);
1814         if (valA == -1) { /* Mark as unsplit */
1815           ierr = DMLabelGetValue(depthLabel, closure[cl], &dep);CHKERRQ(ierr);
1816           ierr = DMLabelSetValue(label, closure[cl], shift2+dep);CHKERRQ(ierr);
1817         } else if (((valA >= shift) && (valA < shift2)) || ((valA <= -shift) && (valA > -shift2))) {
1818           ierr = DMLabelGetValue(depthLabel, closure[cl], &dep);CHKERRQ(ierr);
1819           ierr = DMLabelClearValue(label, closure[cl], valA);CHKERRQ(ierr);
1820           ierr = DMLabelSetValue(label, closure[cl], dep);CHKERRQ(ierr);
1821         }
1822       }
1823       ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1824     }
1825   }
1826   ierr = ISRestoreIndices(dimIS, &points);CHKERRQ(ierr);
1827   ierr = ISDestroy(&dimIS);CHKERRQ(ierr);
1828   ierr = PetscFree(pMax);CHKERRQ(ierr);
1829   PetscFunctionReturn(0);
1830 }
1831 
1832 /*@
1833   DMPlexCreateHybridMesh - Create a mesh with hybrid cells along an internal interface
1834 
1835   Collective on dm
1836 
1837   Input Parameters:
1838 + dm - The original DM
1839 - labelName - The label specifying the interface vertices
1840 
1841   Output Parameters:
1842 + hybridLabel - The label fully marking the interface
1843 - dmHybrid - The new DM
1844 
1845   Level: developer
1846 
1847 .seealso: DMPlexConstructCohesiveCells(), DMPlexLabelCohesiveComplete(), DMCreate()
1848 @*/
1849 PetscErrorCode DMPlexCreateHybridMesh(DM dm, DMLabel label, DMLabel *hybridLabel, DM *dmHybrid)
1850 {
1851   DM             idm;
1852   DMLabel        subpointMap, hlabel;
1853   PetscInt       dim;
1854   PetscErrorCode ierr;
1855 
1856   PetscFunctionBegin;
1857   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1858   if (hybridLabel) PetscValidPointer(hybridLabel, 3);
1859   PetscValidPointer(dmHybrid, 4);
1860   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
1861   ierr = DMPlexCreateSubmesh(dm, label, 1, &idm);CHKERRQ(ierr);
1862   ierr = DMPlexOrient(idm);CHKERRQ(ierr);
1863   ierr = DMPlexGetSubpointMap(idm, &subpointMap);CHKERRQ(ierr);
1864   ierr = DMLabelDuplicate(subpointMap, &hlabel);CHKERRQ(ierr);
1865   ierr = DMLabelClearStratum(hlabel, dim);CHKERRQ(ierr);
1866   ierr = DMPlexLabelCohesiveComplete(dm, hlabel, NULL, PETSC_FALSE, idm);CHKERRQ(ierr);
1867   ierr = DMDestroy(&idm);CHKERRQ(ierr);
1868   ierr = DMPlexConstructCohesiveCells(dm, hlabel, dmHybrid);CHKERRQ(ierr);
1869   if (hybridLabel) *hybridLabel = hlabel;
1870   else             {ierr = DMLabelDestroy(&hlabel);CHKERRQ(ierr);}
1871   PetscFunctionReturn(0);
1872 }
1873 
1874 /* Here we need the explicit assumption that:
1875 
1876      For any marked cell, the marked vertices constitute a single face
1877 */
1878 static PetscErrorCode DMPlexMarkSubmesh_Uninterpolated(DM dm, DMLabel vertexLabel, PetscInt value, DMLabel subpointMap, PetscInt *numFaces, PetscInt *nFV, DM subdm)
1879 {
1880   IS               subvertexIS = NULL;
1881   const PetscInt  *subvertices;
1882   PetscInt        *pStart, *pEnd, *pMax, pSize;
1883   PetscInt         depth, dim, d, numSubVerticesInitial = 0, v;
1884   PetscErrorCode   ierr;
1885 
1886   PetscFunctionBegin;
1887   *numFaces = 0;
1888   *nFV      = 0;
1889   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
1890   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
1891   pSize = PetscMax(depth, dim) + 1;
1892   ierr = PetscMalloc3(pSize,&pStart,pSize,&pEnd,pSize,&pMax);CHKERRQ(ierr);
1893   ierr = DMPlexGetHybridBounds(dm, depth >= 0 ? &pMax[depth] : NULL, depth>1 ? &pMax[depth-1] : NULL, depth>2 ? &pMax[1] : NULL, &pMax[0]);CHKERRQ(ierr);
1894   for (d = 0; d <= depth; ++d) {
1895     ierr = DMPlexGetDepthStratum(dm, d, &pStart[d], &pEnd[d]);CHKERRQ(ierr);
1896     if (pMax[d] >= 0) pEnd[d] = PetscMin(pEnd[d], pMax[d]);
1897   }
1898   /* Loop over initial vertices and mark all faces in the collective star() */
1899   if (vertexLabel) {ierr = DMLabelGetStratumIS(vertexLabel, value, &subvertexIS);CHKERRQ(ierr);}
1900   if (subvertexIS) {
1901     ierr = ISGetSize(subvertexIS, &numSubVerticesInitial);CHKERRQ(ierr);
1902     ierr = ISGetIndices(subvertexIS, &subvertices);CHKERRQ(ierr);
1903   }
1904   for (v = 0; v < numSubVerticesInitial; ++v) {
1905     const PetscInt vertex = subvertices[v];
1906     PetscInt      *star   = NULL;
1907     PetscInt       starSize, s, numCells = 0, c;
1908 
1909     ierr = DMPlexGetTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star);CHKERRQ(ierr);
1910     for (s = 0; s < starSize*2; s += 2) {
1911       const PetscInt point = star[s];
1912       if ((point >= pStart[depth]) && (point < pEnd[depth])) star[numCells++] = point;
1913     }
1914     for (c = 0; c < numCells; ++c) {
1915       const PetscInt cell    = star[c];
1916       PetscInt      *closure = NULL;
1917       PetscInt       closureSize, cl;
1918       PetscInt       cellLoc, numCorners = 0, faceSize = 0;
1919 
1920       ierr = DMLabelGetValue(subpointMap, cell, &cellLoc);CHKERRQ(ierr);
1921       if (cellLoc == 2) continue;
1922       if (cellLoc >= 0) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Cell %d has dimension %d in the surface label", cell, cellLoc);
1923       ierr = DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1924       for (cl = 0; cl < closureSize*2; cl += 2) {
1925         const PetscInt point = closure[cl];
1926         PetscInt       vertexLoc;
1927 
1928         if ((point >= pStart[0]) && (point < pEnd[0])) {
1929           ++numCorners;
1930           ierr = DMLabelGetValue(vertexLabel, point, &vertexLoc);CHKERRQ(ierr);
1931           if (vertexLoc == value) closure[faceSize++] = point;
1932         }
1933       }
1934       if (!(*nFV)) {ierr = DMPlexGetNumFaceVertices(dm, dim, numCorners, nFV);CHKERRQ(ierr);}
1935       if (faceSize > *nFV) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Invalid submesh: Too many vertices %d of an element on the surface", faceSize);
1936       if (faceSize == *nFV) {
1937         const PetscInt *cells = NULL;
1938         PetscInt        numCells, nc;
1939 
1940         ++(*numFaces);
1941         for (cl = 0; cl < faceSize; ++cl) {
1942           ierr = DMLabelSetValue(subpointMap, closure[cl], 0);CHKERRQ(ierr);
1943         }
1944         ierr = DMPlexGetJoin(dm, faceSize, closure, &numCells, &cells);CHKERRQ(ierr);
1945         for (nc = 0; nc < numCells; ++nc) {
1946           ierr = DMLabelSetValue(subpointMap, cells[nc], 2);CHKERRQ(ierr);
1947         }
1948         ierr = DMPlexRestoreJoin(dm, faceSize, closure, &numCells, &cells);CHKERRQ(ierr);
1949       }
1950       ierr = DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1951     }
1952     ierr = DMPlexRestoreTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star);CHKERRQ(ierr);
1953   }
1954   if (subvertexIS) {
1955     ierr = ISRestoreIndices(subvertexIS, &subvertices);CHKERRQ(ierr);
1956   }
1957   ierr = ISDestroy(&subvertexIS);CHKERRQ(ierr);
1958   ierr = PetscFree3(pStart,pEnd,pMax);CHKERRQ(ierr);
1959   PetscFunctionReturn(0);
1960 }
1961 
1962 static PetscErrorCode DMPlexMarkSubmesh_Interpolated(DM dm, DMLabel vertexLabel, PetscInt value, DMLabel subpointMap, DM subdm)
1963 {
1964   IS               subvertexIS = NULL;
1965   const PetscInt  *subvertices;
1966   PetscInt        *pStart, *pEnd, *pMax;
1967   PetscInt         dim, d, numSubVerticesInitial = 0, v;
1968   PetscErrorCode   ierr;
1969 
1970   PetscFunctionBegin;
1971   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
1972   ierr = PetscMalloc3(dim+1,&pStart,dim+1,&pEnd,dim+1,&pMax);CHKERRQ(ierr);
1973   ierr = DMPlexGetHybridBounds(dm, &pMax[dim], dim>1 ? &pMax[dim-1] : NULL, dim > 2 ? &pMax[1] : NULL, &pMax[0]);CHKERRQ(ierr);
1974   for (d = 0; d <= dim; ++d) {
1975     ierr = DMPlexGetDepthStratum(dm, d, &pStart[d], &pEnd[d]);CHKERRQ(ierr);
1976     if (pMax[d] >= 0) pEnd[d] = PetscMin(pEnd[d], pMax[d]);
1977   }
1978   /* Loop over initial vertices and mark all faces in the collective star() */
1979   if (vertexLabel) {
1980     ierr = DMLabelGetStratumIS(vertexLabel, value, &subvertexIS);CHKERRQ(ierr);
1981     if (subvertexIS) {
1982       ierr = ISGetSize(subvertexIS, &numSubVerticesInitial);CHKERRQ(ierr);
1983       ierr = ISGetIndices(subvertexIS, &subvertices);CHKERRQ(ierr);
1984     }
1985   }
1986   for (v = 0; v < numSubVerticesInitial; ++v) {
1987     const PetscInt vertex = subvertices[v];
1988     PetscInt      *star   = NULL;
1989     PetscInt       starSize, s, numFaces = 0, f;
1990 
1991     ierr = DMPlexGetTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star);CHKERRQ(ierr);
1992     for (s = 0; s < starSize*2; s += 2) {
1993       const PetscInt point = star[s];
1994       if ((point >= pStart[dim-1]) && (point < pEnd[dim-1])) star[numFaces++] = point;
1995     }
1996     for (f = 0; f < numFaces; ++f) {
1997       const PetscInt face    = star[f];
1998       PetscInt      *closure = NULL;
1999       PetscInt       closureSize, c;
2000       PetscInt       faceLoc;
2001 
2002       ierr = DMLabelGetValue(subpointMap, face, &faceLoc);CHKERRQ(ierr);
2003       if (faceLoc == dim-1) continue;
2004       if (faceLoc >= 0) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Face %d has dimension %d in the surface label", face, faceLoc);
2005       ierr = DMPlexGetTransitiveClosure(dm, face, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
2006       for (c = 0; c < closureSize*2; c += 2) {
2007         const PetscInt point = closure[c];
2008         PetscInt       vertexLoc;
2009 
2010         if ((point >= pStart[0]) && (point < pEnd[0])) {
2011           ierr = DMLabelGetValue(vertexLabel, point, &vertexLoc);CHKERRQ(ierr);
2012           if (vertexLoc != value) break;
2013         }
2014       }
2015       if (c == closureSize*2) {
2016         const PetscInt *support;
2017         PetscInt        supportSize, s;
2018 
2019         for (c = 0; c < closureSize*2; c += 2) {
2020           const PetscInt point = closure[c];
2021 
2022           for (d = 0; d < dim; ++d) {
2023             if ((point >= pStart[d]) && (point < pEnd[d])) {
2024               ierr = DMLabelSetValue(subpointMap, point, d);CHKERRQ(ierr);
2025               break;
2026             }
2027           }
2028         }
2029         ierr = DMPlexGetSupportSize(dm, face, &supportSize);CHKERRQ(ierr);
2030         ierr = DMPlexGetSupport(dm, face, &support);CHKERRQ(ierr);
2031         for (s = 0; s < supportSize; ++s) {
2032           ierr = DMLabelSetValue(subpointMap, support[s], dim);CHKERRQ(ierr);
2033         }
2034       }
2035       ierr = DMPlexRestoreTransitiveClosure(dm, face, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
2036     }
2037     ierr = DMPlexRestoreTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star);CHKERRQ(ierr);
2038   }
2039   if (subvertexIS) {ierr = ISRestoreIndices(subvertexIS, &subvertices);CHKERRQ(ierr);}
2040   ierr = ISDestroy(&subvertexIS);CHKERRQ(ierr);
2041   ierr = PetscFree3(pStart,pEnd,pMax);CHKERRQ(ierr);
2042   PetscFunctionReturn(0);
2043 }
2044 
2045 static PetscErrorCode DMPlexMarkCohesiveSubmesh_Uninterpolated(DM dm, PetscBool hasLagrange, const char labelname[], PetscInt value, DMLabel subpointMap, PetscInt *numFaces, PetscInt *nFV, PetscInt *subCells[], DM subdm)
2046 {
2047   DMLabel         label = NULL;
2048   const PetscInt *cone;
2049   PetscInt        dim, cMax, cEnd, c, subc = 0, p, coneSize = -1;
2050   PetscErrorCode  ierr;
2051 
2052   PetscFunctionBegin;
2053   *numFaces = 0;
2054   *nFV = 0;
2055   if (labelname) {ierr = DMGetLabel(dm, labelname, &label);CHKERRQ(ierr);}
2056   *subCells = NULL;
2057   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
2058   ierr = DMPlexGetHeightStratum(dm, 0, NULL, &cEnd);CHKERRQ(ierr);
2059   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
2060   if (cMax < 0) PetscFunctionReturn(0);
2061   if (label) {
2062     for (c = cMax; c < cEnd; ++c) {
2063       PetscInt val;
2064 
2065       ierr = DMLabelGetValue(label, c, &val);CHKERRQ(ierr);
2066       if (val == value) {
2067         ++(*numFaces);
2068         ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
2069       }
2070     }
2071   } else {
2072     *numFaces = cEnd - cMax;
2073     ierr = DMPlexGetConeSize(dm, cMax, &coneSize);CHKERRQ(ierr);
2074   }
2075   ierr = PetscMalloc1(*numFaces *2, subCells);CHKERRQ(ierr);
2076   if (!(*numFaces)) PetscFunctionReturn(0);
2077   *nFV = hasLagrange ? coneSize/3 : coneSize/2;
2078   for (c = cMax; c < cEnd; ++c) {
2079     const PetscInt *cells;
2080     PetscInt        numCells;
2081 
2082     if (label) {
2083       PetscInt val;
2084 
2085       ierr = DMLabelGetValue(label, c, &val);CHKERRQ(ierr);
2086       if (val != value) continue;
2087     }
2088     ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
2089     for (p = 0; p < *nFV; ++p) {
2090       ierr = DMLabelSetValue(subpointMap, cone[p], 0);CHKERRQ(ierr);
2091     }
2092     /* Negative face */
2093     ierr = DMPlexGetJoin(dm, *nFV, cone, &numCells, &cells);CHKERRQ(ierr);
2094     /* Not true in parallel
2095     if (numCells != 2) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive cells should separate two cells"); */
2096     for (p = 0; p < numCells; ++p) {
2097       ierr = DMLabelSetValue(subpointMap, cells[p], 2);CHKERRQ(ierr);
2098       (*subCells)[subc++] = cells[p];
2099     }
2100     ierr = DMPlexRestoreJoin(dm, *nFV, cone, &numCells, &cells);CHKERRQ(ierr);
2101     /* Positive face is not included */
2102   }
2103   PetscFunctionReturn(0);
2104 }
2105 
2106 static PetscErrorCode DMPlexMarkCohesiveSubmesh_Interpolated(DM dm, DMLabel label, PetscInt value, DMLabel subpointMap, DM subdm)
2107 {
2108   PetscInt      *pStart, *pEnd;
2109   PetscInt       dim, cMax, cEnd, c, d;
2110   PetscErrorCode ierr;
2111 
2112   PetscFunctionBegin;
2113   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
2114   ierr = DMPlexGetHeightStratum(dm, 0, NULL, &cEnd);CHKERRQ(ierr);
2115   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
2116   if (cMax < 0) PetscFunctionReturn(0);
2117   ierr = PetscMalloc2(dim+1,&pStart,dim+1,&pEnd);CHKERRQ(ierr);
2118   for (d = 0; d <= dim; ++d) {ierr = DMPlexGetDepthStratum(dm, d, &pStart[d], &pEnd[d]);CHKERRQ(ierr);}
2119   for (c = cMax; c < cEnd; ++c) {
2120     const PetscInt *cone;
2121     PetscInt       *closure = NULL;
2122     PetscInt        fconeSize, coneSize, closureSize, cl, val;
2123 
2124     if (label) {
2125       ierr = DMLabelGetValue(label, c, &val);CHKERRQ(ierr);
2126       if (val != value) continue;
2127     }
2128     ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
2129     ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
2130     ierr = DMPlexGetConeSize(dm, cone[0], &fconeSize);CHKERRQ(ierr);
2131     if (coneSize != (fconeSize ? fconeSize : 1) + 2) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive cells should separate two cells");
2132     /* Negative face */
2133     ierr = DMPlexGetTransitiveClosure(dm, cone[0], PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
2134     for (cl = 0; cl < closureSize*2; cl += 2) {
2135       const PetscInt point = closure[cl];
2136 
2137       for (d = 0; d <= dim; ++d) {
2138         if ((point >= pStart[d]) && (point < pEnd[d])) {
2139           ierr = DMLabelSetValue(subpointMap, point, d);CHKERRQ(ierr);
2140           break;
2141         }
2142       }
2143     }
2144     ierr = DMPlexRestoreTransitiveClosure(dm, cone[0], PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
2145     /* Cells -- positive face is not included */
2146     for (cl = 0; cl < 1; ++cl) {
2147       const PetscInt *support;
2148       PetscInt        supportSize, s;
2149 
2150       ierr = DMPlexGetSupportSize(dm, cone[cl], &supportSize);CHKERRQ(ierr);
2151       /* if (supportSize != 2) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive faces should separate two cells"); */
2152       ierr = DMPlexGetSupport(dm, cone[cl], &support);CHKERRQ(ierr);
2153       for (s = 0; s < supportSize; ++s) {
2154         ierr = DMLabelSetValue(subpointMap, support[s], dim);CHKERRQ(ierr);
2155       }
2156     }
2157   }
2158   ierr = PetscFree2(pStart, pEnd);CHKERRQ(ierr);
2159   PetscFunctionReturn(0);
2160 }
2161 
2162 static PetscErrorCode DMPlexGetFaceOrientation(DM dm, PetscInt cell, PetscInt numCorners, PetscInt indices[], PetscInt oppositeVertex, PetscInt origVertices[], PetscInt faceVertices[], PetscBool *posOriented)
2163 {
2164   MPI_Comm       comm;
2165   PetscBool      posOrient = PETSC_FALSE;
2166   const PetscInt debug     = 0;
2167   PetscInt       cellDim, faceSize, f;
2168   PetscErrorCode ierr;
2169 
2170   PetscFunctionBegin;
2171   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
2172   ierr = DMGetDimension(dm, &cellDim);CHKERRQ(ierr);
2173   if (debug) {PetscPrintf(comm, "cellDim: %d numCorners: %d\n", cellDim, numCorners);CHKERRQ(ierr);}
2174 
2175   if (cellDim == 1 && numCorners == 2) {
2176     /* Triangle */
2177     faceSize  = numCorners-1;
2178     posOrient = !(oppositeVertex%2) ? PETSC_TRUE : PETSC_FALSE;
2179   } else if (cellDim == 2 && numCorners == 3) {
2180     /* Triangle */
2181     faceSize  = numCorners-1;
2182     posOrient = !(oppositeVertex%2) ? PETSC_TRUE : PETSC_FALSE;
2183   } else if (cellDim == 3 && numCorners == 4) {
2184     /* Tetrahedron */
2185     faceSize  = numCorners-1;
2186     posOrient = (oppositeVertex%2) ? PETSC_TRUE : PETSC_FALSE;
2187   } else if (cellDim == 1 && numCorners == 3) {
2188     /* Quadratic line */
2189     faceSize  = 1;
2190     posOrient = PETSC_TRUE;
2191   } else if (cellDim == 2 && numCorners == 4) {
2192     /* Quads */
2193     faceSize = 2;
2194     if ((indices[1] > indices[0]) && (indices[1] - indices[0] == 1)) {
2195       posOrient = PETSC_TRUE;
2196     } else if ((indices[0] == 3) && (indices[1] == 0)) {
2197       posOrient = PETSC_TRUE;
2198     } else {
2199       if (((indices[0] > indices[1]) && (indices[0] - indices[1] == 1)) || ((indices[0] == 0) && (indices[1] == 3))) {
2200         posOrient = PETSC_FALSE;
2201       } else SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid quad crossedge");
2202     }
2203   } else if (cellDim == 2 && numCorners == 6) {
2204     /* Quadratic triangle (I hate this) */
2205     /* Edges are determined by the first 2 vertices (corners of edges) */
2206     const PetscInt faceSizeTri = 3;
2207     PetscInt       sortedIndices[3], i, iFace;
2208     PetscBool      found                    = PETSC_FALSE;
2209     PetscInt       faceVerticesTriSorted[9] = {
2210       0, 3,  4, /* bottom */
2211       1, 4,  5, /* right */
2212       2, 3,  5, /* left */
2213     };
2214     PetscInt       faceVerticesTri[9] = {
2215       0, 3,  4, /* bottom */
2216       1, 4,  5, /* right */
2217       2, 5,  3, /* left */
2218     };
2219 
2220     for (i = 0; i < faceSizeTri; ++i) sortedIndices[i] = indices[i];
2221     ierr = PetscSortInt(faceSizeTri, sortedIndices);CHKERRQ(ierr);
2222     for (iFace = 0; iFace < 3; ++iFace) {
2223       const PetscInt ii = iFace*faceSizeTri;
2224       PetscInt       fVertex, cVertex;
2225 
2226       if ((sortedIndices[0] == faceVerticesTriSorted[ii+0]) &&
2227           (sortedIndices[1] == faceVerticesTriSorted[ii+1])) {
2228         for (fVertex = 0; fVertex < faceSizeTri; ++fVertex) {
2229           for (cVertex = 0; cVertex < faceSizeTri; ++cVertex) {
2230             if (indices[cVertex] == faceVerticesTri[ii+fVertex]) {
2231               faceVertices[fVertex] = origVertices[cVertex];
2232               break;
2233             }
2234           }
2235         }
2236         found = PETSC_TRUE;
2237         break;
2238       }
2239     }
2240     if (!found) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid tri crossface");
2241     if (posOriented) *posOriented = PETSC_TRUE;
2242     PetscFunctionReturn(0);
2243   } else if (cellDim == 2 && numCorners == 9) {
2244     /* Quadratic quad (I hate this) */
2245     /* Edges are determined by the first 2 vertices (corners of edges) */
2246     const PetscInt faceSizeQuad = 3;
2247     PetscInt       sortedIndices[3], i, iFace;
2248     PetscBool      found                      = PETSC_FALSE;
2249     PetscInt       faceVerticesQuadSorted[12] = {
2250       0, 1,  4, /* bottom */
2251       1, 2,  5, /* right */
2252       2, 3,  6, /* top */
2253       0, 3,  7, /* left */
2254     };
2255     PetscInt       faceVerticesQuad[12] = {
2256       0, 1,  4, /* bottom */
2257       1, 2,  5, /* right */
2258       2, 3,  6, /* top */
2259       3, 0,  7, /* left */
2260     };
2261 
2262     for (i = 0; i < faceSizeQuad; ++i) sortedIndices[i] = indices[i];
2263     ierr = PetscSortInt(faceSizeQuad, sortedIndices);CHKERRQ(ierr);
2264     for (iFace = 0; iFace < 4; ++iFace) {
2265       const PetscInt ii = iFace*faceSizeQuad;
2266       PetscInt       fVertex, cVertex;
2267 
2268       if ((sortedIndices[0] == faceVerticesQuadSorted[ii+0]) &&
2269           (sortedIndices[1] == faceVerticesQuadSorted[ii+1])) {
2270         for (fVertex = 0; fVertex < faceSizeQuad; ++fVertex) {
2271           for (cVertex = 0; cVertex < faceSizeQuad; ++cVertex) {
2272             if (indices[cVertex] == faceVerticesQuad[ii+fVertex]) {
2273               faceVertices[fVertex] = origVertices[cVertex];
2274               break;
2275             }
2276           }
2277         }
2278         found = PETSC_TRUE;
2279         break;
2280       }
2281     }
2282     if (!found) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid quad crossface");
2283     if (posOriented) *posOriented = PETSC_TRUE;
2284     PetscFunctionReturn(0);
2285   } else if (cellDim == 3 && numCorners == 8) {
2286     /* Hexes
2287        A hex is two oriented quads with the normal of the first
2288        pointing up at the second.
2289 
2290           7---6
2291          /|  /|
2292         4---5 |
2293         | 1-|-2
2294         |/  |/
2295         0---3
2296 
2297         Faces are determined by the first 4 vertices (corners of faces) */
2298     const PetscInt faceSizeHex = 4;
2299     PetscInt       sortedIndices[4], i, iFace;
2300     PetscBool      found                     = PETSC_FALSE;
2301     PetscInt       faceVerticesHexSorted[24] = {
2302       0, 1, 2, 3,  /* bottom */
2303       4, 5, 6, 7,  /* top */
2304       0, 3, 4, 5,  /* front */
2305       2, 3, 5, 6,  /* right */
2306       1, 2, 6, 7,  /* back */
2307       0, 1, 4, 7,  /* left */
2308     };
2309     PetscInt       faceVerticesHex[24] = {
2310       1, 2, 3, 0,  /* bottom */
2311       4, 5, 6, 7,  /* top */
2312       0, 3, 5, 4,  /* front */
2313       3, 2, 6, 5,  /* right */
2314       2, 1, 7, 6,  /* back */
2315       1, 0, 4, 7,  /* left */
2316     };
2317 
2318     for (i = 0; i < faceSizeHex; ++i) sortedIndices[i] = indices[i];
2319     ierr = PetscSortInt(faceSizeHex, sortedIndices);CHKERRQ(ierr);
2320     for (iFace = 0; iFace < 6; ++iFace) {
2321       const PetscInt ii = iFace*faceSizeHex;
2322       PetscInt       fVertex, cVertex;
2323 
2324       if ((sortedIndices[0] == faceVerticesHexSorted[ii+0]) &&
2325           (sortedIndices[1] == faceVerticesHexSorted[ii+1]) &&
2326           (sortedIndices[2] == faceVerticesHexSorted[ii+2]) &&
2327           (sortedIndices[3] == faceVerticesHexSorted[ii+3])) {
2328         for (fVertex = 0; fVertex < faceSizeHex; ++fVertex) {
2329           for (cVertex = 0; cVertex < faceSizeHex; ++cVertex) {
2330             if (indices[cVertex] == faceVerticesHex[ii+fVertex]) {
2331               faceVertices[fVertex] = origVertices[cVertex];
2332               break;
2333             }
2334           }
2335         }
2336         found = PETSC_TRUE;
2337         break;
2338       }
2339     }
2340     if (!found) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid hex crossface");
2341     if (posOriented) *posOriented = PETSC_TRUE;
2342     PetscFunctionReturn(0);
2343   } else if (cellDim == 3 && numCorners == 10) {
2344     /* Quadratic tet */
2345     /* Faces are determined by the first 3 vertices (corners of faces) */
2346     const PetscInt faceSizeTet = 6;
2347     PetscInt       sortedIndices[6], i, iFace;
2348     PetscBool      found                     = PETSC_FALSE;
2349     PetscInt       faceVerticesTetSorted[24] = {
2350       0, 1, 2,  6, 7, 8, /* bottom */
2351       0, 3, 4,  6, 7, 9,  /* front */
2352       1, 4, 5,  7, 8, 9,  /* right */
2353       2, 3, 5,  6, 8, 9,  /* left */
2354     };
2355     PetscInt       faceVerticesTet[24] = {
2356       0, 1, 2,  6, 7, 8, /* bottom */
2357       0, 4, 3,  6, 7, 9,  /* front */
2358       1, 5, 4,  7, 8, 9,  /* right */
2359       2, 3, 5,  8, 6, 9,  /* left */
2360     };
2361 
2362     for (i = 0; i < faceSizeTet; ++i) sortedIndices[i] = indices[i];
2363     ierr = PetscSortInt(faceSizeTet, sortedIndices);CHKERRQ(ierr);
2364     for (iFace=0; iFace < 4; ++iFace) {
2365       const PetscInt ii = iFace*faceSizeTet;
2366       PetscInt       fVertex, cVertex;
2367 
2368       if ((sortedIndices[0] == faceVerticesTetSorted[ii+0]) &&
2369           (sortedIndices[1] == faceVerticesTetSorted[ii+1]) &&
2370           (sortedIndices[2] == faceVerticesTetSorted[ii+2]) &&
2371           (sortedIndices[3] == faceVerticesTetSorted[ii+3])) {
2372         for (fVertex = 0; fVertex < faceSizeTet; ++fVertex) {
2373           for (cVertex = 0; cVertex < faceSizeTet; ++cVertex) {
2374             if (indices[cVertex] == faceVerticesTet[ii+fVertex]) {
2375               faceVertices[fVertex] = origVertices[cVertex];
2376               break;
2377             }
2378           }
2379         }
2380         found = PETSC_TRUE;
2381         break;
2382       }
2383     }
2384     if (!found) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid tet crossface");
2385     if (posOriented) *posOriented = PETSC_TRUE;
2386     PetscFunctionReturn(0);
2387   } else if (cellDim == 3 && numCorners == 27) {
2388     /* Quadratic hexes (I hate this)
2389        A hex is two oriented quads with the normal of the first
2390        pointing up at the second.
2391 
2392          7---6
2393         /|  /|
2394        4---5 |
2395        | 3-|-2
2396        |/  |/
2397        0---1
2398 
2399        Faces are determined by the first 4 vertices (corners of faces) */
2400     const PetscInt faceSizeQuadHex = 9;
2401     PetscInt       sortedIndices[9], i, iFace;
2402     PetscBool      found                         = PETSC_FALSE;
2403     PetscInt       faceVerticesQuadHexSorted[54] = {
2404       0, 1, 2, 3,  8, 9, 10, 11,  24, /* bottom */
2405       4, 5, 6, 7,  12, 13, 14, 15,  25, /* top */
2406       0, 1, 4, 5,  8, 12, 16, 17,  22, /* front */
2407       1, 2, 5, 6,  9, 13, 17, 18,  21, /* right */
2408       2, 3, 6, 7,  10, 14, 18, 19,  23, /* back */
2409       0, 3, 4, 7,  11, 15, 16, 19,  20, /* left */
2410     };
2411     PetscInt       faceVerticesQuadHex[54] = {
2412       3, 2, 1, 0,  10, 9, 8, 11,  24, /* bottom */
2413       4, 5, 6, 7,  12, 13, 14, 15,  25, /* top */
2414       0, 1, 5, 4,  8, 17, 12, 16,  22, /* front */
2415       1, 2, 6, 5,  9, 18, 13, 17,  21, /* right */
2416       2, 3, 7, 6,  10, 19, 14, 18,  23, /* back */
2417       3, 0, 4, 7,  11, 16, 15, 19,  20 /* left */
2418     };
2419 
2420     for (i = 0; i < faceSizeQuadHex; ++i) sortedIndices[i] = indices[i];
2421     ierr = PetscSortInt(faceSizeQuadHex, sortedIndices);CHKERRQ(ierr);
2422     for (iFace = 0; iFace < 6; ++iFace) {
2423       const PetscInt ii = iFace*faceSizeQuadHex;
2424       PetscInt       fVertex, cVertex;
2425 
2426       if ((sortedIndices[0] == faceVerticesQuadHexSorted[ii+0]) &&
2427           (sortedIndices[1] == faceVerticesQuadHexSorted[ii+1]) &&
2428           (sortedIndices[2] == faceVerticesQuadHexSorted[ii+2]) &&
2429           (sortedIndices[3] == faceVerticesQuadHexSorted[ii+3])) {
2430         for (fVertex = 0; fVertex < faceSizeQuadHex; ++fVertex) {
2431           for (cVertex = 0; cVertex < faceSizeQuadHex; ++cVertex) {
2432             if (indices[cVertex] == faceVerticesQuadHex[ii+fVertex]) {
2433               faceVertices[fVertex] = origVertices[cVertex];
2434               break;
2435             }
2436           }
2437         }
2438         found = PETSC_TRUE;
2439         break;
2440       }
2441     }
2442     if (!found) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid hex crossface");
2443     if (posOriented) *posOriented = PETSC_TRUE;
2444     PetscFunctionReturn(0);
2445   } else SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Unknown cell type for faceOrientation().");
2446   if (!posOrient) {
2447     if (debug) {ierr = PetscPrintf(comm, "  Reversing initial face orientation\n");CHKERRQ(ierr);}
2448     for (f = 0; f < faceSize; ++f) faceVertices[f] = origVertices[faceSize-1 - f];
2449   } else {
2450     if (debug) {ierr = PetscPrintf(comm, "  Keeping initial face orientation\n");CHKERRQ(ierr);}
2451     for (f = 0; f < faceSize; ++f) faceVertices[f] = origVertices[f];
2452   }
2453   if (posOriented) *posOriented = posOrient;
2454   PetscFunctionReturn(0);
2455 }
2456 
2457 /*@
2458   DMPlexGetOrientedFace - Given a cell and a face, as a set of vertices, return the oriented face, as a set of vertices,
2459   in faceVertices. The orientation is such that the face normal points out of the cell
2460 
2461   Not collective
2462 
2463   Input Parameters:
2464 + dm           - The original mesh
2465 . cell         - The cell mesh point
2466 . faceSize     - The number of vertices on the face
2467 . face         - The face vertices
2468 . numCorners   - The number of vertices on the cell
2469 . indices      - Local numbering of face vertices in cell cone
2470 - origVertices - Original face vertices
2471 
2472   Output Parameter:
2473 + faceVertices - The face vertices properly oriented
2474 - posOriented  - PETSC_TRUE if the face was oriented with outward normal
2475 
2476   Level: developer
2477 
2478 .seealso: DMPlexGetCone()
2479 @*/
2480 PetscErrorCode DMPlexGetOrientedFace(DM dm, PetscInt cell, PetscInt faceSize, const PetscInt face[], PetscInt numCorners, PetscInt indices[], PetscInt origVertices[], PetscInt faceVertices[], PetscBool *posOriented)
2481 {
2482   const PetscInt *cone = NULL;
2483   PetscInt        coneSize, v, f, v2;
2484   PetscInt        oppositeVertex = -1;
2485   PetscErrorCode  ierr;
2486 
2487   PetscFunctionBegin;
2488   ierr = DMPlexGetConeSize(dm, cell, &coneSize);CHKERRQ(ierr);
2489   ierr = DMPlexGetCone(dm, cell, &cone);CHKERRQ(ierr);
2490   for (v = 0, v2 = 0; v < coneSize; ++v) {
2491     PetscBool found = PETSC_FALSE;
2492 
2493     for (f = 0; f < faceSize; ++f) {
2494       if (face[f] == cone[v]) {
2495         found = PETSC_TRUE; break;
2496       }
2497     }
2498     if (found) {
2499       indices[v2]      = v;
2500       origVertices[v2] = cone[v];
2501       ++v2;
2502     } else {
2503       oppositeVertex = v;
2504     }
2505   }
2506   ierr = DMPlexGetFaceOrientation(dm, cell, numCorners, indices, oppositeVertex, origVertices, faceVertices, posOriented);CHKERRQ(ierr);
2507   PetscFunctionReturn(0);
2508 }
2509 
2510 /*
2511   DMPlexInsertFace_Internal - Puts a face into the mesh
2512 
2513   Not collective
2514 
2515   Input Parameters:
2516   + dm              - The DMPlex
2517   . numFaceVertex   - The number of vertices in the face
2518   . faceVertices    - The vertices in the face for dm
2519   . subfaceVertices - The vertices in the face for subdm
2520   . numCorners      - The number of vertices in the cell
2521   . cell            - A cell in dm containing the face
2522   . subcell         - A cell in subdm containing the face
2523   . firstFace       - First face in the mesh
2524   - newFacePoint    - Next face in the mesh
2525 
2526   Output Parameters:
2527   . newFacePoint - Contains next face point number on input, updated on output
2528 
2529   Level: developer
2530 */
2531 static PetscErrorCode DMPlexInsertFace_Internal(DM dm, DM subdm, PetscInt numFaceVertices, const PetscInt faceVertices[], const PetscInt subfaceVertices[], PetscInt numCorners, PetscInt cell, PetscInt subcell, PetscInt firstFace, PetscInt *newFacePoint)
2532 {
2533   MPI_Comm        comm;
2534   DM_Plex        *submesh = (DM_Plex*) subdm->data;
2535   const PetscInt *faces;
2536   PetscInt        numFaces, coneSize;
2537   PetscErrorCode  ierr;
2538 
2539   PetscFunctionBegin;
2540   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
2541   ierr = DMPlexGetConeSize(subdm, subcell, &coneSize);CHKERRQ(ierr);
2542   if (coneSize != 1) SETERRQ2(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cone size of cell %d is %d != 1", cell, coneSize);
2543 #if 0
2544   /* Cannot use this because support() has not been constructed yet */
2545   ierr = DMPlexGetJoin(subdm, numFaceVertices, subfaceVertices, &numFaces, &faces);CHKERRQ(ierr);
2546 #else
2547   {
2548     PetscInt f;
2549 
2550     numFaces = 0;
2551     ierr     = DMGetWorkArray(subdm, 1, PETSC_INT, (void **) &faces);CHKERRQ(ierr);
2552     for (f = firstFace; f < *newFacePoint; ++f) {
2553       PetscInt dof, off, d;
2554 
2555       ierr = PetscSectionGetDof(submesh->coneSection, f, &dof);CHKERRQ(ierr);
2556       ierr = PetscSectionGetOffset(submesh->coneSection, f, &off);CHKERRQ(ierr);
2557       /* Yes, I know this is quadratic, but I expect the sizes to be <5 */
2558       for (d = 0; d < dof; ++d) {
2559         const PetscInt p = submesh->cones[off+d];
2560         PetscInt       v;
2561 
2562         for (v = 0; v < numFaceVertices; ++v) {
2563           if (subfaceVertices[v] == p) break;
2564         }
2565         if (v == numFaceVertices) break;
2566       }
2567       if (d == dof) {
2568         numFaces               = 1;
2569         ((PetscInt*) faces)[0] = f;
2570       }
2571     }
2572   }
2573 #endif
2574   if (numFaces > 1) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Vertex set had %d faces, not one", numFaces);
2575   else if (numFaces == 1) {
2576     /* Add the other cell neighbor for this face */
2577     ierr = DMPlexSetCone(subdm, subcell, faces);CHKERRQ(ierr);
2578   } else {
2579     PetscInt *indices, *origVertices, *orientedVertices, *orientedSubVertices, v, ov;
2580     PetscBool posOriented;
2581 
2582     ierr                = DMGetWorkArray(subdm, 4*numFaceVertices * sizeof(PetscInt), PETSC_INT, &orientedVertices);CHKERRQ(ierr);
2583     origVertices        = &orientedVertices[numFaceVertices];
2584     indices             = &orientedVertices[numFaceVertices*2];
2585     orientedSubVertices = &orientedVertices[numFaceVertices*3];
2586     ierr                = DMPlexGetOrientedFace(dm, cell, numFaceVertices, faceVertices, numCorners, indices, origVertices, orientedVertices, &posOriented);CHKERRQ(ierr);
2587     /* TODO: I know that routine should return a permutation, not the indices */
2588     for (v = 0; v < numFaceVertices; ++v) {
2589       const PetscInt vertex = faceVertices[v], subvertex = subfaceVertices[v];
2590       for (ov = 0; ov < numFaceVertices; ++ov) {
2591         if (orientedVertices[ov] == vertex) {
2592           orientedSubVertices[ov] = subvertex;
2593           break;
2594         }
2595       }
2596       if (ov == numFaceVertices) SETERRQ1(comm, PETSC_ERR_PLIB, "Could not find face vertex %d in orientated set", vertex);
2597     }
2598     ierr = DMPlexSetCone(subdm, *newFacePoint, orientedSubVertices);CHKERRQ(ierr);
2599     ierr = DMPlexSetCone(subdm, subcell, newFacePoint);CHKERRQ(ierr);
2600     ierr = DMRestoreWorkArray(subdm, 4*numFaceVertices * sizeof(PetscInt), PETSC_INT, &orientedVertices);CHKERRQ(ierr);
2601     ++(*newFacePoint);
2602   }
2603 #if 0
2604   ierr = DMPlexRestoreJoin(subdm, numFaceVertices, subfaceVertices, &numFaces, &faces);CHKERRQ(ierr);
2605 #else
2606   ierr = DMRestoreWorkArray(subdm, 1, PETSC_INT, (void **) &faces);CHKERRQ(ierr);
2607 #endif
2608   PetscFunctionReturn(0);
2609 }
2610 
2611 static PetscErrorCode DMPlexCreateSubmesh_Uninterpolated(DM dm, DMLabel vertexLabel, PetscInt value, DM subdm)
2612 {
2613   MPI_Comm        comm;
2614   DMLabel         subpointMap;
2615   IS              subvertexIS,  subcellIS;
2616   const PetscInt *subVertices, *subCells;
2617   PetscInt        numSubVertices, firstSubVertex, numSubCells;
2618   PetscInt       *subface, maxConeSize, numSubFaces = 0, firstSubFace, newFacePoint, nFV = 0;
2619   PetscInt        vStart, vEnd, c, f;
2620   PetscErrorCode  ierr;
2621 
2622   PetscFunctionBegin;
2623   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
2624   /* Create subpointMap which marks the submesh */
2625   ierr = DMLabelCreate("subpoint_map", &subpointMap);CHKERRQ(ierr);
2626   ierr = DMPlexSetSubpointMap(subdm, subpointMap);CHKERRQ(ierr);
2627   ierr = DMLabelDestroy(&subpointMap);CHKERRQ(ierr);
2628   if (vertexLabel) {ierr = DMPlexMarkSubmesh_Uninterpolated(dm, vertexLabel, value, subpointMap, &numSubFaces, &nFV, subdm);CHKERRQ(ierr);}
2629   /* Setup chart */
2630   ierr = DMLabelGetStratumSize(subpointMap, 0, &numSubVertices);CHKERRQ(ierr);
2631   ierr = DMLabelGetStratumSize(subpointMap, 2, &numSubCells);CHKERRQ(ierr);
2632   ierr = DMPlexSetChart(subdm, 0, numSubCells+numSubFaces+numSubVertices);CHKERRQ(ierr);
2633   ierr = DMPlexSetVTKCellHeight(subdm, 1);CHKERRQ(ierr);
2634   /* Set cone sizes */
2635   firstSubVertex = numSubCells;
2636   firstSubFace   = numSubCells+numSubVertices;
2637   newFacePoint   = firstSubFace;
2638   ierr = DMLabelGetStratumIS(subpointMap, 0, &subvertexIS);CHKERRQ(ierr);
2639   if (subvertexIS) {ierr = ISGetIndices(subvertexIS, &subVertices);CHKERRQ(ierr);}
2640   ierr = DMLabelGetStratumIS(subpointMap, 2, &subcellIS);CHKERRQ(ierr);
2641   if (subcellIS) {ierr = ISGetIndices(subcellIS, &subCells);CHKERRQ(ierr);}
2642   for (c = 0; c < numSubCells; ++c) {
2643     ierr = DMPlexSetConeSize(subdm, c, 1);CHKERRQ(ierr);
2644   }
2645   for (f = firstSubFace; f < firstSubFace+numSubFaces; ++f) {
2646     ierr = DMPlexSetConeSize(subdm, f, nFV);CHKERRQ(ierr);
2647   }
2648   ierr = DMSetUp(subdm);CHKERRQ(ierr);
2649   /* Create face cones */
2650   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
2651   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, NULL);CHKERRQ(ierr);
2652   ierr = DMGetWorkArray(subdm, maxConeSize, PETSC_INT, (void**) &subface);CHKERRQ(ierr);
2653   for (c = 0; c < numSubCells; ++c) {
2654     const PetscInt cell    = subCells[c];
2655     const PetscInt subcell = c;
2656     PetscInt      *closure = NULL;
2657     PetscInt       closureSize, cl, numCorners = 0, faceSize = 0;
2658 
2659     ierr = DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
2660     for (cl = 0; cl < closureSize*2; cl += 2) {
2661       const PetscInt point = closure[cl];
2662       PetscInt       subVertex;
2663 
2664       if ((point >= vStart) && (point < vEnd)) {
2665         ++numCorners;
2666         ierr = PetscFindInt(point, numSubVertices, subVertices, &subVertex);CHKERRQ(ierr);
2667         if (subVertex >= 0) {
2668           closure[faceSize] = point;
2669           subface[faceSize] = firstSubVertex+subVertex;
2670           ++faceSize;
2671         }
2672       }
2673     }
2674     if (faceSize > nFV) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Invalid submesh: Too many vertices %d of an element on the surface", faceSize);
2675     if (faceSize == nFV) {
2676       ierr = DMPlexInsertFace_Internal(dm, subdm, faceSize, closure, subface, numCorners, cell, subcell, firstSubFace, &newFacePoint);CHKERRQ(ierr);
2677     }
2678     ierr = DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
2679   }
2680   ierr = DMRestoreWorkArray(subdm, maxConeSize, PETSC_INT, (void**) &subface);CHKERRQ(ierr);
2681   ierr = DMPlexSymmetrize(subdm);CHKERRQ(ierr);
2682   ierr = DMPlexStratify(subdm);CHKERRQ(ierr);
2683   /* Build coordinates */
2684   {
2685     PetscSection coordSection, subCoordSection;
2686     Vec          coordinates, subCoordinates;
2687     PetscScalar *coords, *subCoords;
2688     PetscInt     numComp, coordSize, v;
2689     const char  *name;
2690 
2691     ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
2692     ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
2693     ierr = DMGetCoordinateSection(subdm, &subCoordSection);CHKERRQ(ierr);
2694     ierr = PetscSectionSetNumFields(subCoordSection, 1);CHKERRQ(ierr);
2695     ierr = PetscSectionGetFieldComponents(coordSection, 0, &numComp);CHKERRQ(ierr);
2696     ierr = PetscSectionSetFieldComponents(subCoordSection, 0, numComp);CHKERRQ(ierr);
2697     ierr = PetscSectionSetChart(subCoordSection, firstSubVertex, firstSubVertex+numSubVertices);CHKERRQ(ierr);
2698     for (v = 0; v < numSubVertices; ++v) {
2699       const PetscInt vertex    = subVertices[v];
2700       const PetscInt subvertex = firstSubVertex+v;
2701       PetscInt       dof;
2702 
2703       ierr = PetscSectionGetDof(coordSection, vertex, &dof);CHKERRQ(ierr);
2704       ierr = PetscSectionSetDof(subCoordSection, subvertex, dof);CHKERRQ(ierr);
2705       ierr = PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof);CHKERRQ(ierr);
2706     }
2707     ierr = PetscSectionSetUp(subCoordSection);CHKERRQ(ierr);
2708     ierr = PetscSectionGetStorageSize(subCoordSection, &coordSize);CHKERRQ(ierr);
2709     ierr = VecCreate(PETSC_COMM_SELF, &subCoordinates);CHKERRQ(ierr);
2710     ierr = PetscObjectGetName((PetscObject)coordinates,&name);CHKERRQ(ierr);
2711     ierr = PetscObjectSetName((PetscObject)subCoordinates,name);CHKERRQ(ierr);
2712     ierr = VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
2713     ierr = VecSetType(subCoordinates,VECSTANDARD);CHKERRQ(ierr);
2714     if (coordSize) {
2715       ierr = VecGetArray(coordinates,    &coords);CHKERRQ(ierr);
2716       ierr = VecGetArray(subCoordinates, &subCoords);CHKERRQ(ierr);
2717       for (v = 0; v < numSubVertices; ++v) {
2718         const PetscInt vertex    = subVertices[v];
2719         const PetscInt subvertex = firstSubVertex+v;
2720         PetscInt       dof, off, sdof, soff, d;
2721 
2722         ierr = PetscSectionGetDof(coordSection, vertex, &dof);CHKERRQ(ierr);
2723         ierr = PetscSectionGetOffset(coordSection, vertex, &off);CHKERRQ(ierr);
2724         ierr = PetscSectionGetDof(subCoordSection, subvertex, &sdof);CHKERRQ(ierr);
2725         ierr = PetscSectionGetOffset(subCoordSection, subvertex, &soff);CHKERRQ(ierr);
2726         if (dof != sdof) SETERRQ4(comm, PETSC_ERR_PLIB, "Coordinate dimension %d on subvertex %d, vertex %d should be %d", sdof, subvertex, vertex, dof);
2727         for (d = 0; d < dof; ++d) subCoords[soff+d] = coords[off+d];
2728       }
2729       ierr = VecRestoreArray(coordinates,    &coords);CHKERRQ(ierr);
2730       ierr = VecRestoreArray(subCoordinates, &subCoords);CHKERRQ(ierr);
2731     }
2732     ierr = DMSetCoordinatesLocal(subdm, subCoordinates);CHKERRQ(ierr);
2733     ierr = VecDestroy(&subCoordinates);CHKERRQ(ierr);
2734   }
2735   /* Cleanup */
2736   if (subvertexIS) {ierr = ISRestoreIndices(subvertexIS, &subVertices);CHKERRQ(ierr);}
2737   ierr = ISDestroy(&subvertexIS);CHKERRQ(ierr);
2738   if (subcellIS) {ierr = ISRestoreIndices(subcellIS, &subCells);CHKERRQ(ierr);}
2739   ierr = ISDestroy(&subcellIS);CHKERRQ(ierr);
2740   PetscFunctionReturn(0);
2741 }
2742 
2743 PETSC_STATIC_INLINE PetscInt DMPlexFilterPoint_Internal(PetscInt point, PetscInt firstSubPoint, PetscInt numSubPoints, const PetscInt subPoints[])
2744 {
2745   PetscInt       subPoint;
2746   PetscErrorCode ierr;
2747 
2748   ierr = PetscFindInt(point, numSubPoints, subPoints, &subPoint); if (ierr < 0) return ierr;
2749   return subPoint < 0 ? subPoint : firstSubPoint+subPoint;
2750 }
2751 
2752 static PetscErrorCode DMPlexCreateSubmeshGeneric_Interpolated(DM dm, DMLabel label, PetscInt value, PetscBool isCohesive, PetscInt cellHeight, DM subdm)
2753 {
2754   MPI_Comm         comm;
2755   DMLabel          subpointMap;
2756   IS              *subpointIS;
2757   const PetscInt **subpoints;
2758   PetscInt        *numSubPoints, *firstSubPoint, *coneNew, *orntNew;
2759   PetscInt         totSubPoints = 0, maxConeSize, cMax, cEnd, dim, p, d, v;
2760   PetscMPIInt      rank;
2761   PetscErrorCode   ierr;
2762 
2763   PetscFunctionBegin;
2764   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
2765   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
2766   /* Create subpointMap which marks the submesh */
2767   ierr = DMLabelCreate("subpoint_map", &subpointMap);CHKERRQ(ierr);
2768   ierr = DMPlexSetSubpointMap(subdm, subpointMap);CHKERRQ(ierr);
2769   if (cellHeight) {
2770     if (isCohesive) {ierr = DMPlexMarkCohesiveSubmesh_Interpolated(dm, label, value, subpointMap, subdm);CHKERRQ(ierr);}
2771     else            {ierr = DMPlexMarkSubmesh_Interpolated(dm, label, value, subpointMap, subdm);CHKERRQ(ierr);}
2772   } else {
2773     DMLabel         depth;
2774     IS              pointIS;
2775     const PetscInt *points;
2776     PetscInt        numPoints;
2777 
2778     ierr = DMPlexGetDepthLabel(dm, &depth);CHKERRQ(ierr);
2779     ierr = DMLabelGetStratumSize(label, value, &numPoints);CHKERRQ(ierr);
2780     ierr = DMLabelGetStratumIS(label, value, &pointIS);CHKERRQ(ierr);
2781     ierr = ISGetIndices(pointIS, &points);CHKERRQ(ierr);
2782     for (p = 0; p < numPoints; ++p) {
2783       PetscInt *closure = NULL;
2784       PetscInt  closureSize, c, pdim;
2785 
2786       ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
2787       for (c = 0; c < closureSize*2; c += 2) {
2788         ierr = DMLabelGetValue(depth, closure[c], &pdim);CHKERRQ(ierr);
2789         ierr = DMLabelSetValue(subpointMap, closure[c], pdim);CHKERRQ(ierr);
2790       }
2791       ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
2792     }
2793     ierr = ISRestoreIndices(pointIS, &points);CHKERRQ(ierr);
2794     ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
2795   }
2796   ierr = DMLabelDestroy(&subpointMap);CHKERRQ(ierr);
2797   ierr = DMPlexGetHeightStratum(dm, 0, NULL, &cEnd);CHKERRQ(ierr);
2798   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
2799   cMax = (cMax < 0) ? cEnd : cMax;
2800   /* Setup chart */
2801   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
2802   ierr = PetscMalloc4(dim+1,&numSubPoints,dim+1,&firstSubPoint,dim+1,&subpointIS,dim+1,&subpoints);CHKERRQ(ierr);
2803   for (d = 0; d <= dim; ++d) {
2804     ierr = DMLabelGetStratumSize(subpointMap, d, &numSubPoints[d]);CHKERRQ(ierr);
2805     totSubPoints += numSubPoints[d];
2806   }
2807   ierr = DMPlexSetChart(subdm, 0, totSubPoints);CHKERRQ(ierr);
2808   ierr = DMPlexSetVTKCellHeight(subdm, cellHeight);CHKERRQ(ierr);
2809   /* Set cone sizes */
2810   firstSubPoint[dim] = 0;
2811   firstSubPoint[0]   = firstSubPoint[dim] + numSubPoints[dim];
2812   if (dim > 1) {firstSubPoint[dim-1] = firstSubPoint[0]     + numSubPoints[0];}
2813   if (dim > 2) {firstSubPoint[dim-2] = firstSubPoint[dim-1] + numSubPoints[dim-1];}
2814   for (d = 0; d <= dim; ++d) {
2815     ierr = DMLabelGetStratumIS(subpointMap, d, &subpointIS[d]);CHKERRQ(ierr);
2816     if (subpointIS[d]) {ierr = ISGetIndices(subpointIS[d], &subpoints[d]);CHKERRQ(ierr);}
2817   }
2818   for (d = 0; d <= dim; ++d) {
2819     for (p = 0; p < numSubPoints[d]; ++p) {
2820       const PetscInt  point    = subpoints[d][p];
2821       const PetscInt  subpoint = firstSubPoint[d] + p;
2822       const PetscInt *cone;
2823       PetscInt        coneSize, coneSizeNew, c, val;
2824 
2825       ierr = DMPlexGetConeSize(dm, point, &coneSize);CHKERRQ(ierr);
2826       ierr = DMPlexSetConeSize(subdm, subpoint, coneSize);CHKERRQ(ierr);
2827       if (cellHeight && (d == dim)) {
2828         ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
2829         for (c = 0, coneSizeNew = 0; c < coneSize; ++c) {
2830           ierr = DMLabelGetValue(subpointMap, cone[c], &val);CHKERRQ(ierr);
2831           if (val >= 0) coneSizeNew++;
2832         }
2833         ierr = DMPlexSetConeSize(subdm, subpoint, coneSizeNew);CHKERRQ(ierr);
2834       }
2835     }
2836   }
2837   ierr = DMSetUp(subdm);CHKERRQ(ierr);
2838   /* Set cones */
2839   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, NULL);CHKERRQ(ierr);
2840   ierr = PetscMalloc2(maxConeSize,&coneNew,maxConeSize,&orntNew);CHKERRQ(ierr);
2841   for (d = 0; d <= dim; ++d) {
2842     for (p = 0; p < numSubPoints[d]; ++p) {
2843       const PetscInt  point    = subpoints[d][p];
2844       const PetscInt  subpoint = firstSubPoint[d] + p;
2845       const PetscInt *cone, *ornt;
2846       PetscInt        coneSize, subconeSize, coneSizeNew, c, subc, fornt = 0;
2847 
2848       if (d == dim-1) {
2849         const PetscInt *support, *cone, *ornt;
2850         PetscInt        supportSize, coneSize, s, subc;
2851 
2852         ierr = DMPlexGetSupport(dm, point, &support);CHKERRQ(ierr);
2853         ierr = DMPlexGetSupportSize(dm, point, &supportSize);CHKERRQ(ierr);
2854         for (s = 0; s < supportSize; ++s) {
2855           if ((support[s] < cMax) || (support[s] >= cEnd)) continue;
2856           ierr = PetscFindInt(support[s], numSubPoints[d+1], subpoints[d+1], &subc);CHKERRQ(ierr);
2857           if (subc >= 0) {
2858             const PetscInt ccell = subpoints[d+1][subc];
2859 
2860             ierr = DMPlexGetCone(dm, ccell, &cone);
2861             ierr = DMPlexGetConeSize(dm, ccell, &coneSize);
2862             ierr = DMPlexGetConeOrientation(dm, ccell, &ornt);
2863             for (c = 0; c < coneSize; ++c) {
2864               if (cone[c] == point) {
2865                 fornt = ornt[c];
2866                 break;
2867               }
2868             }
2869             break;
2870           }
2871         }
2872       }
2873       ierr = DMPlexGetConeSize(dm, point, &coneSize);CHKERRQ(ierr);
2874       ierr = DMPlexGetConeSize(subdm, subpoint, &subconeSize);CHKERRQ(ierr);
2875       ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
2876       ierr = DMPlexGetConeOrientation(dm, point, &ornt);CHKERRQ(ierr);
2877       for (c = 0, coneSizeNew = 0; c < coneSize; ++c) {
2878         ierr = PetscFindInt(cone[c], numSubPoints[d-1], subpoints[d-1], &subc);CHKERRQ(ierr);
2879         if (subc >= 0) {
2880           coneNew[coneSizeNew] = firstSubPoint[d-1] + subc;
2881           orntNew[coneSizeNew] = ornt[c];
2882           ++coneSizeNew;
2883         }
2884       }
2885       if (coneSizeNew != subconeSize) SETERRQ2(comm, PETSC_ERR_PLIB, "Number of cone points located %d does not match subcone size %d", coneSizeNew, subconeSize);
2886       if (fornt < 0) {
2887         /* This should be replaced by a call to DMPlexReverseCell() */
2888 #if 0
2889         ierr = DMPlexReverseCell(subdm, subpoint);CHKERRQ(ierr);
2890 #else
2891         for (c = 0; c < coneSizeNew/2 + coneSizeNew%2; ++c) {
2892           PetscInt faceSize, tmp;
2893 
2894           tmp        = coneNew[c];
2895           coneNew[c] = coneNew[coneSizeNew-1-c];
2896           coneNew[coneSizeNew-1-c] = tmp;
2897           ierr = DMPlexGetConeSize(dm, cone[c], &faceSize);CHKERRQ(ierr);
2898           tmp        = orntNew[c] >= 0 ? -(faceSize-orntNew[c]) : faceSize+orntNew[c];
2899           orntNew[c] = orntNew[coneSizeNew-1-c] >= 0 ? -(faceSize-orntNew[coneSizeNew-1-c]) : faceSize+orntNew[coneSizeNew-1-c];
2900           orntNew[coneSizeNew-1-c] = tmp;
2901         }
2902       }
2903       ierr = DMPlexSetCone(subdm, subpoint, coneNew);CHKERRQ(ierr);
2904       ierr = DMPlexSetConeOrientation(subdm, subpoint, orntNew);CHKERRQ(ierr);
2905 #endif
2906     }
2907   }
2908   ierr = PetscFree2(coneNew,orntNew);CHKERRQ(ierr);
2909   ierr = DMPlexSymmetrize(subdm);CHKERRQ(ierr);
2910   ierr = DMPlexStratify(subdm);CHKERRQ(ierr);
2911   /* Build coordinates */
2912   {
2913     PetscSection coordSection, subCoordSection;
2914     Vec          coordinates, subCoordinates;
2915     PetscScalar *coords, *subCoords;
2916     PetscInt     numComp, coordSize;
2917     const char  *name;
2918 
2919     ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
2920     ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
2921     ierr = DMGetCoordinateSection(subdm, &subCoordSection);CHKERRQ(ierr);
2922     ierr = PetscSectionSetNumFields(subCoordSection, 1);CHKERRQ(ierr);
2923     ierr = PetscSectionGetFieldComponents(coordSection, 0, &numComp);CHKERRQ(ierr);
2924     ierr = PetscSectionSetFieldComponents(subCoordSection, 0, numComp);CHKERRQ(ierr);
2925     ierr = PetscSectionSetChart(subCoordSection, firstSubPoint[0], firstSubPoint[0]+numSubPoints[0]);CHKERRQ(ierr);
2926     for (v = 0; v < numSubPoints[0]; ++v) {
2927       const PetscInt vertex    = subpoints[0][v];
2928       const PetscInt subvertex = firstSubPoint[0]+v;
2929       PetscInt       dof;
2930 
2931       ierr = PetscSectionGetDof(coordSection, vertex, &dof);CHKERRQ(ierr);
2932       ierr = PetscSectionSetDof(subCoordSection, subvertex, dof);CHKERRQ(ierr);
2933       ierr = PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof);CHKERRQ(ierr);
2934     }
2935     ierr = PetscSectionSetUp(subCoordSection);CHKERRQ(ierr);
2936     ierr = PetscSectionGetStorageSize(subCoordSection, &coordSize);CHKERRQ(ierr);
2937     ierr = VecCreate(PETSC_COMM_SELF, &subCoordinates);CHKERRQ(ierr);
2938     ierr = PetscObjectGetName((PetscObject)coordinates,&name);CHKERRQ(ierr);
2939     ierr = PetscObjectSetName((PetscObject)subCoordinates,name);CHKERRQ(ierr);
2940     ierr = VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
2941     ierr = VecSetType(subCoordinates,VECSTANDARD);CHKERRQ(ierr);
2942     ierr = VecGetArray(coordinates,    &coords);CHKERRQ(ierr);
2943     ierr = VecGetArray(subCoordinates, &subCoords);CHKERRQ(ierr);
2944     for (v = 0; v < numSubPoints[0]; ++v) {
2945       const PetscInt vertex    = subpoints[0][v];
2946       const PetscInt subvertex = firstSubPoint[0]+v;
2947       PetscInt dof, off, sdof, soff, d;
2948 
2949       ierr = PetscSectionGetDof(coordSection, vertex, &dof);CHKERRQ(ierr);
2950       ierr = PetscSectionGetOffset(coordSection, vertex, &off);CHKERRQ(ierr);
2951       ierr = PetscSectionGetDof(subCoordSection, subvertex, &sdof);CHKERRQ(ierr);
2952       ierr = PetscSectionGetOffset(subCoordSection, subvertex, &soff);CHKERRQ(ierr);
2953       if (dof != sdof) SETERRQ4(comm, PETSC_ERR_PLIB, "Coordinate dimension %d on subvertex %d, vertex %d should be %d", sdof, subvertex, vertex, dof);
2954       for (d = 0; d < dof; ++d) subCoords[soff+d] = coords[off+d];
2955     }
2956     ierr = VecRestoreArray(coordinates,    &coords);CHKERRQ(ierr);
2957     ierr = VecRestoreArray(subCoordinates, &subCoords);CHKERRQ(ierr);
2958     ierr = DMSetCoordinatesLocal(subdm, subCoordinates);CHKERRQ(ierr);
2959     ierr = VecDestroy(&subCoordinates);CHKERRQ(ierr);
2960   }
2961   /* Build SF: We need this complexity because subpoints might not be selected on the owning process */
2962   {
2963     PetscSF            sfPoint, sfPointSub;
2964     IS                 subpIS;
2965     const PetscSFNode *remotePoints;
2966     PetscSFNode       *sremotePoints, *newLocalPoints, *newOwners;
2967     const PetscInt    *localPoints, *subpoints;
2968     PetscInt          *slocalPoints;
2969     PetscInt           numRoots, numLeaves, numSubpoints = 0, numSubroots, numSubleaves = 0, l, sl, ll, pStart, pEnd, p;
2970     PetscMPIInt        rank;
2971 
2972     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRQ(ierr);
2973     ierr = DMGetPointSF(dm, &sfPoint);CHKERRQ(ierr);
2974     ierr = DMGetPointSF(subdm, &sfPointSub);CHKERRQ(ierr);
2975     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
2976     ierr = DMPlexGetChart(subdm, NULL, &numSubroots);CHKERRQ(ierr);
2977     ierr = DMPlexCreateSubpointIS(subdm, &subpIS);CHKERRQ(ierr);
2978     if (subpIS) {
2979       ierr = ISGetIndices(subpIS, &subpoints);CHKERRQ(ierr);
2980       ierr = ISGetLocalSize(subpIS, &numSubpoints);CHKERRQ(ierr);
2981     }
2982     ierr = PetscSFGetGraph(sfPoint, &numRoots, &numLeaves, &localPoints, &remotePoints);CHKERRQ(ierr);
2983     if (numRoots >= 0) {
2984       ierr = PetscMalloc2(pEnd-pStart,&newLocalPoints,numRoots,&newOwners);CHKERRQ(ierr);
2985       for (p = 0; p < pEnd-pStart; ++p) {
2986         newLocalPoints[p].rank  = -2;
2987         newLocalPoints[p].index = -2;
2988       }
2989       /* Set subleaves */
2990       for (l = 0; l < numLeaves; ++l) {
2991         const PetscInt point    = localPoints[l];
2992         const PetscInt subpoint = DMPlexFilterPoint_Internal(point, 0, numSubpoints, subpoints);
2993 
2994         if (subpoint < 0) continue;
2995         newLocalPoints[point-pStart].rank  = rank;
2996         newLocalPoints[point-pStart].index = subpoint;
2997         ++numSubleaves;
2998       }
2999       /* Must put in owned subpoints */
3000       for (p = pStart; p < pEnd; ++p) {
3001         const PetscInt subpoint = DMPlexFilterPoint_Internal(p, 0, numSubpoints, subpoints);
3002 
3003         if (subpoint < 0) {
3004           newOwners[p-pStart].rank  = -3;
3005           newOwners[p-pStart].index = -3;
3006         } else {
3007           newOwners[p-pStart].rank  = rank;
3008           newOwners[p-pStart].index = subpoint;
3009         }
3010       }
3011       ierr = PetscSFReduceBegin(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC);CHKERRQ(ierr);
3012       ierr = PetscSFReduceEnd(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC);CHKERRQ(ierr);
3013       ierr = PetscSFBcastBegin(sfPoint, MPIU_2INT, newOwners, newLocalPoints);CHKERRQ(ierr);
3014       ierr = PetscSFBcastEnd(sfPoint, MPIU_2INT, newOwners, newLocalPoints);CHKERRQ(ierr);
3015       ierr = PetscMalloc1(numSubleaves, &slocalPoints);CHKERRQ(ierr);
3016       ierr = PetscMalloc1(numSubleaves, &sremotePoints);CHKERRQ(ierr);
3017       for (l = 0, sl = 0, ll = 0; l < numLeaves; ++l) {
3018         const PetscInt point    = localPoints[l];
3019         const PetscInt subpoint = DMPlexFilterPoint_Internal(point, 0, numSubpoints, subpoints);
3020 
3021         if (subpoint < 0) continue;
3022         if (newLocalPoints[point].rank == rank) {++ll; continue;}
3023         slocalPoints[sl]        = subpoint;
3024         sremotePoints[sl].rank  = newLocalPoints[point].rank;
3025         sremotePoints[sl].index = newLocalPoints[point].index;
3026         if (sremotePoints[sl].rank  < 0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote rank for local point %d", point);
3027         if (sremotePoints[sl].index < 0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote subpoint for local point %d", point);
3028         ++sl;
3029       }
3030       if (sl + ll != numSubleaves) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatch in number of subleaves %d + %d != %d", sl, ll, numSubleaves);
3031       ierr = PetscFree2(newLocalPoints,newOwners);CHKERRQ(ierr);
3032       ierr = PetscSFSetGraph(sfPointSub, numSubroots, sl, slocalPoints, PETSC_OWN_POINTER, sremotePoints, PETSC_OWN_POINTER);CHKERRQ(ierr);
3033     }
3034     if (subpIS) {
3035       ierr = ISRestoreIndices(subpIS, &subpoints);CHKERRQ(ierr);
3036       ierr = ISDestroy(&subpIS);CHKERRQ(ierr);
3037     }
3038   }
3039   /* Cleanup */
3040   for (d = 0; d <= dim; ++d) {
3041     if (subpointIS[d]) {ierr = ISRestoreIndices(subpointIS[d], &subpoints[d]);CHKERRQ(ierr);}
3042     ierr = ISDestroy(&subpointIS[d]);CHKERRQ(ierr);
3043   }
3044   ierr = PetscFree4(numSubPoints,firstSubPoint,subpointIS,subpoints);CHKERRQ(ierr);
3045   PetscFunctionReturn(0);
3046 }
3047 
3048 static PetscErrorCode DMPlexCreateSubmesh_Interpolated(DM dm, DMLabel vertexLabel, PetscInt value, DM subdm)
3049 {
3050   PetscErrorCode ierr;
3051 
3052   PetscFunctionBegin;
3053   ierr = DMPlexCreateSubmeshGeneric_Interpolated(dm, vertexLabel, value, PETSC_FALSE, 1, subdm);CHKERRQ(ierr);
3054   PetscFunctionReturn(0);
3055 }
3056 
3057 /*@
3058   DMPlexCreateSubmesh - Extract a hypersurface from the mesh using vertices defined by a label
3059 
3060   Input Parameters:
3061 + dm           - The original mesh
3062 . vertexLabel  - The DMLabel marking vertices contained in the surface
3063 - value        - The label value to use
3064 
3065   Output Parameter:
3066 . subdm - The surface mesh
3067 
3068   Note: This function produces a DMLabel mapping original points in the submesh to their depth. This can be obtained using DMPlexGetSubpointMap().
3069 
3070   Level: developer
3071 
3072 .seealso: DMPlexGetSubpointMap(), DMGetLabel(), DMLabelSetValue()
3073 @*/
3074 PetscErrorCode DMPlexCreateSubmesh(DM dm, DMLabel vertexLabel, PetscInt value, DM *subdm)
3075 {
3076   PetscInt       dim, cdim, depth;
3077   PetscErrorCode ierr;
3078 
3079   PetscFunctionBegin;
3080   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3081   PetscValidPointer(subdm, 3);
3082   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
3083   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3084   ierr = DMCreate(PetscObjectComm((PetscObject)dm), subdm);CHKERRQ(ierr);
3085   ierr = DMSetType(*subdm, DMPLEX);CHKERRQ(ierr);
3086   ierr = DMSetDimension(*subdm, dim-1);CHKERRQ(ierr);
3087   ierr = DMGetCoordinateDim(dm, &cdim);CHKERRQ(ierr);
3088   ierr = DMSetCoordinateDim(*subdm, cdim);CHKERRQ(ierr);
3089   if (depth == dim) {
3090     ierr = DMPlexCreateSubmesh_Interpolated(dm, vertexLabel, value, *subdm);CHKERRQ(ierr);
3091   } else {
3092     ierr = DMPlexCreateSubmesh_Uninterpolated(dm, vertexLabel, value, *subdm);CHKERRQ(ierr);
3093   }
3094   PetscFunctionReturn(0);
3095 }
3096 
3097 static PetscErrorCode DMPlexCreateCohesiveSubmesh_Uninterpolated(DM dm, PetscBool hasLagrange, const char label[], PetscInt value, DM subdm)
3098 {
3099   MPI_Comm        comm;
3100   DMLabel         subpointMap;
3101   IS              subvertexIS;
3102   const PetscInt *subVertices;
3103   PetscInt        numSubVertices, firstSubVertex, numSubCells, *subCells = NULL;
3104   PetscInt       *subface, maxConeSize, numSubFaces, firstSubFace, newFacePoint, nFV;
3105   PetscInt        cMax, c, f;
3106   PetscErrorCode  ierr;
3107 
3108   PetscFunctionBegin;
3109   ierr = PetscObjectGetComm((PetscObject)dm, &comm);CHKERRQ(ierr);
3110   /* Create subpointMap which marks the submesh */
3111   ierr = DMLabelCreate("subpoint_map", &subpointMap);CHKERRQ(ierr);
3112   ierr = DMPlexSetSubpointMap(subdm, subpointMap);CHKERRQ(ierr);
3113   ierr = DMLabelDestroy(&subpointMap);CHKERRQ(ierr);
3114   ierr = DMPlexMarkCohesiveSubmesh_Uninterpolated(dm, hasLagrange, label, value, subpointMap, &numSubFaces, &nFV, &subCells, subdm);CHKERRQ(ierr);
3115   /* Setup chart */
3116   ierr = DMLabelGetStratumSize(subpointMap, 0, &numSubVertices);CHKERRQ(ierr);
3117   ierr = DMLabelGetStratumSize(subpointMap, 2, &numSubCells);CHKERRQ(ierr);
3118   ierr = DMPlexSetChart(subdm, 0, numSubCells+numSubFaces+numSubVertices);CHKERRQ(ierr);
3119   ierr = DMPlexSetVTKCellHeight(subdm, 1);CHKERRQ(ierr);
3120   /* Set cone sizes */
3121   firstSubVertex = numSubCells;
3122   firstSubFace   = numSubCells+numSubVertices;
3123   newFacePoint   = firstSubFace;
3124   ierr = DMLabelGetStratumIS(subpointMap, 0, &subvertexIS);CHKERRQ(ierr);
3125   if (subvertexIS) {ierr = ISGetIndices(subvertexIS, &subVertices);CHKERRQ(ierr);}
3126   for (c = 0; c < numSubCells; ++c) {
3127     ierr = DMPlexSetConeSize(subdm, c, 1);CHKERRQ(ierr);
3128   }
3129   for (f = firstSubFace; f < firstSubFace+numSubFaces; ++f) {
3130     ierr = DMPlexSetConeSize(subdm, f, nFV);CHKERRQ(ierr);
3131   }
3132   ierr = DMSetUp(subdm);CHKERRQ(ierr);
3133   /* Create face cones */
3134   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, NULL);CHKERRQ(ierr);
3135   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
3136   ierr = DMGetWorkArray(subdm, maxConeSize, PETSC_INT, (void**) &subface);CHKERRQ(ierr);
3137   for (c = 0; c < numSubCells; ++c) {
3138     const PetscInt  cell    = subCells[c];
3139     const PetscInt  subcell = c;
3140     const PetscInt *cone, *cells;
3141     PetscInt        numCells, subVertex, p, v;
3142 
3143     if (cell < cMax) continue;
3144     ierr = DMPlexGetCone(dm, cell, &cone);CHKERRQ(ierr);
3145     for (v = 0; v < nFV; ++v) {
3146       ierr = PetscFindInt(cone[v], numSubVertices, subVertices, &subVertex);CHKERRQ(ierr);
3147       subface[v] = firstSubVertex+subVertex;
3148     }
3149     ierr = DMPlexSetCone(subdm, newFacePoint, subface);CHKERRQ(ierr);
3150     ierr = DMPlexSetCone(subdm, subcell, &newFacePoint);CHKERRQ(ierr);
3151     ierr = DMPlexGetJoin(dm, nFV, cone, &numCells, &cells);CHKERRQ(ierr);
3152     /* Not true in parallel
3153     if (numCells != 2) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive cells should separate two cells"); */
3154     for (p = 0; p < numCells; ++p) {
3155       PetscInt negsubcell;
3156 
3157       if (cells[p] >= cMax) continue;
3158       /* I know this is a crap search */
3159       for (negsubcell = 0; negsubcell < numSubCells; ++negsubcell) {
3160         if (subCells[negsubcell] == cells[p]) break;
3161       }
3162       if (negsubcell == numSubCells) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not find negative face neighbor for cohesive cell %d", cell);
3163       ierr = DMPlexSetCone(subdm, negsubcell, &newFacePoint);CHKERRQ(ierr);
3164     }
3165     ierr = DMPlexRestoreJoin(dm, nFV, cone, &numCells, &cells);CHKERRQ(ierr);
3166     ++newFacePoint;
3167   }
3168   ierr = DMRestoreWorkArray(subdm, maxConeSize, PETSC_INT, (void**) &subface);CHKERRQ(ierr);
3169   ierr = DMPlexSymmetrize(subdm);CHKERRQ(ierr);
3170   ierr = DMPlexStratify(subdm);CHKERRQ(ierr);
3171   /* Build coordinates */
3172   {
3173     PetscSection coordSection, subCoordSection;
3174     Vec          coordinates, subCoordinates;
3175     PetscScalar *coords, *subCoords;
3176     PetscInt     numComp, coordSize, v;
3177     const char  *name;
3178 
3179     ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
3180     ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
3181     ierr = DMGetCoordinateSection(subdm, &subCoordSection);CHKERRQ(ierr);
3182     ierr = PetscSectionSetNumFields(subCoordSection, 1);CHKERRQ(ierr);
3183     ierr = PetscSectionGetFieldComponents(coordSection, 0, &numComp);CHKERRQ(ierr);
3184     ierr = PetscSectionSetFieldComponents(subCoordSection, 0, numComp);CHKERRQ(ierr);
3185     ierr = PetscSectionSetChart(subCoordSection, firstSubVertex, firstSubVertex+numSubVertices);CHKERRQ(ierr);
3186     for (v = 0; v < numSubVertices; ++v) {
3187       const PetscInt vertex    = subVertices[v];
3188       const PetscInt subvertex = firstSubVertex+v;
3189       PetscInt       dof;
3190 
3191       ierr = PetscSectionGetDof(coordSection, vertex, &dof);CHKERRQ(ierr);
3192       ierr = PetscSectionSetDof(subCoordSection, subvertex, dof);CHKERRQ(ierr);
3193       ierr = PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof);CHKERRQ(ierr);
3194     }
3195     ierr = PetscSectionSetUp(subCoordSection);CHKERRQ(ierr);
3196     ierr = PetscSectionGetStorageSize(subCoordSection, &coordSize);CHKERRQ(ierr);
3197     ierr = VecCreate(PETSC_COMM_SELF, &subCoordinates);CHKERRQ(ierr);
3198     ierr = PetscObjectGetName((PetscObject)coordinates,&name);CHKERRQ(ierr);
3199     ierr = PetscObjectSetName((PetscObject)subCoordinates,name);CHKERRQ(ierr);
3200     ierr = VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
3201     ierr = VecSetType(subCoordinates,VECSTANDARD);CHKERRQ(ierr);
3202     ierr = VecGetArray(coordinates,    &coords);CHKERRQ(ierr);
3203     ierr = VecGetArray(subCoordinates, &subCoords);CHKERRQ(ierr);
3204     for (v = 0; v < numSubVertices; ++v) {
3205       const PetscInt vertex    = subVertices[v];
3206       const PetscInt subvertex = firstSubVertex+v;
3207       PetscInt       dof, off, sdof, soff, d;
3208 
3209       ierr = PetscSectionGetDof(coordSection, vertex, &dof);CHKERRQ(ierr);
3210       ierr = PetscSectionGetOffset(coordSection, vertex, &off);CHKERRQ(ierr);
3211       ierr = PetscSectionGetDof(subCoordSection, subvertex, &sdof);CHKERRQ(ierr);
3212       ierr = PetscSectionGetOffset(subCoordSection, subvertex, &soff);CHKERRQ(ierr);
3213       if (dof != sdof) SETERRQ4(comm, PETSC_ERR_PLIB, "Coordinate dimension %d on subvertex %d, vertex %d should be %d", sdof, subvertex, vertex, dof);
3214       for (d = 0; d < dof; ++d) subCoords[soff+d] = coords[off+d];
3215     }
3216     ierr = VecRestoreArray(coordinates,    &coords);CHKERRQ(ierr);
3217     ierr = VecRestoreArray(subCoordinates, &subCoords);CHKERRQ(ierr);
3218     ierr = DMSetCoordinatesLocal(subdm, subCoordinates);CHKERRQ(ierr);
3219     ierr = VecDestroy(&subCoordinates);CHKERRQ(ierr);
3220   }
3221   /* Build SF */
3222   CHKMEMQ;
3223   {
3224     PetscSF            sfPoint, sfPointSub;
3225     const PetscSFNode *remotePoints;
3226     PetscSFNode       *sremotePoints, *newLocalPoints, *newOwners;
3227     const PetscInt    *localPoints;
3228     PetscInt          *slocalPoints;
3229     PetscInt           numRoots, numLeaves, numSubRoots = numSubCells+numSubFaces+numSubVertices, numSubLeaves = 0, l, sl, ll, pStart, pEnd, p, vStart, vEnd;
3230     PetscMPIInt        rank;
3231 
3232     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRQ(ierr);
3233     ierr = DMGetPointSF(dm, &sfPoint);CHKERRQ(ierr);
3234     ierr = DMGetPointSF(subdm, &sfPointSub);CHKERRQ(ierr);
3235     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
3236     ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
3237     ierr = PetscSFGetGraph(sfPoint, &numRoots, &numLeaves, &localPoints, &remotePoints);CHKERRQ(ierr);
3238     if (numRoots >= 0) {
3239       /* Only vertices should be shared */
3240       ierr = PetscMalloc2(pEnd-pStart,&newLocalPoints,numRoots,&newOwners);CHKERRQ(ierr);
3241       for (p = 0; p < pEnd-pStart; ++p) {
3242         newLocalPoints[p].rank  = -2;
3243         newLocalPoints[p].index = -2;
3244       }
3245       /* Set subleaves */
3246       for (l = 0; l < numLeaves; ++l) {
3247         const PetscInt point    = localPoints[l];
3248         const PetscInt subPoint = DMPlexFilterPoint_Internal(point, firstSubVertex, numSubVertices, subVertices);
3249 
3250         if ((point < vStart) && (point >= vEnd)) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Should not be mapping anything but vertices, %d", point);
3251         if (subPoint < 0) continue;
3252         newLocalPoints[point-pStart].rank  = rank;
3253         newLocalPoints[point-pStart].index = subPoint;
3254         ++numSubLeaves;
3255       }
3256       /* Must put in owned subpoints */
3257       for (p = pStart; p < pEnd; ++p) {
3258         const PetscInt subPoint = DMPlexFilterPoint_Internal(p, firstSubVertex, numSubVertices, subVertices);
3259 
3260         if (subPoint < 0) {
3261           newOwners[p-pStart].rank  = -3;
3262           newOwners[p-pStart].index = -3;
3263         } else {
3264           newOwners[p-pStart].rank  = rank;
3265           newOwners[p-pStart].index = subPoint;
3266         }
3267       }
3268       ierr = PetscSFReduceBegin(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC);CHKERRQ(ierr);
3269       ierr = PetscSFReduceEnd(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC);CHKERRQ(ierr);
3270       ierr = PetscSFBcastBegin(sfPoint, MPIU_2INT, newOwners, newLocalPoints);CHKERRQ(ierr);
3271       ierr = PetscSFBcastEnd(sfPoint, MPIU_2INT, newOwners, newLocalPoints);CHKERRQ(ierr);
3272       ierr = PetscMalloc1(numSubLeaves,    &slocalPoints);CHKERRQ(ierr);
3273       ierr = PetscMalloc1(numSubLeaves, &sremotePoints);CHKERRQ(ierr);
3274       for (l = 0, sl = 0, ll = 0; l < numLeaves; ++l) {
3275         const PetscInt point    = localPoints[l];
3276         const PetscInt subPoint = DMPlexFilterPoint_Internal(point, firstSubVertex, numSubVertices, subVertices);
3277 
3278         if (subPoint < 0) continue;
3279         if (newLocalPoints[point].rank == rank) {++ll; continue;}
3280         slocalPoints[sl]        = subPoint;
3281         sremotePoints[sl].rank  = newLocalPoints[point].rank;
3282         sremotePoints[sl].index = newLocalPoints[point].index;
3283         if (sremotePoints[sl].rank  < 0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote rank for local point %d", point);
3284         if (sremotePoints[sl].index < 0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote subpoint for local point %d", point);
3285         ++sl;
3286       }
3287       ierr = PetscFree2(newLocalPoints,newOwners);CHKERRQ(ierr);
3288       if (sl + ll != numSubLeaves) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatch in number of subleaves %d + %d != %d", sl, ll, numSubLeaves);
3289       ierr = PetscSFSetGraph(sfPointSub, numSubRoots, sl, slocalPoints, PETSC_OWN_POINTER, sremotePoints, PETSC_OWN_POINTER);CHKERRQ(ierr);
3290     }
3291   }
3292   CHKMEMQ;
3293   /* Cleanup */
3294   if (subvertexIS) {ierr = ISRestoreIndices(subvertexIS, &subVertices);CHKERRQ(ierr);}
3295   ierr = ISDestroy(&subvertexIS);CHKERRQ(ierr);
3296   ierr = PetscFree(subCells);CHKERRQ(ierr);
3297   PetscFunctionReturn(0);
3298 }
3299 
3300 static PetscErrorCode DMPlexCreateCohesiveSubmesh_Interpolated(DM dm, const char labelname[], PetscInt value, DM subdm)
3301 {
3302   DMLabel        label = NULL;
3303   PetscErrorCode ierr;
3304 
3305   PetscFunctionBegin;
3306   if (labelname) {ierr = DMGetLabel(dm, labelname, &label);CHKERRQ(ierr);}
3307   ierr = DMPlexCreateSubmeshGeneric_Interpolated(dm, label, value, PETSC_TRUE, 1, subdm);CHKERRQ(ierr);
3308   PetscFunctionReturn(0);
3309 }
3310 
3311 /*@C
3312   DMPlexCreateCohesiveSubmesh - Extract from a mesh with cohesive cells the hypersurface defined by one face of the cells. Optionally, a Label an be given to restrict the cells.
3313 
3314   Input Parameters:
3315 + dm          - The original mesh
3316 . hasLagrange - The mesh has Lagrange unknowns in the cohesive cells
3317 . label       - A label name, or NULL
3318 - value  - A label value
3319 
3320   Output Parameter:
3321 . subdm - The surface mesh
3322 
3323   Note: This function produces a DMLabel mapping original points in the submesh to their depth. This can be obtained using DMPlexGetSubpointMap().
3324 
3325   Level: developer
3326 
3327 .seealso: DMPlexGetSubpointMap(), DMPlexCreateSubmesh()
3328 @*/
3329 PetscErrorCode DMPlexCreateCohesiveSubmesh(DM dm, PetscBool hasLagrange, const char label[], PetscInt value, DM *subdm)
3330 {
3331   PetscInt       dim, depth;
3332   PetscErrorCode ierr;
3333 
3334   PetscFunctionBegin;
3335   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3336   PetscValidPointer(subdm, 5);
3337   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
3338   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3339   ierr = DMCreate(PetscObjectComm((PetscObject)dm), subdm);CHKERRQ(ierr);
3340   ierr = DMSetType(*subdm, DMPLEX);CHKERRQ(ierr);
3341   ierr = DMSetDimension(*subdm, dim-1);CHKERRQ(ierr);
3342   if (depth == dim) {
3343     ierr = DMPlexCreateCohesiveSubmesh_Interpolated(dm, label, value, *subdm);CHKERRQ(ierr);
3344   } else {
3345     ierr = DMPlexCreateCohesiveSubmesh_Uninterpolated(dm, hasLagrange, label, value, *subdm);CHKERRQ(ierr);
3346   }
3347   PetscFunctionReturn(0);
3348 }
3349 
3350 /*@
3351   DMPlexFilter - Extract a subset of mesh cells defined by a label as a separate mesh
3352 
3353   Input Parameters:
3354 + dm        - The original mesh
3355 . cellLabel - The DMLabel marking cells contained in the new mesh
3356 - value     - The label value to use
3357 
3358   Output Parameter:
3359 . subdm - The new mesh
3360 
3361   Note: This function produces a DMLabel mapping original points in the submesh to their depth. This can be obtained using DMPlexGetSubpointMap().
3362 
3363   Level: developer
3364 
3365 .seealso: DMPlexGetSubpointMap(), DMGetLabel(), DMLabelSetValue()
3366 @*/
3367 PetscErrorCode DMPlexFilter(DM dm, DMLabel cellLabel, PetscInt value, DM *subdm)
3368 {
3369   PetscInt       dim;
3370   PetscErrorCode ierr;
3371 
3372   PetscFunctionBegin;
3373   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3374   PetscValidPointer(subdm, 3);
3375   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
3376   ierr = DMCreate(PetscObjectComm((PetscObject) dm), subdm);CHKERRQ(ierr);
3377   ierr = DMSetType(*subdm, DMPLEX);CHKERRQ(ierr);
3378   ierr = DMSetDimension(*subdm, dim);CHKERRQ(ierr);
3379   /* Extract submesh in place, could be empty on some procs, could have inconsistency if procs do not both extract a shared cell */
3380   ierr = DMPlexCreateSubmeshGeneric_Interpolated(dm, cellLabel, value, PETSC_FALSE, 0, *subdm);CHKERRQ(ierr);
3381   PetscFunctionReturn(0);
3382 }
3383 
3384 /*@
3385   DMPlexGetSubpointMap - Returns a DMLabel with point dimension as values
3386 
3387   Input Parameter:
3388 . dm - The submesh DM
3389 
3390   Output Parameter:
3391 . subpointMap - The DMLabel of all the points from the original mesh in this submesh, or NULL if this is not a submesh
3392 
3393   Level: developer
3394 
3395 .seealso: DMPlexCreateSubmesh(), DMPlexCreateSubpointIS()
3396 @*/
3397 PetscErrorCode DMPlexGetSubpointMap(DM dm, DMLabel *subpointMap)
3398 {
3399   PetscFunctionBegin;
3400   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3401   PetscValidPointer(subpointMap, 2);
3402   *subpointMap = ((DM_Plex*) dm->data)->subpointMap;
3403   PetscFunctionReturn(0);
3404 }
3405 
3406 /*@
3407   DMPlexSetSubpointMap - Sets the DMLabel with point dimension as values
3408 
3409   Input Parameters:
3410 + dm - The submesh DM
3411 - subpointMap - The DMLabel of all the points from the original mesh in this submesh
3412 
3413   Note: Should normally not be called by the user, since it is set in DMPlexCreateSubmesh()
3414 
3415   Level: developer
3416 
3417 .seealso: DMPlexCreateSubmesh(), DMPlexCreateSubpointIS()
3418 @*/
3419 PetscErrorCode DMPlexSetSubpointMap(DM dm, DMLabel subpointMap)
3420 {
3421   DM_Plex       *mesh = (DM_Plex *) dm->data;
3422   DMLabel        tmp;
3423   PetscErrorCode ierr;
3424 
3425   PetscFunctionBegin;
3426   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3427   tmp  = mesh->subpointMap;
3428   mesh->subpointMap = subpointMap;
3429   ++mesh->subpointMap->refct;
3430   ierr = DMLabelDestroy(&tmp);CHKERRQ(ierr);
3431   PetscFunctionReturn(0);
3432 }
3433 
3434 /*@
3435   DMPlexCreateSubpointIS - Creates an IS covering the entire subdm chart with the original points as data
3436 
3437   Input Parameter:
3438 . dm - The submesh DM
3439 
3440   Output Parameter:
3441 . subpointIS - The IS of all the points from the original mesh in this submesh, or NULL if this is not a submesh
3442 
3443   Note: This IS is guaranteed to be sorted by the construction of the submesh
3444 
3445   Level: developer
3446 
3447 .seealso: DMPlexCreateSubmesh(), DMPlexGetSubpointMap()
3448 @*/
3449 PetscErrorCode DMPlexCreateSubpointIS(DM dm, IS *subpointIS)
3450 {
3451   MPI_Comm        comm;
3452   DMLabel         subpointMap;
3453   IS              is;
3454   const PetscInt *opoints;
3455   PetscInt       *points, *depths;
3456   PetscInt        depth, depStart, depEnd, d, pStart, pEnd, p, n, off;
3457   PetscErrorCode  ierr;
3458 
3459   PetscFunctionBegin;
3460   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3461   PetscValidPointer(subpointIS, 2);
3462   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
3463   *subpointIS = NULL;
3464   ierr = DMPlexGetSubpointMap(dm, &subpointMap);CHKERRQ(ierr);
3465   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3466   if (subpointMap && depth >= 0) {
3467     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
3468     if (pStart) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Submeshes must start the point numbering at 0, not %d", pStart);
3469     ierr = DMGetWorkArray(dm, depth+1, PETSC_INT, &depths);CHKERRQ(ierr);
3470     depths[0] = depth;
3471     depths[1] = 0;
3472     for(d = 2; d <= depth; ++d) {depths[d] = depth+1 - d;}
3473     ierr = PetscMalloc1(pEnd, &points);CHKERRQ(ierr);
3474     for(d = 0, off = 0; d <= depth; ++d) {
3475       const PetscInt dep = depths[d];
3476 
3477       ierr = DMPlexGetDepthStratum(dm, dep, &depStart, &depEnd);CHKERRQ(ierr);
3478       ierr = DMLabelGetStratumSize(subpointMap, dep, &n);CHKERRQ(ierr);
3479       if (((d < 2) && (depth > 1)) || (d == 1)) { /* Only check vertices and cells for now since the map is broken for others */
3480         if (n != depEnd-depStart) SETERRQ3(comm, PETSC_ERR_ARG_WRONG, "The number of mapped submesh points %d at depth %d should be %d", n, dep, depEnd-depStart);
3481       } else {
3482         if (!n) {
3483           if (d == 0) {
3484             /* Missing cells */
3485             for(p = 0; p < depEnd-depStart; ++p, ++off) points[off] = -1;
3486           } else {
3487             /* Missing faces */
3488             for(p = 0; p < depEnd-depStart; ++p, ++off) points[off] = PETSC_MAX_INT;
3489           }
3490         }
3491       }
3492       if (n) {
3493         ierr = DMLabelGetStratumIS(subpointMap, dep, &is);CHKERRQ(ierr);
3494         ierr = ISGetIndices(is, &opoints);CHKERRQ(ierr);
3495         for(p = 0; p < n; ++p, ++off) points[off] = opoints[p];
3496         ierr = ISRestoreIndices(is, &opoints);CHKERRQ(ierr);
3497         ierr = ISDestroy(&is);CHKERRQ(ierr);
3498       }
3499     }
3500     ierr = DMRestoreWorkArray(dm, depth+1, PETSC_INT, &depths);CHKERRQ(ierr);
3501     if (off != pEnd) SETERRQ2(comm, PETSC_ERR_ARG_WRONG, "The number of mapped submesh points %d should be %d", off, pEnd);
3502     ierr = ISCreateGeneral(PETSC_COMM_SELF, pEnd, points, PETSC_OWN_POINTER, subpointIS);CHKERRQ(ierr);
3503   }
3504   PetscFunctionReturn(0);
3505 }
3506