xref: /petsc/src/dm/impls/plex/plexsubmesh.c (revision 1ebf93c6b7d760d592de6ebe6cdc0debaa3caf75)
1 #include <petsc/private/dmpleximpl.h>    /*I      "petscdmplex.h"    I*/
2 #include <petsc/private/dmlabelimpl.h>   /*I      "petscdmlabel.h"   I*/
3 #include <petscsf.h>
4 
5 #undef __FUNCT__
6 #define __FUNCT__ "DMPlexMarkBoundaryFaces_Internal"
7 PetscErrorCode DMPlexMarkBoundaryFaces_Internal(DM dm, PetscInt cellHeight, DMLabel label)
8 {
9   PetscInt       fStart, fEnd, f;
10   PetscErrorCode ierr;
11 
12   PetscFunctionBegin;
13   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
14   ierr = DMPlexGetHeightStratum(dm, cellHeight+1, &fStart, &fEnd);CHKERRQ(ierr);
15   for (f = fStart; f < fEnd; ++f) {
16     PetscInt supportSize;
17 
18     ierr = DMPlexGetSupportSize(dm, f, &supportSize);CHKERRQ(ierr);
19     if (supportSize == 1) {ierr = DMLabelSetValue(label, f, 1);CHKERRQ(ierr);}
20   }
21   PetscFunctionReturn(0);
22 }
23 
24 #undef __FUNCT__
25 #define __FUNCT__ "DMPlexMarkBoundaryFaces"
26 /*@
27   DMPlexMarkBoundaryFaces - Mark all faces on the boundary
28 
29   Not Collective
30 
31   Input Parameter:
32 . dm - The original DM
33 
34   Output Parameter:
35 . label - The DMLabel marking boundary faces with value 1
36 
37   Level: developer
38 
39 .seealso: DMLabelCreate(), DMCreateLabel()
40 @*/
41 PetscErrorCode DMPlexMarkBoundaryFaces(DM dm, DMLabel label)
42 {
43   PetscErrorCode ierr;
44 
45   PetscFunctionBegin;
46   ierr = DMPlexMarkBoundaryFaces_Internal(dm, 0, label);CHKERRQ(ierr);
47   PetscFunctionReturn(0);
48 }
49 
50 #undef __FUNCT__
51 #define __FUNCT__ "DMPlexLabelComplete_Internal"
52 PetscErrorCode DMPlexLabelComplete_Internal(DM dm, DMLabel label, PetscBool completeCells)
53 {
54   IS              valueIS;
55   const PetscInt *values;
56   PetscInt        numValues, v, cStart, cEnd;
57   PetscErrorCode  ierr;
58 
59   PetscFunctionBegin;
60   ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
61   ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
62   ierr = DMPlexGetHeightStratum(dm,0,&cStart,&cEnd);CHKERRQ(ierr);
63   ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
64   for (v = 0; v < numValues; ++v) {
65     IS              pointIS;
66     const PetscInt *points;
67     PetscInt        numPoints, p;
68 
69     ierr = DMLabelGetStratumSize(label, values[v], &numPoints);CHKERRQ(ierr);
70     ierr = DMLabelGetStratumIS(label, values[v], &pointIS);CHKERRQ(ierr);
71     ierr = ISGetIndices(pointIS, &points);CHKERRQ(ierr);
72     for (p = 0; p < numPoints; ++p) {
73       PetscInt  q = points[p];
74       PetscInt *closure = NULL;
75       PetscInt  closureSize, c;
76 
77       if (cStart <= q && q < cEnd && !completeCells) { /* skip cells */
78         continue;
79       }
80       ierr = DMPlexGetTransitiveClosure(dm, q, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
81       for (c = 0; c < closureSize*2; c += 2) {
82         ierr = DMLabelSetValue(label, closure[c], values[v]);CHKERRQ(ierr);
83       }
84       ierr = DMPlexRestoreTransitiveClosure(dm, q, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
85     }
86     ierr = ISRestoreIndices(pointIS, &points);CHKERRQ(ierr);
87     ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
88   }
89   ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
90   ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
91   PetscFunctionReturn(0);
92 }
93 
94 #undef __FUNCT__
95 #define __FUNCT__ "DMPlexLabelComplete"
96 /*@
97   DMPlexLabelComplete - Starting with a label marking points on a surface, we add the transitive closure to the surface
98 
99   Input Parameters:
100 + dm - The DM
101 - label - A DMLabel marking the surface points
102 
103   Output Parameter:
104 . label - A DMLabel marking all surface points in the transitive closure
105 
106   Level: developer
107 
108 .seealso: DMPlexLabelCohesiveComplete()
109 @*/
110 PetscErrorCode DMPlexLabelComplete(DM dm, DMLabel label)
111 {
112   PetscErrorCode ierr;
113 
114   PetscFunctionBegin;
115   ierr = DMPlexLabelComplete_Internal(dm, label, PETSC_TRUE);CHKERRQ(ierr);
116   PetscFunctionReturn(0);
117 }
118 
119 #undef __FUNCT__
120 #define __FUNCT__ "DMPlexLabelAddCells"
121 /*@
122   DMPlexLabelAddCells - Starting with a label marking faces on a surface, we add a cell for each face
123 
124   Input Parameters:
125 + dm - The DM
126 - label - A DMLabel marking the surface points
127 
128   Output Parameter:
129 . label - A DMLabel incorporating cells
130 
131   Level: developer
132 
133   Note: The cells allow FEM boundary conditions to be applied using the cell geometry
134 
135 .seealso: DMPlexLabelComplete(), DMPlexLabelCohesiveComplete()
136 @*/
137 PetscErrorCode DMPlexLabelAddCells(DM dm, DMLabel label)
138 {
139   IS              valueIS;
140   const PetscInt *values;
141   PetscInt        numValues, v, cStart, cEnd, cEndInterior;
142   PetscErrorCode  ierr;
143 
144   PetscFunctionBegin;
145   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
146   ierr = DMPlexGetHybridBounds(dm, &cEndInterior, NULL, NULL, NULL);CHKERRQ(ierr);
147   cEnd = cEndInterior < 0 ? cEnd : cEndInterior;
148   ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
149   ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
150   ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
151   for (v = 0; v < numValues; ++v) {
152     IS              pointIS;
153     const PetscInt *points;
154     PetscInt        numPoints, p;
155 
156     ierr = DMLabelGetStratumSize(label, values[v], &numPoints);CHKERRQ(ierr);
157     ierr = DMLabelGetStratumIS(label, values[v], &pointIS);CHKERRQ(ierr);
158     ierr = ISGetIndices(pointIS, &points);CHKERRQ(ierr);
159     for (p = 0; p < numPoints; ++p) {
160       PetscInt *closure = NULL;
161       PetscInt  closureSize, point, cl;
162 
163       ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_FALSE, &closureSize, &closure);CHKERRQ(ierr);
164       for (cl = closureSize-1; cl > 0; --cl) {
165         point = closure[cl*2];
166         if ((point >= cStart) && (point < cEnd)) {ierr = DMLabelSetValue(label, point, values[v]);CHKERRQ(ierr); break;}
167       }
168       ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_FALSE, &closureSize, &closure);CHKERRQ(ierr);
169     }
170     ierr = ISRestoreIndices(pointIS, &points);CHKERRQ(ierr);
171     ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
172   }
173   ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
174   ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
175   PetscFunctionReturn(0);
176 }
177 
178 #undef __FUNCT__
179 #define __FUNCT__ "DMPlexLabelClearCells"
180 /*@
181   DMPlexLabelClearCells - Remove cells from a label
182 
183   Input Parameters:
184 + dm - The DM
185 - label - A DMLabel marking surface points and their adjacent cells
186 
187   Output Parameter:
188 . label - A DMLabel without cells
189 
190   Level: developer
191 
192   Note: This undoes DMPlexLabelAddCells()
193 
194 .seealso: DMPlexLabelComplete(), DMPlexLabelCohesiveComplete(), DMPlexLabelAddCells()
195 @*/
196 PetscErrorCode DMPlexLabelClearCells(DM dm, DMLabel label)
197 {
198   IS              valueIS;
199   const PetscInt *values;
200   PetscInt        numValues, v, cStart, cEnd, cEndInterior;
201   PetscErrorCode  ierr;
202 
203   PetscFunctionBegin;
204   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
205   ierr = DMPlexGetHybridBounds(dm, &cEndInterior, NULL, NULL, NULL);CHKERRQ(ierr);
206   cEnd = cEndInterior < 0 ? cEnd : cEndInterior;
207   ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
208   ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
209   ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
210   for (v = 0; v < numValues; ++v) {
211     IS              pointIS;
212     const PetscInt *points;
213     PetscInt        numPoints, p;
214 
215     ierr = DMLabelGetStratumSize(label, values[v], &numPoints);CHKERRQ(ierr);
216     ierr = DMLabelGetStratumIS(label, values[v], &pointIS);CHKERRQ(ierr);
217     ierr = ISGetIndices(pointIS, &points);CHKERRQ(ierr);
218     for (p = 0; p < numPoints; ++p) {
219       PetscInt point = points[p];
220 
221       if (point >= cStart && point < cEnd) {
222         ierr = DMLabelClearValue(label,point,values[v]);CHKERRQ(ierr);
223       }
224     }
225     ierr = ISRestoreIndices(pointIS, &points);CHKERRQ(ierr);
226     ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
227   }
228   ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
229   ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
230   PetscFunctionReturn(0);
231 }
232 
233 #undef __FUNCT__
234 #define __FUNCT__ "DMPlexShiftPointSetUp_Internal"
235 /* take (oldEnd, added) pairs, ordered by height and convert them to (oldstart, newstart) pairs, ordered by ascending
236  * index (skipping first, which is (0,0)) */
237 PETSC_STATIC_INLINE PetscErrorCode DMPlexShiftPointSetUp_Internal(PetscInt depth, PetscInt depthShift[])
238 {
239   PetscInt d, off = 0;
240 
241   PetscFunctionBegin;
242   /* sort by (oldend): yes this is an O(n^2) sort, we expect depth <= 3 */
243   for (d = 0; d < depth; d++) {
244     PetscInt firstd = d;
245     PetscInt firstStart = depthShift[2*d];
246     PetscInt e;
247 
248     for (e = d+1; e <= depth; e++) {
249       if (depthShift[2*e] < firstStart) {
250         firstd = e;
251         firstStart = depthShift[2*d];
252       }
253     }
254     if (firstd != d) {
255       PetscInt swap[2];
256 
257       e = firstd;
258       swap[0] = depthShift[2*d];
259       swap[1] = depthShift[2*d+1];
260       depthShift[2*d]   = depthShift[2*e];
261       depthShift[2*d+1] = depthShift[2*e+1];
262       depthShift[2*e]   = swap[0];
263       depthShift[2*e+1] = swap[1];
264     }
265   }
266   /* convert (oldstart, added) to (oldstart, newstart) */
267   for (d = 0; d <= depth; d++) {
268     off += depthShift[2*d+1];
269     depthShift[2*d+1] = depthShift[2*d] + off;
270   }
271   PetscFunctionReturn(0);
272 }
273 
274 #undef __FUNCT__
275 #define __FUNCT__ "DMPlexShiftPoint_Internal"
276 /* depthShift is a list of (old, new) pairs */
277 PETSC_STATIC_INLINE PetscInt DMPlexShiftPoint_Internal(PetscInt p, PetscInt depth, PetscInt depthShift[])
278 {
279   PetscInt d;
280   PetscInt newOff = 0;
281 
282   for (d = 0; d <= depth; d++) {
283     if (p < depthShift[2*d]) return p + newOff;
284     else newOff = depthShift[2*d+1] - depthShift[2*d];
285   }
286   return p + newOff;
287 }
288 
289 #undef __FUNCT__
290 #define __FUNCT__ "DMPlexShiftPointInverse_Internal"
291 /* depthShift is a list of (old, new) pairs */
292 PETSC_STATIC_INLINE PetscInt DMPlexShiftPointInverse_Internal(PetscInt p, PetscInt depth, PetscInt depthShift[])
293 {
294   PetscInt d;
295   PetscInt newOff = 0;
296 
297   for (d = 0; d <= depth; d++) {
298     if (p < depthShift[2*d+1]) return p + newOff;
299     else newOff = depthShift[2*d] - depthShift[2*d+1];
300   }
301   return p + newOff;
302 }
303 
304 #undef __FUNCT__
305 #define __FUNCT__ "DMPlexShiftSizes_Internal"
306 static PetscErrorCode DMPlexShiftSizes_Internal(DM dm, PetscInt depthShift[], DM dmNew)
307 {
308   PetscInt       depth = 0, d, pStart, pEnd, p;
309   PetscErrorCode ierr;
310 
311   PetscFunctionBegin;
312   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
313   if (depth < 0) PetscFunctionReturn(0);
314   /* Step 1: Expand chart */
315   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
316   pEnd = DMPlexShiftPoint_Internal(pEnd,depth,depthShift);
317   ierr = DMPlexSetChart(dmNew, pStart, pEnd);CHKERRQ(ierr);
318   /* Step 2: Set cone and support sizes */
319   for (d = 0; d <= depth; ++d) {
320     ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
321     for (p = pStart; p < pEnd; ++p) {
322       PetscInt newp = DMPlexShiftPoint_Internal(p, depth, depthShift);
323       PetscInt size;
324 
325       ierr = DMPlexGetConeSize(dm, p, &size);CHKERRQ(ierr);
326       ierr = DMPlexSetConeSize(dmNew, newp, size);CHKERRQ(ierr);
327       ierr = DMPlexGetSupportSize(dm, p, &size);CHKERRQ(ierr);
328       ierr = DMPlexSetSupportSize(dmNew, newp, size);CHKERRQ(ierr);
329     }
330   }
331   PetscFunctionReturn(0);
332 }
333 
334 #undef __FUNCT__
335 #define __FUNCT__ "DMPlexShiftPoints_Internal"
336 static PetscErrorCode DMPlexShiftPoints_Internal(DM dm, PetscInt depthShift[], DM dmNew)
337 {
338   PetscInt      *newpoints;
339   PetscInt       depth = 0, maxConeSize, maxSupportSize, maxConeSizeNew, maxSupportSizeNew, pStart, pEnd, p;
340   PetscErrorCode ierr;
341 
342   PetscFunctionBegin;
343   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
344   if (depth < 0) PetscFunctionReturn(0);
345   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);CHKERRQ(ierr);
346   ierr = DMPlexGetMaxSizes(dmNew, &maxConeSizeNew, &maxSupportSizeNew);CHKERRQ(ierr);
347   ierr = PetscMalloc1(PetscMax(PetscMax(maxConeSize, maxSupportSize), PetscMax(maxConeSizeNew, maxSupportSizeNew)),&newpoints);CHKERRQ(ierr);
348   /* Step 5: Set cones and supports */
349   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
350   for (p = pStart; p < pEnd; ++p) {
351     const PetscInt *points = NULL, *orientations = NULL;
352     PetscInt        size,sizeNew, i, newp = DMPlexShiftPoint_Internal(p, depth, depthShift);
353 
354     ierr = DMPlexGetConeSize(dm, p, &size);CHKERRQ(ierr);
355     ierr = DMPlexGetCone(dm, p, &points);CHKERRQ(ierr);
356     ierr = DMPlexGetConeOrientation(dm, p, &orientations);CHKERRQ(ierr);
357     for (i = 0; i < size; ++i) {
358       newpoints[i] = DMPlexShiftPoint_Internal(points[i], depth, depthShift);
359     }
360     ierr = DMPlexSetCone(dmNew, newp, newpoints);CHKERRQ(ierr);
361     ierr = DMPlexSetConeOrientation(dmNew, newp, orientations);CHKERRQ(ierr);
362     ierr = DMPlexGetSupportSize(dm, p, &size);CHKERRQ(ierr);
363     ierr = DMPlexGetSupportSize(dmNew, newp, &sizeNew);CHKERRQ(ierr);
364     ierr = DMPlexGetSupport(dm, p, &points);CHKERRQ(ierr);
365     for (i = 0; i < size; ++i) {
366       newpoints[i] = DMPlexShiftPoint_Internal(points[i], depth, depthShift);
367     }
368     for (i = size; i < sizeNew; ++i) newpoints[i] = 0;
369     ierr = DMPlexSetSupport(dmNew, newp, newpoints);CHKERRQ(ierr);
370   }
371   ierr = PetscFree(newpoints);CHKERRQ(ierr);
372   PetscFunctionReturn(0);
373 }
374 
375 #undef __FUNCT__
376 #define __FUNCT__ "DMPlexShiftCoordinates_Internal"
377 static PetscErrorCode DMPlexShiftCoordinates_Internal(DM dm, PetscInt depthShift[], DM dmNew)
378 {
379   PetscSection   coordSection, newCoordSection;
380   Vec            coordinates, newCoordinates;
381   PetscScalar   *coords, *newCoords;
382   PetscInt       coordSize, sStart, sEnd;
383   PetscInt       dim, depth = 0, cStart, cEnd, cStartNew, cEndNew, c, vStart, vEnd, vStartNew, vEndNew, v;
384   PetscBool      hasCells;
385   PetscErrorCode ierr;
386 
387   PetscFunctionBegin;
388   ierr = DMGetCoordinateDim(dm, &dim);CHKERRQ(ierr);
389   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
390   /* Step 8: Convert coordinates */
391   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
392   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
393   ierr = DMPlexGetDepthStratum(dmNew, 0, &vStartNew, &vEndNew);CHKERRQ(ierr);
394   ierr = DMPlexGetHeightStratum(dmNew, 0, &cStartNew, &cEndNew);CHKERRQ(ierr);
395   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
396   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &newCoordSection);CHKERRQ(ierr);
397   ierr = PetscSectionSetNumFields(newCoordSection, 1);CHKERRQ(ierr);
398   ierr = PetscSectionSetFieldComponents(newCoordSection, 0, dim);CHKERRQ(ierr);
399   ierr = PetscSectionGetChart(coordSection, &sStart, &sEnd);CHKERRQ(ierr);
400   hasCells = sStart == cStart ? PETSC_TRUE : PETSC_FALSE;
401   ierr = PetscSectionSetChart(newCoordSection, hasCells ? cStartNew : vStartNew, vEndNew);CHKERRQ(ierr);
402   if (hasCells) {
403     for (c = cStart; c < cEnd; ++c) {
404       PetscInt cNew = DMPlexShiftPoint_Internal(c, depth, depthShift), dof;
405 
406       ierr = PetscSectionGetDof(coordSection, c, &dof);CHKERRQ(ierr);
407       ierr = PetscSectionSetDof(newCoordSection, cNew, dof);CHKERRQ(ierr);
408       ierr = PetscSectionSetFieldDof(newCoordSection, cNew, 0, dof);CHKERRQ(ierr);
409     }
410   }
411   for (v = vStartNew; v < vEndNew; ++v) {
412     ierr = PetscSectionSetDof(newCoordSection, v, dim);CHKERRQ(ierr);
413     ierr = PetscSectionSetFieldDof(newCoordSection, v, 0, dim);CHKERRQ(ierr);
414   }
415   ierr = PetscSectionSetUp(newCoordSection);CHKERRQ(ierr);
416   ierr = DMSetCoordinateSection(dmNew, PETSC_DETERMINE, newCoordSection);CHKERRQ(ierr);
417   ierr = PetscSectionGetStorageSize(newCoordSection, &coordSize);CHKERRQ(ierr);
418   ierr = VecCreate(PetscObjectComm((PetscObject)dm), &newCoordinates);CHKERRQ(ierr);
419   ierr = PetscObjectSetName((PetscObject) newCoordinates, "coordinates");CHKERRQ(ierr);
420   ierr = VecSetSizes(newCoordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
421   ierr = VecSetBlockSize(newCoordinates, dim);CHKERRQ(ierr);
422   ierr = VecSetType(newCoordinates,VECSTANDARD);CHKERRQ(ierr);
423   ierr = DMSetCoordinatesLocal(dmNew, newCoordinates);CHKERRQ(ierr);
424   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
425   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
426   ierr = VecGetArray(newCoordinates, &newCoords);CHKERRQ(ierr);
427   if (hasCells) {
428     for (c = cStart; c < cEnd; ++c) {
429       PetscInt cNew = DMPlexShiftPoint_Internal(c, depth, depthShift), dof, off, noff, d;
430 
431       ierr = PetscSectionGetDof(coordSection, c, &dof);CHKERRQ(ierr);
432       ierr = PetscSectionGetOffset(coordSection, c, &off);CHKERRQ(ierr);
433       ierr = PetscSectionGetOffset(newCoordSection, cNew, &noff);CHKERRQ(ierr);
434       for (d = 0; d < dof; ++d) newCoords[noff+d] = coords[off+d];
435     }
436   }
437   for (v = vStart; v < vEnd; ++v) {
438     PetscInt dof, off, noff, d;
439 
440     ierr = PetscSectionGetDof(coordSection, v, &dof);CHKERRQ(ierr);
441     ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
442     ierr = PetscSectionGetOffset(newCoordSection, DMPlexShiftPoint_Internal(v, depth, depthShift), &noff);CHKERRQ(ierr);
443     for (d = 0; d < dof; ++d) newCoords[noff+d] = coords[off+d];
444   }
445   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
446   ierr = VecRestoreArray(newCoordinates, &newCoords);CHKERRQ(ierr);
447   ierr = VecDestroy(&newCoordinates);CHKERRQ(ierr);
448   ierr = PetscSectionDestroy(&newCoordSection);CHKERRQ(ierr);
449   PetscFunctionReturn(0);
450 }
451 
452 #undef __FUNCT__
453 #define __FUNCT__ "DMPlexShiftSF_Internal"
454 static PetscErrorCode DMPlexShiftSF_Internal(DM dm, PetscInt depthShift[], DM dmNew)
455 {
456   PetscInt           depth = 0;
457   PetscSF            sfPoint, sfPointNew;
458   const PetscSFNode *remotePoints;
459   PetscSFNode       *gremotePoints;
460   const PetscInt    *localPoints;
461   PetscInt          *glocalPoints, *newLocation, *newRemoteLocation;
462   PetscInt           numRoots, numLeaves, l, pStart, pEnd, totShift = 0;
463   PetscErrorCode     ierr;
464 
465   PetscFunctionBegin;
466   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
467   /* Step 9: Convert pointSF */
468   ierr = DMGetPointSF(dm, &sfPoint);CHKERRQ(ierr);
469   ierr = DMGetPointSF(dmNew, &sfPointNew);CHKERRQ(ierr);
470   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
471   ierr = PetscSFGetGraph(sfPoint, &numRoots, &numLeaves, &localPoints, &remotePoints);CHKERRQ(ierr);
472   totShift = DMPlexShiftPoint_Internal(pEnd,depth,depthShift) - pEnd;
473   if (numRoots >= 0) {
474     ierr = PetscMalloc2(numRoots,&newLocation,pEnd-pStart,&newRemoteLocation);CHKERRQ(ierr);
475     for (l=0; l<numRoots; l++) newLocation[l] = DMPlexShiftPoint_Internal(l, depth, depthShift);
476     ierr = PetscSFBcastBegin(sfPoint, MPIU_INT, newLocation, newRemoteLocation);CHKERRQ(ierr);
477     ierr = PetscSFBcastEnd(sfPoint, MPIU_INT, newLocation, newRemoteLocation);CHKERRQ(ierr);
478     ierr = PetscMalloc1(numLeaves,    &glocalPoints);CHKERRQ(ierr);
479     ierr = PetscMalloc1(numLeaves, &gremotePoints);CHKERRQ(ierr);
480     for (l = 0; l < numLeaves; ++l) {
481       glocalPoints[l]        = DMPlexShiftPoint_Internal(localPoints[l], depth, depthShift);
482       gremotePoints[l].rank  = remotePoints[l].rank;
483       gremotePoints[l].index = newRemoteLocation[localPoints[l]];
484     }
485     ierr = PetscFree2(newLocation,newRemoteLocation);CHKERRQ(ierr);
486     ierr = PetscSFSetGraph(sfPointNew, numRoots + totShift, numLeaves, glocalPoints, PETSC_OWN_POINTER, gremotePoints, PETSC_OWN_POINTER);CHKERRQ(ierr);
487   }
488   PetscFunctionReturn(0);
489 }
490 
491 #undef __FUNCT__
492 #define __FUNCT__ "DMPlexShiftLabels_Internal"
493 static PetscErrorCode DMPlexShiftLabels_Internal(DM dm, PetscInt depthShift[], DM dmNew)
494 {
495   PetscSF            sfPoint;
496   DMLabel            vtkLabel, ghostLabel;
497   const PetscSFNode *leafRemote;
498   const PetscInt    *leafLocal;
499   PetscInt           depth = 0, numLeaves, numLabels, l, cStart, cEnd, c, fStart, fEnd, f;
500   PetscMPIInt        rank;
501   PetscErrorCode     ierr;
502 
503   PetscFunctionBegin;
504   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
505   /* Step 10: Convert labels */
506   ierr = DMGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
507   for (l = 0; l < numLabels; ++l) {
508     DMLabel         label, newlabel;
509     const char     *lname;
510     PetscBool       isDepth;
511     IS              valueIS;
512     const PetscInt *values;
513     PetscInt        numValues, val;
514 
515     ierr = DMGetLabelName(dm, l, &lname);CHKERRQ(ierr);
516     ierr = PetscStrcmp(lname, "depth", &isDepth);CHKERRQ(ierr);
517     if (isDepth) continue;
518     ierr = DMCreateLabel(dmNew, lname);CHKERRQ(ierr);
519     ierr = DMGetLabel(dm, lname, &label);CHKERRQ(ierr);
520     ierr = DMGetLabel(dmNew, lname, &newlabel);CHKERRQ(ierr);
521     ierr = DMLabelGetDefaultValue(label,&val);CHKERRQ(ierr);
522     ierr = DMLabelSetDefaultValue(newlabel,val);CHKERRQ(ierr);
523     ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
524     ierr = ISGetLocalSize(valueIS, &numValues);CHKERRQ(ierr);
525     ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
526     for (val = 0; val < numValues; ++val) {
527       IS              pointIS;
528       const PetscInt *points;
529       PetscInt        numPoints, p;
530 
531       ierr = DMLabelGetStratumIS(label, values[val], &pointIS);CHKERRQ(ierr);
532       ierr = ISGetLocalSize(pointIS, &numPoints);CHKERRQ(ierr);
533       ierr = ISGetIndices(pointIS, &points);CHKERRQ(ierr);
534       for (p = 0; p < numPoints; ++p) {
535         const PetscInt newpoint = DMPlexShiftPoint_Internal(points[p], depth, depthShift);
536 
537         ierr = DMLabelSetValue(newlabel, newpoint, values[val]);CHKERRQ(ierr);
538       }
539       ierr = ISRestoreIndices(pointIS, &points);CHKERRQ(ierr);
540       ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
541     }
542     ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
543     ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
544   }
545   /* Step 11: Make label for output (vtk) and to mark ghost points (ghost) */
546   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
547   ierr = DMGetPointSF(dm, &sfPoint);CHKERRQ(ierr);
548   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
549   ierr = PetscSFGetGraph(sfPoint, NULL, &numLeaves, &leafLocal, &leafRemote);CHKERRQ(ierr);
550   ierr = DMCreateLabel(dmNew, "vtk");CHKERRQ(ierr);
551   ierr = DMCreateLabel(dmNew, "ghost");CHKERRQ(ierr);
552   ierr = DMGetLabel(dmNew, "vtk", &vtkLabel);CHKERRQ(ierr);
553   ierr = DMGetLabel(dmNew, "ghost", &ghostLabel);CHKERRQ(ierr);
554   for (l = 0, c = cStart; l < numLeaves && c < cEnd; ++l, ++c) {
555     for (; c < leafLocal[l] && c < cEnd; ++c) {
556       ierr = DMLabelSetValue(vtkLabel, c, 1);CHKERRQ(ierr);
557     }
558     if (leafLocal[l] >= cEnd) break;
559     if (leafRemote[l].rank == rank) {
560       ierr = DMLabelSetValue(vtkLabel, c, 1);CHKERRQ(ierr);
561     } else {
562       ierr = DMLabelSetValue(ghostLabel, c, 2);CHKERRQ(ierr);
563     }
564   }
565   for (; c < cEnd; ++c) {
566     ierr = DMLabelSetValue(vtkLabel, c, 1);CHKERRQ(ierr);
567   }
568   if (0) {
569     ierr = DMLabelView(vtkLabel, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
570   }
571   ierr = DMPlexGetHeightStratum(dmNew, 1, &fStart, &fEnd);CHKERRQ(ierr);
572   for (f = fStart; f < fEnd; ++f) {
573     PetscInt numCells;
574 
575     ierr = DMPlexGetSupportSize(dmNew, f, &numCells);CHKERRQ(ierr);
576     if (numCells < 2) {
577       ierr = DMLabelSetValue(ghostLabel, f, 1);CHKERRQ(ierr);
578     } else {
579       const PetscInt *cells = NULL;
580       PetscInt        vA, vB;
581 
582       ierr = DMPlexGetSupport(dmNew, f, &cells);CHKERRQ(ierr);
583       ierr = DMLabelGetValue(vtkLabel, cells[0], &vA);CHKERRQ(ierr);
584       ierr = DMLabelGetValue(vtkLabel, cells[1], &vB);CHKERRQ(ierr);
585       if (!vA && !vB) {ierr = DMLabelSetValue(ghostLabel, f, 1);CHKERRQ(ierr);}
586     }
587   }
588   if (0) {
589     ierr = DMLabelView(ghostLabel, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
590   }
591   PetscFunctionReturn(0);
592 }
593 
594 #undef __FUNCT__
595 #define __FUNCT__ "DMPlexShiftTree_Internal"
596 static PetscErrorCode DMPlexShiftTree_Internal(DM dm, PetscInt depthShift[], DM dmNew)
597 {
598   DM             refTree;
599   PetscSection   pSec;
600   PetscInt       *parents, *childIDs;
601   PetscErrorCode ierr;
602 
603   PetscFunctionBegin;
604   ierr = DMPlexGetReferenceTree(dm,&refTree);CHKERRQ(ierr);
605   ierr = DMPlexSetReferenceTree(dmNew,refTree);CHKERRQ(ierr);
606   ierr = DMPlexGetTree(dm,&pSec,&parents,&childIDs,NULL,NULL);CHKERRQ(ierr);
607   if (pSec) {
608     PetscInt p, pStart, pEnd, *parentsShifted, pStartShifted, pEndShifted, depth;
609     PetscInt *childIDsShifted;
610     PetscSection pSecShifted;
611 
612     ierr = PetscSectionGetChart(pSec,&pStart,&pEnd);CHKERRQ(ierr);
613     ierr = DMPlexGetDepth(dm,&depth);CHKERRQ(ierr);
614     pStartShifted = DMPlexShiftPoint_Internal(pStart,depth,depthShift);
615     pEndShifted   = DMPlexShiftPoint_Internal(pEnd,depth,depthShift);
616     ierr = PetscMalloc2(pEndShifted - pStartShifted,&parentsShifted,pEndShifted-pStartShifted,&childIDsShifted);CHKERRQ(ierr);
617     ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dmNew),&pSecShifted);CHKERRQ(ierr);
618     ierr = PetscSectionSetChart(pSecShifted,pStartShifted,pEndShifted);CHKERRQ(ierr);
619     for (p = pStartShifted; p < pEndShifted; p++) {
620       /* start off assuming no children */
621       ierr = PetscSectionSetDof(pSecShifted,p,0);CHKERRQ(ierr);
622     }
623     for (p = pStart; p < pEnd; p++) {
624       PetscInt dof;
625       PetscInt pNew = DMPlexShiftPoint_Internal(p,depth,depthShift);
626 
627       ierr = PetscSectionGetDof(pSec,p,&dof);CHKERRQ(ierr);
628       ierr = PetscSectionSetDof(pSecShifted,pNew,dof);CHKERRQ(ierr);
629     }
630     ierr = PetscSectionSetUp(pSecShifted);CHKERRQ(ierr);
631     for (p = pStart; p < pEnd; p++) {
632       PetscInt dof;
633       PetscInt pNew = DMPlexShiftPoint_Internal(p,depth,depthShift);
634 
635       ierr = PetscSectionGetDof(pSec,p,&dof);CHKERRQ(ierr);
636       if (dof) {
637         PetscInt off, offNew;
638 
639         ierr = PetscSectionGetOffset(pSec,p,&off);CHKERRQ(ierr);
640         ierr = PetscSectionGetOffset(pSecShifted,pNew,&offNew);CHKERRQ(ierr);
641         parentsShifted[offNew] = DMPlexShiftPoint_Internal(parents[off],depth,depthShift);
642         childIDsShifted[offNew] = childIDs[off];
643       }
644     }
645     ierr = DMPlexSetTree(dmNew,pSecShifted,parentsShifted,childIDsShifted);CHKERRQ(ierr);
646     ierr = PetscFree2(parentsShifted,childIDsShifted);CHKERRQ(ierr);
647     ierr = PetscSectionDestroy(&pSecShifted);CHKERRQ(ierr);
648   }
649   PetscFunctionReturn(0);
650 }
651 
652 #undef __FUNCT__
653 #define __FUNCT__ "DMPlexConstructGhostCells_Internal"
654 static PetscErrorCode DMPlexConstructGhostCells_Internal(DM dm, DMLabel label, PetscInt *numGhostCells, DM gdm)
655 {
656   PetscSF         sf;
657   IS              valueIS;
658   const PetscInt *values, *leaves;
659   PetscInt       *depthShift;
660   PetscInt        d, depth = 0, nleaves, loc, Ng, numFS, fs, fStart, fEnd, ghostCell, cEnd, c;
661   PetscErrorCode  ierr;
662 
663   PetscFunctionBegin;
664   ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
665   ierr = PetscSFGetGraph(sf, NULL, &nleaves, &leaves, NULL);CHKERRQ(ierr);
666   nleaves = PetscMax(0, nleaves);
667   ierr = DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd);CHKERRQ(ierr);
668   /* Count ghost cells */
669   ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
670   ierr = ISGetLocalSize(valueIS, &numFS);CHKERRQ(ierr);
671   ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
672   Ng   = 0;
673   for (fs = 0; fs < numFS; ++fs) {
674     IS              faceIS;
675     const PetscInt *faces;
676     PetscInt        numFaces, f, numBdFaces = 0;
677 
678     ierr = DMLabelGetStratumIS(label, values[fs], &faceIS);CHKERRQ(ierr);
679     ierr = ISGetLocalSize(faceIS, &numFaces);CHKERRQ(ierr);
680     ierr = ISGetIndices(faceIS, &faces);CHKERRQ(ierr);
681     for (f = 0; f < numFaces; ++f) {
682       PetscInt numChildren;
683 
684       ierr = PetscFindInt(faces[f], nleaves, leaves, &loc);CHKERRQ(ierr);
685       ierr = DMPlexGetTreeChildren(dm,faces[f],&numChildren,NULL);CHKERRQ(ierr);
686       /* non-local and ancestors points don't get to register ghosts */
687       if (loc >= 0 || numChildren) continue;
688       if ((faces[f] >= fStart) && (faces[f] < fEnd)) ++numBdFaces;
689     }
690     Ng += numBdFaces;
691     ierr = ISDestroy(&faceIS);CHKERRQ(ierr);
692   }
693   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
694   ierr = PetscMalloc1(2*(depth+1), &depthShift);CHKERRQ(ierr);
695   for (d = 0; d <= depth; d++) {
696     PetscInt dEnd;
697 
698     ierr = DMPlexGetDepthStratum(dm,d,NULL,&dEnd);CHKERRQ(ierr);
699     depthShift[2*d]   = dEnd;
700     depthShift[2*d+1] = 0;
701   }
702   if (depth >= 0) depthShift[2*depth+1] = Ng;
703   ierr = DMPlexShiftPointSetUp_Internal(depth,depthShift);CHKERRQ(ierr);
704   ierr = DMPlexShiftSizes_Internal(dm, depthShift, gdm);CHKERRQ(ierr);
705   /* Step 3: Set cone/support sizes for new points */
706   ierr = DMPlexGetHeightStratum(dm, 0, NULL, &cEnd);CHKERRQ(ierr);
707   ierr = DMPlexSetHybridBounds(gdm, cEnd, PETSC_DETERMINE, PETSC_DETERMINE, PETSC_DETERMINE);CHKERRQ(ierr);
708   for (c = cEnd; c < cEnd + Ng; ++c) {
709     ierr = DMPlexSetConeSize(gdm, c, 1);CHKERRQ(ierr);
710   }
711   for (fs = 0; fs < numFS; ++fs) {
712     IS              faceIS;
713     const PetscInt *faces;
714     PetscInt        numFaces, f;
715 
716     ierr = DMLabelGetStratumIS(label, values[fs], &faceIS);CHKERRQ(ierr);
717     ierr = ISGetLocalSize(faceIS, &numFaces);CHKERRQ(ierr);
718     ierr = ISGetIndices(faceIS, &faces);CHKERRQ(ierr);
719     for (f = 0; f < numFaces; ++f) {
720       PetscInt size, numChildren;
721 
722       ierr = PetscFindInt(faces[f], nleaves, leaves, &loc);CHKERRQ(ierr);
723       ierr = DMPlexGetTreeChildren(dm,faces[f],&numChildren,NULL);CHKERRQ(ierr);
724       if (loc >= 0 || numChildren) continue;
725       if ((faces[f] < fStart) || (faces[f] >= fEnd)) continue;
726       ierr = DMPlexGetSupportSize(dm, faces[f], &size);CHKERRQ(ierr);
727       if (size != 1) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "DM has boundary face %d with %d support cells", faces[f], size);
728       ierr = DMPlexSetSupportSize(gdm, faces[f] + Ng, 2);CHKERRQ(ierr);
729     }
730     ierr = ISRestoreIndices(faceIS, &faces);CHKERRQ(ierr);
731     ierr = ISDestroy(&faceIS);CHKERRQ(ierr);
732   }
733   /* Step 4: Setup ghosted DM */
734   ierr = DMSetUp(gdm);CHKERRQ(ierr);
735   ierr = DMPlexShiftPoints_Internal(dm, depthShift, gdm);CHKERRQ(ierr);
736   /* Step 6: Set cones and supports for new points */
737   ghostCell = cEnd;
738   for (fs = 0; fs < numFS; ++fs) {
739     IS              faceIS;
740     const PetscInt *faces;
741     PetscInt        numFaces, f;
742 
743     ierr = DMLabelGetStratumIS(label, values[fs], &faceIS);CHKERRQ(ierr);
744     ierr = ISGetLocalSize(faceIS, &numFaces);CHKERRQ(ierr);
745     ierr = ISGetIndices(faceIS, &faces);CHKERRQ(ierr);
746     for (f = 0; f < numFaces; ++f) {
747       PetscInt newFace = faces[f] + Ng, numChildren;
748 
749       ierr = PetscFindInt(faces[f], nleaves, leaves, &loc);CHKERRQ(ierr);
750       ierr = DMPlexGetTreeChildren(dm,faces[f],&numChildren,NULL);CHKERRQ(ierr);
751       if (loc >= 0 || numChildren) continue;
752       if ((faces[f] < fStart) || (faces[f] >= fEnd)) continue;
753       ierr = DMPlexSetCone(gdm, ghostCell, &newFace);CHKERRQ(ierr);
754       ierr = DMPlexInsertSupport(gdm, newFace, 1, ghostCell);CHKERRQ(ierr);
755       ++ghostCell;
756     }
757     ierr = ISRestoreIndices(faceIS, &faces);CHKERRQ(ierr);
758     ierr = ISDestroy(&faceIS);CHKERRQ(ierr);
759   }
760   ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
761   ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
762   /* Step 7: Stratify */
763   ierr = DMPlexStratify(gdm);CHKERRQ(ierr);
764   ierr = DMPlexShiftCoordinates_Internal(dm, depthShift, gdm);CHKERRQ(ierr);
765   ierr = DMPlexShiftSF_Internal(dm, depthShift, gdm);CHKERRQ(ierr);
766   ierr = DMPlexShiftLabels_Internal(dm, depthShift, gdm);CHKERRQ(ierr);
767   ierr = DMPlexShiftTree_Internal(dm, depthShift, gdm);CHKERRQ(ierr);
768   ierr = PetscFree(depthShift);CHKERRQ(ierr);
769   /* Step 7: Periodicity */
770   if (dm->maxCell) {
771     const PetscReal *maxCell, *L;
772     const DMBoundaryType *bd;
773     ierr = DMGetPeriodicity(dm,  &maxCell, &L, &bd);CHKERRQ(ierr);
774     ierr = DMSetPeriodicity(gdm,  maxCell,  L,  bd);CHKERRQ(ierr);
775   }
776   if (numGhostCells) *numGhostCells = Ng;
777   PetscFunctionReturn(0);
778 }
779 
780 #undef __FUNCT__
781 #define __FUNCT__ "DMPlexConstructGhostCells"
782 /*@C
783   DMPlexConstructGhostCells - Construct ghost cells which connect to every boundary face
784 
785   Collective on dm
786 
787   Input Parameters:
788 + dm - The original DM
789 - labelName - The label specifying the boundary faces, or "Face Sets" if this is NULL
790 
791   Output Parameters:
792 + numGhostCells - The number of ghost cells added to the DM
793 - dmGhosted - The new DM
794 
795   Note: If no label exists of that name, one will be created marking all boundary faces
796 
797   Level: developer
798 
799 .seealso: DMCreate()
800 @*/
801 PetscErrorCode DMPlexConstructGhostCells(DM dm, const char labelName[], PetscInt *numGhostCells, DM *dmGhosted)
802 {
803   DM             gdm;
804   DMLabel        label;
805   const char    *name = labelName ? labelName : "Face Sets";
806   PetscInt       dim;
807   PetscBool      flag;
808   PetscErrorCode ierr;
809 
810   PetscFunctionBegin;
811   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
812   if (numGhostCells) PetscValidPointer(numGhostCells, 3);
813   PetscValidPointer(dmGhosted, 4);
814   ierr = DMCreate(PetscObjectComm((PetscObject)dm), &gdm);CHKERRQ(ierr);
815   ierr = DMSetType(gdm, DMPLEX);CHKERRQ(ierr);
816   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
817   ierr = DMSetDimension(gdm, dim);CHKERRQ(ierr);
818   ierr = DMPlexGetAdjacencyUseCone(dm, &flag);CHKERRQ(ierr);
819   ierr = DMPlexSetAdjacencyUseCone(gdm, flag);CHKERRQ(ierr);
820   ierr = DMPlexGetAdjacencyUseClosure(dm, &flag);CHKERRQ(ierr);
821   ierr = DMPlexSetAdjacencyUseClosure(gdm, flag);CHKERRQ(ierr);
822   ierr = DMGetLabel(dm, name, &label);CHKERRQ(ierr);
823   if (!label) {
824     /* Get label for boundary faces */
825     ierr = DMCreateLabel(dm, name);CHKERRQ(ierr);
826     ierr = DMGetLabel(dm, name, &label);CHKERRQ(ierr);
827     ierr = DMPlexMarkBoundaryFaces(dm, label);CHKERRQ(ierr);
828   }
829   ierr = DMPlexConstructGhostCells_Internal(dm, label, numGhostCells, gdm);CHKERRQ(ierr);
830   ierr = DMCopyBoundary(dm, gdm);CHKERRQ(ierr);
831   *dmGhosted = gdm;
832   PetscFunctionReturn(0);
833 }
834 
835 #undef __FUNCT__
836 #define __FUNCT__ "DMPlexConstructCohesiveCells_Internal"
837 /*
838   We are adding three kinds of points here:
839     Replicated:     Copies of points which exist in the mesh, such as vertices identified across a fault
840     Non-replicated: Points which exist on the fault, but are not replicated
841     Hybrid:         Entirely new points, such as cohesive cells
842 
843   When creating subsequent cohesive cells, we shift the old hybrid cells to the end of the numbering at
844   each depth so that the new split/hybrid points can be inserted as a block.
845 */
846 static PetscErrorCode DMPlexConstructCohesiveCells_Internal(DM dm, DMLabel label, DM sdm)
847 {
848   MPI_Comm         comm;
849   IS               valueIS;
850   PetscInt         numSP = 0;       /* The number of depths for which we have replicated points */
851   const PetscInt  *values;          /* List of depths for which we have replicated points */
852   IS              *splitIS;
853   IS              *unsplitIS;
854   PetscInt        *numSplitPoints;     /* The number of replicated points at each depth */
855   PetscInt        *numUnsplitPoints;   /* The number of non-replicated points at each depth which still give rise to hybrid points */
856   PetscInt        *numHybridPoints;    /* The number of new hybrid points at each depth */
857   PetscInt        *numHybridPointsOld; /* The number of existing hybrid points at each depth */
858   const PetscInt **splitPoints;        /* Replicated points for each depth */
859   const PetscInt **unsplitPoints;      /* Non-replicated points for each depth */
860   PetscSection     coordSection;
861   Vec              coordinates;
862   PetscScalar     *coords;
863   PetscInt        *depthMax;           /* The first hybrid point at each depth in the original mesh */
864   PetscInt        *depthEnd;           /* The point limit at each depth in the original mesh */
865   PetscInt        *depthShift;         /* Number of replicated+hybrid points at each depth */
866   PetscInt        *pMaxNew;            /* The first replicated point at each depth in the new mesh, hybrids come after this */
867   PetscInt        *coneNew, *coneONew, *supportNew;
868   PetscInt         shift = 100, shift2 = 200, depth = 0, dep, dim, d, sp, maxConeSize, maxSupportSize, maxConeSizeNew, maxSupportSizeNew, numLabels, vStart, vEnd, pEnd, p, v;
869   PetscErrorCode   ierr;
870 
871   PetscFunctionBegin;
872   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
873   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
874   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
875   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
876   /* Count split points and add cohesive cells */
877   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);CHKERRQ(ierr);
878   ierr = PetscMalloc5(depth+1,&depthMax,depth+1,&depthEnd,2*(depth+1),&depthShift,depth+1,&pMaxNew,depth+1,&numHybridPointsOld);CHKERRQ(ierr);
879   ierr = PetscMalloc7(depth+1,&splitIS,depth+1,&unsplitIS,depth+1,&numSplitPoints,depth+1,&numUnsplitPoints,depth+1,&numHybridPoints,depth+1,&splitPoints,depth+1,&unsplitPoints);CHKERRQ(ierr);
880   ierr = DMPlexGetHybridBounds(dm, depth >= 0 ? &depthMax[depth] : NULL, depth>1 ? &depthMax[depth-1] : NULL, depth>2 ? &depthMax[1] : NULL, depth >= 0 ? &depthMax[0] : NULL);CHKERRQ(ierr);
881   for (d = 0; d <= depth; ++d) {
882     ierr = DMPlexGetDepthStratum(dm, d, NULL, &pMaxNew[d]);CHKERRQ(ierr);
883     depthEnd[d]           = pMaxNew[d];
884     depthMax[d]           = depthMax[d] < 0 ? depthEnd[d] : depthMax[d];
885     numSplitPoints[d]     = 0;
886     numUnsplitPoints[d]   = 0;
887     numHybridPoints[d]    = 0;
888     numHybridPointsOld[d] = depthMax[d] < 0 ? 0 : depthEnd[d] - depthMax[d];
889     splitPoints[d]        = NULL;
890     unsplitPoints[d]      = NULL;
891     splitIS[d]            = NULL;
892     unsplitIS[d]          = NULL;
893     /* we are shifting the existing hybrid points with the stratum behind them, so
894      * the split comes at the end of the normal points, i.e., at depthMax[d] */
895     depthShift[2*d]       = depthMax[d];
896     depthShift[2*d+1]     = 0;
897   }
898   if (label) {
899     ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
900     ierr = ISGetLocalSize(valueIS, &numSP);CHKERRQ(ierr);
901     ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
902   }
903   for (sp = 0; sp < numSP; ++sp) {
904     const PetscInt dep = values[sp];
905 
906     if ((dep < 0) || (dep > depth)) continue;
907     ierr = DMLabelGetStratumIS(label, dep, &splitIS[dep]);CHKERRQ(ierr);
908     if (splitIS[dep]) {
909       ierr = ISGetLocalSize(splitIS[dep], &numSplitPoints[dep]);CHKERRQ(ierr);
910       ierr = ISGetIndices(splitIS[dep], &splitPoints[dep]);CHKERRQ(ierr);
911     }
912     ierr = DMLabelGetStratumIS(label, shift2+dep, &unsplitIS[dep]);CHKERRQ(ierr);
913     if (unsplitIS[dep]) {
914       ierr = ISGetLocalSize(unsplitIS[dep], &numUnsplitPoints[dep]);CHKERRQ(ierr);
915       ierr = ISGetIndices(unsplitIS[dep], &unsplitPoints[dep]);CHKERRQ(ierr);
916     }
917   }
918   /* Calculate number of hybrid points */
919   for (d = 1; d <= depth; ++d) numHybridPoints[d]     = numSplitPoints[d-1] + numUnsplitPoints[d-1]; /* There is a hybrid cell/face/edge for every split face/edge/vertex   */
920   for (d = 0; d <= depth; ++d) depthShift[2*d+1]      = numSplitPoints[d] + numHybridPoints[d];
921   ierr = DMPlexShiftPointSetUp_Internal(depth,depthShift);
922   /* the end of the points in this stratum that come before the new points:
923    * shifting pMaxNew[d] gets the new start of the next stratum, then count back the old hybrid points and the newly
924    * added points */
925   for (d = 0; d <= depth; ++d) pMaxNew[d]             = DMPlexShiftPoint_Internal(pMaxNew[d],depth,depthShift) - (numHybridPointsOld[d] + numSplitPoints[d] + numHybridPoints[d]);
926   ierr = DMPlexShiftSizes_Internal(dm, depthShift, sdm);CHKERRQ(ierr);
927   /* Step 3: Set cone/support sizes for new points */
928   for (dep = 0; dep <= depth; ++dep) {
929     for (p = 0; p < numSplitPoints[dep]; ++p) {
930       const PetscInt  oldp   = splitPoints[dep][p];
931       const PetscInt  newp   = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*oldp + depthOffset[dep]*/;
932       const PetscInt  splitp = p    + pMaxNew[dep];
933       const PetscInt *support;
934       PetscInt        coneSize, supportSize, qf, qn, qp, e;
935 
936       ierr = DMPlexGetConeSize(dm, oldp, &coneSize);CHKERRQ(ierr);
937       ierr = DMPlexSetConeSize(sdm, splitp, coneSize);CHKERRQ(ierr);
938       ierr = DMPlexGetSupportSize(dm, oldp, &supportSize);CHKERRQ(ierr);
939       ierr = DMPlexSetSupportSize(sdm, splitp, supportSize);CHKERRQ(ierr);
940       if (dep == depth-1) {
941         const PetscInt hybcell = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
942 
943         /* Add cohesive cells, they are prisms */
944         ierr = DMPlexSetConeSize(sdm, hybcell, 2 + coneSize);CHKERRQ(ierr);
945       } else if (dep == 0) {
946         const PetscInt hybedge = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
947 
948         ierr = DMPlexGetSupport(dm, oldp, &support);CHKERRQ(ierr);
949         for (e = 0, qn = 0, qp = 0, qf = 0; e < supportSize; ++e) {
950           PetscInt val;
951 
952           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
953           if (val == 1) ++qf;
954           if ((val == 1) || (val ==  (shift + 1))) ++qn;
955           if ((val == 1) || (val == -(shift + 1))) ++qp;
956         }
957         /* Split old vertex: Edges into original vertex and new cohesive edge */
958         ierr = DMPlexSetSupportSize(sdm, newp, qn+1);CHKERRQ(ierr);
959         /* Split new vertex: Edges into split vertex and new cohesive edge */
960         ierr = DMPlexSetSupportSize(sdm, splitp, qp+1);CHKERRQ(ierr);
961         /* Add hybrid edge */
962         ierr = DMPlexSetConeSize(sdm, hybedge, 2);CHKERRQ(ierr);
963         ierr = DMPlexSetSupportSize(sdm, hybedge, qf);CHKERRQ(ierr);
964       } else if (dep == dim-2) {
965         const PetscInt hybface = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
966 
967         ierr = DMPlexGetSupport(dm, oldp, &support);CHKERRQ(ierr);
968         for (e = 0, qn = 0, qp = 0, qf = 0; e < supportSize; ++e) {
969           PetscInt val;
970 
971           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
972           if (val == dim-1) ++qf;
973           if ((val == dim-1) || (val ==  (shift + dim-1))) ++qn;
974           if ((val == dim-1) || (val == -(shift + dim-1))) ++qp;
975         }
976         /* Split old edge: Faces into original edge and cohesive face (positive side?) */
977         ierr = DMPlexSetSupportSize(sdm, newp, qn+1);CHKERRQ(ierr);
978         /* Split new edge: Faces into split edge and cohesive face (negative side?) */
979         ierr = DMPlexSetSupportSize(sdm, splitp, qp+1);CHKERRQ(ierr);
980         /* Add hybrid face */
981         ierr = DMPlexSetConeSize(sdm, hybface, 4);CHKERRQ(ierr);
982         ierr = DMPlexSetSupportSize(sdm, hybface, qf);CHKERRQ(ierr);
983       }
984     }
985   }
986   for (dep = 0; dep <= depth; ++dep) {
987     for (p = 0; p < numUnsplitPoints[dep]; ++p) {
988       const PetscInt  oldp   = unsplitPoints[dep][p];
989       const PetscInt  newp   = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*oldp + depthOffset[dep]*/;
990       const PetscInt *support;
991       PetscInt        coneSize, supportSize, qf, e, s;
992 
993       ierr = DMPlexGetConeSize(dm, oldp, &coneSize);CHKERRQ(ierr);
994       ierr = DMPlexGetSupportSize(dm, oldp, &supportSize);CHKERRQ(ierr);
995       ierr = DMPlexGetSupport(dm, oldp, &support);CHKERRQ(ierr);
996       if (dep == 0) {
997         const PetscInt hybedge = p + pMaxNew[dep+1] + numSplitPoints[dep+1] + numSplitPoints[dep];
998 
999         /* Unsplit vertex: Edges into original vertex, split edges, and new cohesive edge twice */
1000         for (s = 0, qf = 0; s < supportSize; ++s, ++qf) {
1001           ierr = PetscFindInt(support[s], numSplitPoints[dep+1], splitPoints[dep+1], &e);CHKERRQ(ierr);
1002           if (e >= 0) ++qf;
1003         }
1004         ierr = DMPlexSetSupportSize(sdm, newp, qf+2);CHKERRQ(ierr);
1005         /* Add hybrid edge */
1006         ierr = DMPlexSetConeSize(sdm, hybedge, 2);CHKERRQ(ierr);
1007         for (e = 0, qf = 0; e < supportSize; ++e) {
1008           PetscInt val;
1009 
1010           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
1011           /* Split and unsplit edges produce hybrid faces */
1012           if (val == 1) ++qf;
1013           if (val == (shift2 + 1)) ++qf;
1014         }
1015         ierr = DMPlexSetSupportSize(sdm, hybedge, qf);CHKERRQ(ierr);
1016       } else if (dep == dim-2) {
1017         const PetscInt hybface = p + pMaxNew[dep+1] + numSplitPoints[dep+1] + numSplitPoints[dep];
1018         PetscInt       val;
1019 
1020         for (e = 0, qf = 0; e < supportSize; ++e) {
1021           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
1022           if (val == dim-1) qf += 2;
1023           else              ++qf;
1024         }
1025         /* Unsplit edge: Faces into original edge, split face, and cohesive face twice */
1026         ierr = DMPlexSetSupportSize(sdm, newp, qf+2);CHKERRQ(ierr);
1027         /* Add hybrid face */
1028         for (e = 0, qf = 0; e < supportSize; ++e) {
1029           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
1030           if (val == dim-1) ++qf;
1031         }
1032         ierr = DMPlexSetConeSize(sdm, hybface, 4);CHKERRQ(ierr);
1033         ierr = DMPlexSetSupportSize(sdm, hybface, qf);CHKERRQ(ierr);
1034       }
1035     }
1036   }
1037   /* Step 4: Setup split DM */
1038   ierr = DMSetUp(sdm);CHKERRQ(ierr);
1039   ierr = DMPlexShiftPoints_Internal(dm, depthShift, sdm);CHKERRQ(ierr);
1040   ierr = DMPlexGetMaxSizes(sdm, &maxConeSizeNew, &maxSupportSizeNew);CHKERRQ(ierr);
1041   ierr = PetscMalloc3(PetscMax(maxConeSize, maxConeSizeNew)*3,&coneNew,PetscMax(maxConeSize, maxConeSizeNew)*3,&coneONew,PetscMax(maxSupportSize, maxSupportSizeNew),&supportNew);CHKERRQ(ierr);
1042   /* Step 6: Set cones and supports for new points */
1043   for (dep = 0; dep <= depth; ++dep) {
1044     for (p = 0; p < numSplitPoints[dep]; ++p) {
1045       const PetscInt  oldp   = splitPoints[dep][p];
1046       const PetscInt  newp   = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*oldp + depthOffset[dep]*/;
1047       const PetscInt  splitp = p    + pMaxNew[dep];
1048       const PetscInt *cone, *support, *ornt;
1049       PetscInt        coneSize, supportSize, q, qf, qn, qp, v, e, s;
1050 
1051       ierr = DMPlexGetConeSize(dm, oldp, &coneSize);CHKERRQ(ierr);
1052       ierr = DMPlexGetCone(dm, oldp, &cone);CHKERRQ(ierr);
1053       ierr = DMPlexGetConeOrientation(dm, oldp, &ornt);CHKERRQ(ierr);
1054       ierr = DMPlexGetSupportSize(dm, oldp, &supportSize);CHKERRQ(ierr);
1055       ierr = DMPlexGetSupport(dm, oldp, &support);CHKERRQ(ierr);
1056       if (dep == depth-1) {
1057         PetscBool       hasUnsplit = PETSC_FALSE;
1058         const PetscInt  hybcell    = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1059         const PetscInt *supportF;
1060 
1061         /* Split face:       copy in old face to new face to start */
1062         ierr = DMPlexGetSupport(sdm, newp,  &supportF);CHKERRQ(ierr);
1063         ierr = DMPlexSetSupport(sdm, splitp, supportF);CHKERRQ(ierr);
1064         /* Split old face:   old vertices/edges in cone so no change */
1065         /* Split new face:   new vertices/edges in cone */
1066         for (q = 0; q < coneSize; ++q) {
1067           ierr = PetscFindInt(cone[q], numSplitPoints[dep-1], splitPoints[dep-1], &v);CHKERRQ(ierr);
1068           if (v < 0) {
1069             ierr = PetscFindInt(cone[q], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &v);CHKERRQ(ierr);
1070             if (v < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %d in split or unsplit points of depth %d", cone[q], dep-1);
1071             coneNew[2+q] = DMPlexShiftPoint_Internal(cone[q], depth, depthShift) /*cone[q] + depthOffset[dep-1]*/;
1072             hasUnsplit   = PETSC_TRUE;
1073           } else {
1074             coneNew[2+q] = v + pMaxNew[dep-1];
1075             if (dep > 1) {
1076               const PetscInt *econe;
1077               PetscInt        econeSize, r, vs, vu;
1078 
1079               ierr = DMPlexGetConeSize(dm, cone[q], &econeSize);CHKERRQ(ierr);
1080               ierr = DMPlexGetCone(dm, cone[q], &econe);CHKERRQ(ierr);
1081               for (r = 0; r < econeSize; ++r) {
1082                 ierr = PetscFindInt(econe[r], numSplitPoints[dep-2],   splitPoints[dep-2],   &vs);CHKERRQ(ierr);
1083                 ierr = PetscFindInt(econe[r], numUnsplitPoints[dep-2], unsplitPoints[dep-2], &vu);CHKERRQ(ierr);
1084                 if (vs >= 0) continue;
1085                 if (vu < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %d in split or unsplit points of depth %d", econe[r], dep-2);
1086                 hasUnsplit   = PETSC_TRUE;
1087               }
1088             }
1089           }
1090         }
1091         ierr = DMPlexSetCone(sdm, splitp, &coneNew[2]);CHKERRQ(ierr);
1092         ierr = DMPlexSetConeOrientation(sdm, splitp, ornt);CHKERRQ(ierr);
1093         /* Face support */
1094         for (s = 0; s < supportSize; ++s) {
1095           PetscInt val;
1096 
1097           ierr = DMLabelGetValue(label, support[s], &val);CHKERRQ(ierr);
1098           if (val < 0) {
1099             /* Split old face:   Replace negative side cell with cohesive cell */
1100              ierr = DMPlexInsertSupport(sdm, newp, s, hybcell);CHKERRQ(ierr);
1101           } else {
1102             /* Split new face:   Replace positive side cell with cohesive cell */
1103             ierr = DMPlexInsertSupport(sdm, splitp, s, hybcell);CHKERRQ(ierr);
1104             /* Get orientation for cohesive face */
1105             {
1106               const PetscInt *ncone, *nconeO;
1107               PetscInt        nconeSize, nc;
1108 
1109               ierr = DMPlexGetConeSize(dm, support[s], &nconeSize);CHKERRQ(ierr);
1110               ierr = DMPlexGetCone(dm, support[s], &ncone);CHKERRQ(ierr);
1111               ierr = DMPlexGetConeOrientation(dm, support[s], &nconeO);CHKERRQ(ierr);
1112               for (nc = 0; nc < nconeSize; ++nc) {
1113                 if (ncone[nc] == oldp) {
1114                   coneONew[0] = nconeO[nc];
1115                   break;
1116                 }
1117               }
1118               if (nc >= nconeSize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate face %d in neighboring cell %d", oldp, support[s]);
1119             }
1120           }
1121         }
1122         /* Cohesive cell:    Old and new split face, then new cohesive faces */
1123         coneNew[0]  = newp;   /* Extracted negative side orientation above */
1124         coneNew[1]  = splitp;
1125         coneONew[1] = coneONew[0];
1126         for (q = 0; q < coneSize; ++q) {
1127           ierr = PetscFindInt(cone[q], numSplitPoints[dep-1], splitPoints[dep-1], &v);CHKERRQ(ierr);
1128           if (v < 0) {
1129             ierr = PetscFindInt(cone[q], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &v);CHKERRQ(ierr);
1130             coneNew[2+q]  = v + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep-1];
1131             coneONew[2+q] = 0;
1132           } else {
1133             coneNew[2+q]  = v + pMaxNew[dep] + numSplitPoints[dep];
1134           }
1135           coneONew[2+q] = 0;
1136         }
1137         ierr = DMPlexSetCone(sdm, hybcell, coneNew);CHKERRQ(ierr);
1138         ierr = DMPlexSetConeOrientation(sdm, hybcell, coneONew);CHKERRQ(ierr);
1139         /* Label the hybrid cells on the boundary of the split */
1140         if (hasUnsplit) {ierr = DMLabelSetValue(label, -hybcell, dim);CHKERRQ(ierr);}
1141       } else if (dep == 0) {
1142         const PetscInt hybedge = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1143 
1144         /* Split old vertex: Edges in old split faces and new cohesive edge */
1145         for (e = 0, qn = 0; e < supportSize; ++e) {
1146           PetscInt val;
1147 
1148           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
1149           if ((val == 1) || (val == (shift + 1))) {
1150             supportNew[qn++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1151           }
1152         }
1153         supportNew[qn] = hybedge;
1154         ierr = DMPlexSetSupport(sdm, newp, supportNew);CHKERRQ(ierr);
1155         /* Split new vertex: Edges in new split faces and new cohesive edge */
1156         for (e = 0, qp = 0; e < supportSize; ++e) {
1157           PetscInt val, edge;
1158 
1159           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
1160           if (val == 1) {
1161             ierr = PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &edge);CHKERRQ(ierr);
1162             if (edge < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Edge %d is not a split edge", support[e]);
1163             supportNew[qp++] = edge + pMaxNew[dep+1];
1164           } else if (val == -(shift + 1)) {
1165             supportNew[qp++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1166           }
1167         }
1168         supportNew[qp] = hybedge;
1169         ierr = DMPlexSetSupport(sdm, splitp, supportNew);CHKERRQ(ierr);
1170         /* Hybrid edge:    Old and new split vertex */
1171         coneNew[0] = newp;
1172         coneNew[1] = splitp;
1173         ierr = DMPlexSetCone(sdm, hybedge, coneNew);CHKERRQ(ierr);
1174         for (e = 0, qf = 0; e < supportSize; ++e) {
1175           PetscInt val, edge;
1176 
1177           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
1178           if (val == 1) {
1179             ierr = PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &edge);CHKERRQ(ierr);
1180             if (edge < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Edge %d is not a split edge", support[e]);
1181             supportNew[qf++] = edge + pMaxNew[dep+2] + numSplitPoints[dep+2];
1182           }
1183         }
1184         ierr = DMPlexSetSupport(sdm, hybedge, supportNew);CHKERRQ(ierr);
1185       } else if (dep == dim-2) {
1186         const PetscInt hybface = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1187 
1188         /* Split old edge:   old vertices in cone so no change */
1189         /* Split new edge:   new vertices in cone */
1190         for (q = 0; q < coneSize; ++q) {
1191           ierr = PetscFindInt(cone[q], numSplitPoints[dep-1], splitPoints[dep-1], &v);CHKERRQ(ierr);
1192           if (v < 0) {
1193             ierr = PetscFindInt(cone[q], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &v);CHKERRQ(ierr);
1194             if (v < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %d in split or unsplit points of depth %d", cone[q], dep-1);
1195             coneNew[q] = DMPlexShiftPoint_Internal(cone[q], depth, depthShift) /*cone[q] + depthOffset[dep-1]*/;
1196           } else {
1197             coneNew[q] = v + pMaxNew[dep-1];
1198           }
1199         }
1200         ierr = DMPlexSetCone(sdm, splitp, coneNew);CHKERRQ(ierr);
1201         /* Split old edge: Faces in positive side cells and old split faces */
1202         for (e = 0, q = 0; e < supportSize; ++e) {
1203           PetscInt val;
1204 
1205           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
1206           if (val == dim-1) {
1207             supportNew[q++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1208           } else if (val == (shift + dim-1)) {
1209             supportNew[q++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1210           }
1211         }
1212         supportNew[q++] = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1213         ierr = DMPlexSetSupport(sdm, newp, supportNew);CHKERRQ(ierr);
1214         /* Split new edge: Faces in negative side cells and new split faces */
1215         for (e = 0, q = 0; e < supportSize; ++e) {
1216           PetscInt val, face;
1217 
1218           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
1219           if (val == dim-1) {
1220             ierr = PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &face);CHKERRQ(ierr);
1221             if (face < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Face %d is not a split face", support[e]);
1222             supportNew[q++] = face + pMaxNew[dep+1];
1223           } else if (val == -(shift + dim-1)) {
1224             supportNew[q++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1225           }
1226         }
1227         supportNew[q++] = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1228         ierr = DMPlexSetSupport(sdm, splitp, supportNew);CHKERRQ(ierr);
1229         /* Hybrid face */
1230         coneNew[0] = newp;
1231         coneNew[1] = splitp;
1232         for (v = 0; v < coneSize; ++v) {
1233           PetscInt vertex;
1234           ierr = PetscFindInt(cone[v], numSplitPoints[dep-1], splitPoints[dep-1], &vertex);CHKERRQ(ierr);
1235           if (vertex < 0) {
1236             ierr = PetscFindInt(cone[v], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &vertex);CHKERRQ(ierr);
1237             if (vertex < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %d in split or unsplit points of depth %d", cone[v], dep-1);
1238             coneNew[2+v] = vertex + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep-1];
1239           } else {
1240             coneNew[2+v] = vertex + pMaxNew[dep] + numSplitPoints[dep];
1241           }
1242         }
1243         ierr = DMPlexSetCone(sdm, hybface, coneNew);CHKERRQ(ierr);
1244         for (e = 0, qf = 0; e < supportSize; ++e) {
1245           PetscInt val, face;
1246 
1247           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
1248           if (val == dim-1) {
1249             ierr = PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &face);CHKERRQ(ierr);
1250             if (face < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Face %d is not a split face", support[e]);
1251             supportNew[qf++] = face + pMaxNew[dep+2] + numSplitPoints[dep+2];
1252           }
1253         }
1254         ierr = DMPlexSetSupport(sdm, hybface, supportNew);CHKERRQ(ierr);
1255       }
1256     }
1257   }
1258   for (dep = 0; dep <= depth; ++dep) {
1259     for (p = 0; p < numUnsplitPoints[dep]; ++p) {
1260       const PetscInt  oldp   = unsplitPoints[dep][p];
1261       const PetscInt  newp   = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*oldp + depthOffset[dep]*/;
1262       const PetscInt *cone, *support, *ornt;
1263       PetscInt        coneSize, supportSize, supportSizeNew, q, qf, e, f, s;
1264 
1265       ierr = DMPlexGetConeSize(dm, oldp, &coneSize);CHKERRQ(ierr);
1266       ierr = DMPlexGetCone(dm, oldp, &cone);CHKERRQ(ierr);
1267       ierr = DMPlexGetConeOrientation(dm, oldp, &ornt);CHKERRQ(ierr);
1268       ierr = DMPlexGetSupportSize(dm, oldp, &supportSize);CHKERRQ(ierr);
1269       ierr = DMPlexGetSupport(dm, oldp, &support);CHKERRQ(ierr);
1270       if (dep == 0) {
1271         const PetscInt hybedge = p + pMaxNew[dep+1] + numSplitPoints[dep+1] + numSplitPoints[dep];
1272 
1273         /* Unsplit vertex */
1274         ierr = DMPlexGetSupportSize(sdm, newp, &supportSizeNew);CHKERRQ(ierr);
1275         for (s = 0, q = 0; s < supportSize; ++s) {
1276           supportNew[q++] = DMPlexShiftPoint_Internal(support[s], depth, depthShift) /*support[s] + depthOffset[dep+1]*/;
1277           ierr = PetscFindInt(support[s], numSplitPoints[dep+1], splitPoints[dep+1], &e);CHKERRQ(ierr);
1278           if (e >= 0) {
1279             supportNew[q++] = e + pMaxNew[dep+1];
1280           }
1281         }
1282         supportNew[q++] = hybedge;
1283         supportNew[q++] = hybedge;
1284         if (q != supportSizeNew) SETERRQ3(comm, PETSC_ERR_ARG_WRONG, "Support size %d != %d for vertex %d", q, supportSizeNew, newp);
1285         ierr = DMPlexSetSupport(sdm, newp, supportNew);CHKERRQ(ierr);
1286         /* Hybrid edge */
1287         coneNew[0] = newp;
1288         coneNew[1] = newp;
1289         ierr = DMPlexSetCone(sdm, hybedge, coneNew);CHKERRQ(ierr);
1290         for (e = 0, qf = 0; e < supportSize; ++e) {
1291           PetscInt val, edge;
1292 
1293           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
1294           if (val == 1) {
1295             ierr = PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &edge);CHKERRQ(ierr);
1296             if (edge < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Edge %d is not a split edge", support[e]);
1297             supportNew[qf++] = edge + pMaxNew[dep+2] + numSplitPoints[dep+2];
1298           } else if  (val ==  (shift2 + 1)) {
1299             ierr = PetscFindInt(support[e], numUnsplitPoints[dep+1], unsplitPoints[dep+1], &edge);CHKERRQ(ierr);
1300             if (edge < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Edge %d is not a unsplit edge", support[e]);
1301             supportNew[qf++] = edge + pMaxNew[dep+2] + numSplitPoints[dep+2] + numSplitPoints[dep+1];
1302           }
1303         }
1304         ierr = DMPlexSetSupport(sdm, hybedge, supportNew);CHKERRQ(ierr);
1305       } else if (dep == dim-2) {
1306         const PetscInt hybface = p + pMaxNew[dep+1] + numSplitPoints[dep+1] + numSplitPoints[dep];
1307 
1308         /* Unsplit edge: Faces into original edge, split face, and hybrid face twice */
1309         for (f = 0, qf = 0; f < supportSize; ++f) {
1310           PetscInt val, face;
1311 
1312           ierr = DMLabelGetValue(label, support[f], &val);CHKERRQ(ierr);
1313           if (val == dim-1) {
1314             ierr = PetscFindInt(support[f], numSplitPoints[dep+1], splitPoints[dep+1], &face);CHKERRQ(ierr);
1315             if (face < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Face %d is not a split face", support[f]);
1316             supportNew[qf++] = DMPlexShiftPoint_Internal(support[f], depth, depthShift) /*support[f] + depthOffset[dep+1]*/;
1317             supportNew[qf++] = face + pMaxNew[dep+1];
1318           } else {
1319             supportNew[qf++] = DMPlexShiftPoint_Internal(support[f], depth, depthShift) /*support[f] + depthOffset[dep+1]*/;
1320           }
1321         }
1322         supportNew[qf++] = hybface;
1323         supportNew[qf++] = hybface;
1324         ierr = DMPlexGetSupportSize(sdm, newp, &supportSizeNew);CHKERRQ(ierr);
1325         if (qf != supportSizeNew) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Support size for unsplit edge %d is %d != %d\n", newp, qf, supportSizeNew);
1326         ierr = DMPlexSetSupport(sdm, newp, supportNew);CHKERRQ(ierr);
1327         /* Add hybrid face */
1328         coneNew[0] = newp;
1329         coneNew[1] = newp;
1330         ierr = PetscFindInt(cone[0], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &v);CHKERRQ(ierr);
1331         if (v < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Vertex %d is not an unsplit vertex", cone[0]);
1332         coneNew[2] = v + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep-1];
1333         ierr = PetscFindInt(cone[1], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &v);CHKERRQ(ierr);
1334         if (v < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Vertex %d is not an unsplit vertex", cone[1]);
1335         coneNew[3] = v + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep-1];
1336         ierr = DMPlexSetCone(sdm, hybface, coneNew);CHKERRQ(ierr);
1337         for (f = 0, qf = 0; f < supportSize; ++f) {
1338           PetscInt val, face;
1339 
1340           ierr = DMLabelGetValue(label, support[f], &val);CHKERRQ(ierr);
1341           if (val == dim-1) {
1342             ierr = PetscFindInt(support[f], numSplitPoints[dep+1], splitPoints[dep+1], &face);CHKERRQ(ierr);
1343             supportNew[qf++] = face + pMaxNew[dep+2] + numSplitPoints[dep+2];
1344           }
1345         }
1346         ierr = DMPlexGetSupportSize(sdm, hybface, &supportSizeNew);CHKERRQ(ierr);
1347         if (qf != supportSizeNew) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Support size for hybrid face %d is %d != %d\n", hybface, qf, supportSizeNew);
1348         ierr = DMPlexSetSupport(sdm, hybface, supportNew);CHKERRQ(ierr);
1349       }
1350     }
1351   }
1352   /* Step 6b: Replace split points in negative side cones */
1353   for (sp = 0; sp < numSP; ++sp) {
1354     PetscInt        dep = values[sp];
1355     IS              pIS;
1356     PetscInt        numPoints;
1357     const PetscInt *points;
1358 
1359     if (dep >= 0) continue;
1360     ierr = DMLabelGetStratumIS(label, dep, &pIS);CHKERRQ(ierr);
1361     if (!pIS) continue;
1362     dep  = -dep - shift;
1363     ierr = ISGetLocalSize(pIS, &numPoints);CHKERRQ(ierr);
1364     ierr = ISGetIndices(pIS, &points);CHKERRQ(ierr);
1365     for (p = 0; p < numPoints; ++p) {
1366       const PetscInt  oldp = points[p];
1367       const PetscInt  newp = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*depthOffset[dep] + oldp*/;
1368       const PetscInt *cone;
1369       PetscInt        coneSize, c;
1370       /* PetscBool       replaced = PETSC_FALSE; */
1371 
1372       /* Negative edge: replace split vertex */
1373       /* Negative cell: replace split face */
1374       ierr = DMPlexGetConeSize(sdm, newp, &coneSize);CHKERRQ(ierr);
1375       ierr = DMPlexGetCone(sdm, newp, &cone);CHKERRQ(ierr);
1376       for (c = 0; c < coneSize; ++c) {
1377         const PetscInt coldp = DMPlexShiftPointInverse_Internal(cone[c],depth,depthShift);
1378         PetscInt       csplitp, cp, val;
1379 
1380         ierr = DMLabelGetValue(label, coldp, &val);CHKERRQ(ierr);
1381         if (val == dep-1) {
1382           ierr = PetscFindInt(coldp, numSplitPoints[dep-1], splitPoints[dep-1], &cp);CHKERRQ(ierr);
1383           if (cp < 0) SETERRQ2(comm, PETSC_ERR_ARG_WRONG, "Point %d is not a split point of dimension %d", oldp, dep-1);
1384           csplitp  = pMaxNew[dep-1] + cp;
1385           ierr     = DMPlexInsertCone(sdm, newp, c, csplitp);CHKERRQ(ierr);
1386           /* replaced = PETSC_TRUE; */
1387         }
1388       }
1389       /* Cells with only a vertex or edge on the submesh have no replacement */
1390       /* if (!replaced) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "The cone of point %d does not contain split points", oldp); */
1391     }
1392     ierr = ISRestoreIndices(pIS, &points);CHKERRQ(ierr);
1393     ierr = ISDestroy(&pIS);CHKERRQ(ierr);
1394   }
1395   /* Step 7: Stratify */
1396   ierr = DMPlexStratify(sdm);CHKERRQ(ierr);
1397   /* Step 8: Coordinates */
1398   ierr = DMPlexShiftCoordinates_Internal(dm, depthShift, sdm);CHKERRQ(ierr);
1399   ierr = DMGetCoordinateSection(sdm, &coordSection);CHKERRQ(ierr);
1400   ierr = DMGetCoordinatesLocal(sdm, &coordinates);CHKERRQ(ierr);
1401   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
1402   for (v = 0; v < (numSplitPoints ? numSplitPoints[0] : 0); ++v) {
1403     const PetscInt newp   = DMPlexShiftPoint_Internal(splitPoints[0][v], depth, depthShift) /*depthOffset[0] + splitPoints[0][v]*/;
1404     const PetscInt splitp = pMaxNew[0] + v;
1405     PetscInt       dof, off, soff, d;
1406 
1407     ierr = PetscSectionGetDof(coordSection, newp, &dof);CHKERRQ(ierr);
1408     ierr = PetscSectionGetOffset(coordSection, newp, &off);CHKERRQ(ierr);
1409     ierr = PetscSectionGetOffset(coordSection, splitp, &soff);CHKERRQ(ierr);
1410     for (d = 0; d < dof; ++d) coords[soff+d] = coords[off+d];
1411   }
1412   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
1413   /* Step 9: SF, if I can figure this out we can split the mesh in parallel */
1414   ierr = DMPlexShiftSF_Internal(dm, depthShift, sdm);CHKERRQ(ierr);
1415   /* Step 10: Labels */
1416   ierr = DMPlexShiftLabels_Internal(dm, depthShift, sdm);CHKERRQ(ierr);
1417   ierr = DMGetNumLabels(sdm, &numLabels);CHKERRQ(ierr);
1418   for (dep = 0; dep <= depth; ++dep) {
1419     for (p = 0; p < numSplitPoints[dep]; ++p) {
1420       const PetscInt newp   = DMPlexShiftPoint_Internal(splitPoints[dep][p], depth, depthShift) /*depthOffset[dep] + splitPoints[dep][p]*/;
1421       const PetscInt splitp = pMaxNew[dep] + p;
1422       PetscInt       l;
1423 
1424       for (l = 0; l < numLabels; ++l) {
1425         DMLabel     mlabel;
1426         const char *lname;
1427         PetscInt    val;
1428         PetscBool   isDepth;
1429 
1430         ierr = DMGetLabelName(sdm, l, &lname);CHKERRQ(ierr);
1431         ierr = PetscStrcmp(lname, "depth", &isDepth);CHKERRQ(ierr);
1432         if (isDepth) continue;
1433         ierr = DMGetLabel(sdm, lname, &mlabel);CHKERRQ(ierr);
1434         ierr = DMLabelGetValue(mlabel, newp, &val);CHKERRQ(ierr);
1435         if (val >= 0) {
1436           ierr = DMLabelSetValue(mlabel, splitp, val);CHKERRQ(ierr);
1437 #if 0
1438           /* Do not put cohesive edges into the label */
1439           if (dep == 0) {
1440             const PetscInt cedge = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1441             ierr = DMLabelSetValue(mlabel, cedge, val);CHKERRQ(ierr);
1442           } else if (dep == dim-2) {
1443             const PetscInt cface = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1444             ierr = DMLabelSetValue(mlabel, cface, val);CHKERRQ(ierr);
1445           }
1446           /* Do not put cohesive faces into the label */
1447 #endif
1448         }
1449       }
1450     }
1451   }
1452   for (sp = 0; sp < numSP; ++sp) {
1453     const PetscInt dep = values[sp];
1454 
1455     if ((dep < 0) || (dep > depth)) continue;
1456     if (splitIS[dep]) {ierr = ISRestoreIndices(splitIS[dep], &splitPoints[dep]);CHKERRQ(ierr);}
1457     ierr = ISDestroy(&splitIS[dep]);CHKERRQ(ierr);
1458     if (unsplitIS[dep]) {ierr = ISRestoreIndices(unsplitIS[dep], &unsplitPoints[dep]);CHKERRQ(ierr);}
1459     ierr = ISDestroy(&unsplitIS[dep]);CHKERRQ(ierr);
1460   }
1461   if (label) {
1462     ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
1463     ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
1464   }
1465   for (d = 0; d <= depth; ++d) {
1466     ierr = DMPlexGetDepthStratum(sdm, d, NULL, &pEnd);CHKERRQ(ierr);
1467     pMaxNew[d] = pEnd - numHybridPoints[d] - numHybridPointsOld[d];
1468   }
1469   ierr = DMPlexSetHybridBounds(sdm, depth >= 0 ? pMaxNew[depth] : PETSC_DETERMINE, depth>1 ? pMaxNew[depth-1] : PETSC_DETERMINE, depth>2 ? pMaxNew[1] : PETSC_DETERMINE, depth >= 0 ? pMaxNew[0] : PETSC_DETERMINE);CHKERRQ(ierr);
1470   ierr = PetscFree3(coneNew, coneONew, supportNew);CHKERRQ(ierr);
1471   ierr = PetscFree5(depthMax, depthEnd, depthShift, pMaxNew, numHybridPointsOld);CHKERRQ(ierr);
1472   ierr = PetscFree7(splitIS, unsplitIS, numSplitPoints, numUnsplitPoints, numHybridPoints, splitPoints, unsplitPoints);CHKERRQ(ierr);
1473   PetscFunctionReturn(0);
1474 }
1475 
1476 #undef __FUNCT__
1477 #define __FUNCT__ "DMPlexConstructCohesiveCells"
1478 /*@C
1479   DMPlexConstructCohesiveCells - Construct cohesive cells which split the face along an internal interface
1480 
1481   Collective on dm
1482 
1483   Input Parameters:
1484 + dm - The original DM
1485 - label - The label specifying the boundary faces (this could be auto-generated)
1486 
1487   Output Parameters:
1488 - dmSplit - The new DM
1489 
1490   Level: developer
1491 
1492 .seealso: DMCreate(), DMPlexLabelCohesiveComplete()
1493 @*/
1494 PetscErrorCode DMPlexConstructCohesiveCells(DM dm, DMLabel label, DM *dmSplit)
1495 {
1496   DM             sdm;
1497   PetscInt       dim;
1498   PetscErrorCode ierr;
1499 
1500   PetscFunctionBegin;
1501   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1502   PetscValidPointer(dmSplit, 3);
1503   ierr = DMCreate(PetscObjectComm((PetscObject)dm), &sdm);CHKERRQ(ierr);
1504   ierr = DMSetType(sdm, DMPLEX);CHKERRQ(ierr);
1505   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
1506   ierr = DMSetDimension(sdm, dim);CHKERRQ(ierr);
1507   switch (dim) {
1508   case 2:
1509   case 3:
1510     ierr = DMPlexConstructCohesiveCells_Internal(dm, label, sdm);CHKERRQ(ierr);
1511     break;
1512   default:
1513     SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cannot construct cohesive cells for dimension %d", dim);
1514   }
1515   *dmSplit = sdm;
1516   PetscFunctionReturn(0);
1517 }
1518 
1519 #undef __FUNCT__
1520 #define __FUNCT__ "GetSurfaceSide_Static"
1521 /* Returns the side of the surface for a given cell with a face on the surface */
1522 static PetscErrorCode GetSurfaceSide_Static(DM dm, DM subdm, PetscInt numSubpoints, const PetscInt *subpoints, PetscInt cell, PetscInt face, PetscBool *pos)
1523 {
1524   const PetscInt *cone, *ornt;
1525   PetscInt        dim, coneSize, c;
1526   PetscErrorCode  ierr;
1527 
1528   PetscFunctionBegin;
1529   *pos = PETSC_TRUE;
1530   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
1531   ierr = DMPlexGetConeSize(dm, cell, &coneSize);CHKERRQ(ierr);
1532   ierr = DMPlexGetCone(dm, cell, &cone);CHKERRQ(ierr);
1533   ierr = DMPlexGetConeOrientation(dm, cell, &ornt);CHKERRQ(ierr);
1534   for (c = 0; c < coneSize; ++c) {
1535     if (cone[c] == face) {
1536       PetscInt o = ornt[c];
1537 
1538       if (subdm) {
1539         const PetscInt *subcone, *subornt;
1540         PetscInt        subpoint, subface, subconeSize, sc;
1541 
1542         ierr = PetscFindInt(cell, numSubpoints, subpoints, &subpoint);CHKERRQ(ierr);
1543         ierr = PetscFindInt(face, numSubpoints, subpoints, &subface);CHKERRQ(ierr);
1544         ierr = DMPlexGetConeSize(subdm, subpoint, &subconeSize);CHKERRQ(ierr);
1545         ierr = DMPlexGetCone(subdm, subpoint, &subcone);CHKERRQ(ierr);
1546         ierr = DMPlexGetConeOrientation(subdm, subpoint, &subornt);CHKERRQ(ierr);
1547         for (sc = 0; sc < subconeSize; ++sc) {
1548           if (subcone[sc] == subface) {
1549             o = subornt[0];
1550             break;
1551           }
1552         }
1553         if (sc >= subconeSize) SETERRQ4(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not find subpoint %d (%d) in cone for subpoint %d (%d)", subface, face, subpoint, cell);
1554       }
1555       if (o >= 0) *pos = PETSC_TRUE;
1556       else        *pos = PETSC_FALSE;
1557       break;
1558     }
1559   }
1560   if (c == coneSize) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Cell %d in split face %d support does not have it in the cone", cell, face);
1561   PetscFunctionReturn(0);
1562 }
1563 
1564 #undef __FUNCT__
1565 #define __FUNCT__ "DMPlexLabelCohesiveComplete"
1566 /*@
1567   DMPlexLabelCohesiveComplete - Starting with a label marking points on an internal surface, we add all other mesh pieces
1568   to complete the surface
1569 
1570   Input Parameters:
1571 + dm     - The DM
1572 . label  - A DMLabel marking the surface
1573 . blabel - A DMLabel marking the vertices on the boundary which will not be duplicated, or NULL to find them automatically
1574 . flip   - Flag to flip the submesh normal and replace points on the other side
1575 - subdm  - The subDM associated with the label, or NULL
1576 
1577   Output Parameter:
1578 . label - A DMLabel marking all surface points
1579 
1580   Note: The vertices in blabel are called "unsplit" in the terminology from hybrid cell creation.
1581 
1582   Level: developer
1583 
1584 .seealso: DMPlexConstructCohesiveCells(), DMPlexLabelComplete()
1585 @*/
1586 PetscErrorCode DMPlexLabelCohesiveComplete(DM dm, DMLabel label, DMLabel blabel, PetscBool flip, DM subdm)
1587 {
1588   DMLabel         depthLabel;
1589   IS              dimIS, subpointIS, facePosIS, faceNegIS, crossEdgeIS = NULL;
1590   const PetscInt *points, *subpoints;
1591   const PetscInt  rev   = flip ? -1 : 1;
1592   PetscInt       *pMax;
1593   PetscInt        shift = 100, shift2 = 200, dim, depth, pSize, dep, cStart, cEnd, cMax, fStart, fEnd, vStart, vEnd, numPoints, numSubpoints, p, val;
1594   PetscErrorCode  ierr;
1595 
1596   PetscFunctionBegin;
1597   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
1598   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
1599   pSize = PetscMax(depth, dim) + 1;
1600   ierr = PetscMalloc1(pSize,&pMax);CHKERRQ(ierr);
1601   ierr = DMPlexGetHybridBounds(dm, depth >= 0 ? &pMax[depth] : NULL, depth>1 ? &pMax[depth-1] : NULL, depth>2 ? &pMax[1] : NULL, &pMax[0]);CHKERRQ(ierr);
1602   ierr = DMPlexGetDepthLabel(dm, &depthLabel);CHKERRQ(ierr);
1603   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
1604   if (subdm) {
1605     ierr = DMPlexCreateSubpointIS(subdm, &subpointIS);CHKERRQ(ierr);
1606     if (subpointIS) {
1607       ierr = ISGetLocalSize(subpointIS, &numSubpoints);CHKERRQ(ierr);
1608       ierr = ISGetIndices(subpointIS, &subpoints);CHKERRQ(ierr);
1609     }
1610   }
1611   /* Mark cell on the fault, and its faces which touch the fault: cell orientation for face gives the side of the fault */
1612   ierr = DMLabelGetStratumIS(label, dim-1, &dimIS);CHKERRQ(ierr);
1613   if (!dimIS) {
1614     ierr = PetscFree(pMax);CHKERRQ(ierr);
1615     PetscFunctionReturn(0);
1616   }
1617   ierr = ISGetLocalSize(dimIS, &numPoints);CHKERRQ(ierr);
1618   ierr = ISGetIndices(dimIS, &points);CHKERRQ(ierr);
1619   for (p = 0; p < numPoints; ++p) { /* Loop over fault faces */
1620     const PetscInt *support;
1621     PetscInt        supportSize, s;
1622 
1623     ierr = DMPlexGetSupportSize(dm, points[p], &supportSize);CHKERRQ(ierr);
1624     if (supportSize != 2) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Split face %d has %d != 2 supports", points[p], supportSize);
1625     ierr = DMPlexGetSupport(dm, points[p], &support);CHKERRQ(ierr);
1626     for (s = 0; s < supportSize; ++s) {
1627       const PetscInt *cone;
1628       PetscInt        coneSize, c;
1629       PetscBool       pos;
1630 
1631       ierr = GetSurfaceSide_Static(dm, subdm, numSubpoints, subpoints, support[s], points[p], &pos);CHKERRQ(ierr);
1632       if (pos) {ierr = DMLabelSetValue(label, support[s],  rev*(shift+dim));CHKERRQ(ierr);}
1633       else     {ierr = DMLabelSetValue(label, support[s], -rev*(shift+dim));CHKERRQ(ierr);}
1634       if (rev < 0) pos = !pos ? PETSC_TRUE : PETSC_FALSE;
1635       /* Put faces touching the fault in the label */
1636       ierr = DMPlexGetConeSize(dm, support[s], &coneSize);CHKERRQ(ierr);
1637       ierr = DMPlexGetCone(dm, support[s], &cone);CHKERRQ(ierr);
1638       for (c = 0; c < coneSize; ++c) {
1639         const PetscInt point = cone[c];
1640 
1641         ierr = DMLabelGetValue(label, point, &val);CHKERRQ(ierr);
1642         if (val == -1) {
1643           PetscInt *closure = NULL;
1644           PetscInt  closureSize, cl;
1645 
1646           ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1647           for (cl = 0; cl < closureSize*2; cl += 2) {
1648             const PetscInt clp  = closure[cl];
1649             PetscInt       bval = -1;
1650 
1651             ierr = DMLabelGetValue(label, clp, &val);CHKERRQ(ierr);
1652             if (blabel) {ierr = DMLabelGetValue(blabel, clp, &bval);CHKERRQ(ierr);}
1653             if ((val >= 0) && (val < dim-1) && (bval < 0)) {
1654               ierr = DMLabelSetValue(label, point, pos == PETSC_TRUE ? shift+dim-1 : -(shift+dim-1));CHKERRQ(ierr);
1655               break;
1656             }
1657           }
1658           ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1659         }
1660       }
1661     }
1662   }
1663   ierr = ISRestoreIndices(dimIS, &points);CHKERRQ(ierr);
1664   ierr = ISDestroy(&dimIS);CHKERRQ(ierr);
1665   if (subdm) {
1666     if (subpointIS) {ierr = ISRestoreIndices(subpointIS, &subpoints);CHKERRQ(ierr);}
1667     ierr = ISDestroy(&subpointIS);CHKERRQ(ierr);
1668   }
1669   /* Mark boundary points as unsplit */
1670   if (blabel) {
1671     ierr = DMLabelGetStratumIS(blabel, 1, &dimIS);CHKERRQ(ierr);
1672     ierr = ISGetLocalSize(dimIS, &numPoints);CHKERRQ(ierr);
1673     ierr = ISGetIndices(dimIS, &points);CHKERRQ(ierr);
1674     for (p = 0; p < numPoints; ++p) {
1675       const PetscInt point = points[p];
1676       PetscInt       val, bval;
1677 
1678       ierr = DMLabelGetValue(blabel, point, &bval);CHKERRQ(ierr);
1679       if (bval >= 0) {
1680         ierr = DMLabelGetValue(label, point, &val);CHKERRQ(ierr);
1681         if ((val < 0) || (val > dim)) {
1682           /* This could be a point added from splitting a vertex on an adjacent fault, otherwise its just wrong */
1683           ierr = DMLabelClearValue(blabel, point, bval);CHKERRQ(ierr);
1684         }
1685       }
1686     }
1687     for (p = 0; p < numPoints; ++p) {
1688       const PetscInt point = points[p];
1689       PetscInt       val, bval;
1690 
1691       ierr = DMLabelGetValue(blabel, point, &bval);CHKERRQ(ierr);
1692       if (bval >= 0) {
1693         const PetscInt *cone,    *support;
1694         PetscInt        coneSize, supportSize, s, valA, valB, valE;
1695 
1696         /* Mark as unsplit */
1697         ierr = DMLabelGetValue(label, point, &val);CHKERRQ(ierr);
1698         if ((val < 0) || (val > dim)) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %d has label value %d, should be part of the fault", point, val);
1699         ierr = DMLabelClearValue(label, point, val);CHKERRQ(ierr);
1700         ierr = DMLabelSetValue(label, point, shift2+val);CHKERRQ(ierr);
1701         /* Check for cross-edge
1702              A cross-edge has endpoints which are both on the boundary of the surface, but the edge itself is not. */
1703         if (val != 0) continue;
1704         ierr = DMPlexGetSupport(dm, point, &support);CHKERRQ(ierr);
1705         ierr = DMPlexGetSupportSize(dm, point, &supportSize);CHKERRQ(ierr);
1706         for (s = 0; s < supportSize; ++s) {
1707           ierr = DMPlexGetCone(dm, support[s], &cone);CHKERRQ(ierr);
1708           ierr = DMPlexGetConeSize(dm, support[s], &coneSize);CHKERRQ(ierr);
1709           if (coneSize != 2) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Edge %D has %D vertices != 2", support[s], coneSize);
1710           ierr = DMLabelGetValue(blabel, cone[0], &valA);CHKERRQ(ierr);
1711           ierr = DMLabelGetValue(blabel, cone[1], &valB);CHKERRQ(ierr);
1712           ierr = DMLabelGetValue(blabel, support[s], &valE);CHKERRQ(ierr);
1713           if ((valE < 0) && (valA >= 0) && (valB >= 0) && (cone[0] != cone[1])) {ierr = DMLabelSetValue(blabel, support[s], 2);CHKERRQ(ierr);}
1714         }
1715       }
1716     }
1717     ierr = ISRestoreIndices(dimIS, &points);CHKERRQ(ierr);
1718     ierr = ISDestroy(&dimIS);CHKERRQ(ierr);
1719   }
1720   /* Search for other cells/faces/edges connected to the fault by a vertex */
1721   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
1722   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
1723   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
1724   cMax = cMax < 0 ? cEnd : cMax;
1725   ierr = DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd);CHKERRQ(ierr);
1726   ierr = DMLabelGetStratumIS(label, 0, &dimIS);CHKERRQ(ierr);
1727   if (blabel) {ierr = DMLabelGetStratumIS(blabel, 2, &crossEdgeIS);CHKERRQ(ierr);}
1728   if (dimIS && crossEdgeIS) {
1729     IS vertIS = dimIS;
1730 
1731     ierr = ISExpand(vertIS, crossEdgeIS, &dimIS);CHKERRQ(ierr);
1732     ierr = ISDestroy(&crossEdgeIS);CHKERRQ(ierr);
1733     ierr = ISDestroy(&vertIS);CHKERRQ(ierr);
1734   }
1735   if (!dimIS) {
1736     ierr = PetscFree(pMax);CHKERRQ(ierr);
1737     PetscFunctionReturn(0);
1738   }
1739   ierr = ISGetLocalSize(dimIS, &numPoints);CHKERRQ(ierr);
1740   ierr = ISGetIndices(dimIS, &points);CHKERRQ(ierr);
1741   for (p = 0; p < numPoints; ++p) { /* Loop over fault vertices */
1742     PetscInt *star = NULL;
1743     PetscInt  starSize, s;
1744     PetscInt  again = 1;  /* 0: Finished 1: Keep iterating after a change 2: No change */
1745 
1746     /* All points connected to the fault are inside a cell, so at the top level we will only check cells */
1747     ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_FALSE, &starSize, &star);CHKERRQ(ierr);
1748     while (again) {
1749       if (again > 1) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Could not classify all cells connected to the fault");
1750       again = 0;
1751       for (s = 0; s < starSize*2; s += 2) {
1752         const PetscInt  point = star[s];
1753         const PetscInt *cone;
1754         PetscInt        coneSize, c;
1755 
1756         if ((point < cStart) || (point >= cMax)) continue;
1757         ierr = DMLabelGetValue(label, point, &val);CHKERRQ(ierr);
1758         if (val != -1) continue;
1759         again = again == 1 ? 1 : 2;
1760         ierr  = DMPlexGetConeSize(dm, point, &coneSize);CHKERRQ(ierr);
1761         ierr  = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
1762         for (c = 0; c < coneSize; ++c) {
1763           ierr = DMLabelGetValue(label, cone[c], &val);CHKERRQ(ierr);
1764           if (val != -1) {
1765             const PetscInt *ccone;
1766             PetscInt        cconeSize, cc, side;
1767 
1768             if (PetscAbs(val) < shift) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Face %d on cell %d has an invalid label %d", cone[c], point, val);
1769             if (val > 0) side =  1;
1770             else         side = -1;
1771             ierr = DMLabelSetValue(label, point, side*(shift+dim));CHKERRQ(ierr);
1772             /* Mark cell faces which touch the fault */
1773             ierr = DMPlexGetConeSize(dm, point, &cconeSize);CHKERRQ(ierr);
1774             ierr = DMPlexGetCone(dm, point, &ccone);CHKERRQ(ierr);
1775             for (cc = 0; cc < cconeSize; ++cc) {
1776               PetscInt *closure = NULL;
1777               PetscInt  closureSize, cl;
1778 
1779               ierr = DMLabelGetValue(label, ccone[cc], &val);CHKERRQ(ierr);
1780               if (val != -1) continue;
1781               ierr = DMPlexGetTransitiveClosure(dm, ccone[cc], PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1782               for (cl = 0; cl < closureSize*2; cl += 2) {
1783                 const PetscInt clp = closure[cl];
1784 
1785                 ierr = DMLabelGetValue(label, clp, &val);CHKERRQ(ierr);
1786                 if (val == -1) continue;
1787                 ierr = DMLabelSetValue(label, ccone[cc], side*(shift+dim-1));CHKERRQ(ierr);
1788                 break;
1789               }
1790               ierr = DMPlexRestoreTransitiveClosure(dm, ccone[cc], PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1791             }
1792             again = 1;
1793             break;
1794           }
1795         }
1796       }
1797     }
1798     /* Classify the rest by cell membership */
1799     for (s = 0; s < starSize*2; s += 2) {
1800       const PetscInt point = star[s];
1801 
1802       ierr = DMLabelGetValue(label, point, &val);CHKERRQ(ierr);
1803       if (val == -1) {
1804         PetscInt  *sstar = NULL;
1805         PetscInt   sstarSize, ss;
1806         PetscBool  marked = PETSC_FALSE;
1807 
1808         ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_FALSE, &sstarSize, &sstar);CHKERRQ(ierr);
1809         for (ss = 0; ss < sstarSize*2; ss += 2) {
1810           const PetscInt spoint = sstar[ss];
1811 
1812           if ((spoint < cStart) || (spoint >= cMax)) continue;
1813           ierr = DMLabelGetValue(label, spoint, &val);CHKERRQ(ierr);
1814           if (val == -1) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Cell %d in star of %d does not have a valid label", spoint, point);
1815           ierr = DMLabelGetValue(depthLabel, point, &dep);CHKERRQ(ierr);
1816           if (val > 0) {
1817             ierr = DMLabelSetValue(label, point,   shift+dep);CHKERRQ(ierr);
1818           } else {
1819             ierr = DMLabelSetValue(label, point, -(shift+dep));CHKERRQ(ierr);
1820           }
1821           marked = PETSC_TRUE;
1822           break;
1823         }
1824         ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_FALSE, &sstarSize, &sstar);CHKERRQ(ierr);
1825         ierr = DMLabelGetValue(depthLabel, point, &dep);CHKERRQ(ierr);
1826         if (point < pMax[dep] && !marked) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d could not be classified", point);
1827       }
1828     }
1829     ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_FALSE, &starSize, &star);CHKERRQ(ierr);
1830   }
1831   ierr = ISRestoreIndices(dimIS, &points);CHKERRQ(ierr);
1832   ierr = ISDestroy(&dimIS);CHKERRQ(ierr);
1833   /* If any faces touching the fault divide cells on either side, split them */
1834   ierr = DMLabelGetStratumIS(label,   shift+dim-1,  &facePosIS);CHKERRQ(ierr);
1835   ierr = DMLabelGetStratumIS(label, -(shift+dim-1), &faceNegIS);CHKERRQ(ierr);
1836   ierr = ISExpand(facePosIS, faceNegIS, &dimIS);CHKERRQ(ierr);
1837   ierr = ISDestroy(&facePosIS);CHKERRQ(ierr);
1838   ierr = ISDestroy(&faceNegIS);CHKERRQ(ierr);
1839   ierr = ISGetLocalSize(dimIS, &numPoints);CHKERRQ(ierr);
1840   ierr = ISGetIndices(dimIS, &points);CHKERRQ(ierr);
1841   for (p = 0; p < numPoints; ++p) {
1842     const PetscInt  point = points[p];
1843     const PetscInt *support;
1844     PetscInt        supportSize, valA, valB;
1845 
1846     ierr = DMPlexGetSupportSize(dm, point, &supportSize);CHKERRQ(ierr);
1847     if (supportSize != 2) continue;
1848     ierr = DMPlexGetSupport(dm, point, &support);CHKERRQ(ierr);
1849     ierr = DMLabelGetValue(label, support[0], &valA);CHKERRQ(ierr);
1850     ierr = DMLabelGetValue(label, support[1], &valB);CHKERRQ(ierr);
1851     if ((valA == -1) || (valB == -1)) continue;
1852     if (valA*valB > 0) continue;
1853     /* Split the face */
1854     ierr = DMLabelGetValue(label, point, &valA);CHKERRQ(ierr);
1855     ierr = DMLabelClearValue(label, point, valA);CHKERRQ(ierr);
1856     ierr = DMLabelSetValue(label, point, dim-1);CHKERRQ(ierr);
1857     /* Label its closure:
1858       unmarked: label as unsplit
1859       incident: relabel as split
1860       split:    do nothing
1861     */
1862     {
1863       PetscInt *closure = NULL;
1864       PetscInt  closureSize, cl;
1865 
1866       ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1867       for (cl = 0; cl < closureSize*2; cl += 2) {
1868         ierr = DMLabelGetValue(label, closure[cl], &valA);CHKERRQ(ierr);
1869         if (valA == -1) { /* Mark as unsplit */
1870           ierr = DMLabelGetValue(depthLabel, closure[cl], &dep);CHKERRQ(ierr);
1871           ierr = DMLabelSetValue(label, closure[cl], shift2+dep);CHKERRQ(ierr);
1872         } else if (((valA >= shift) && (valA < shift2)) || ((valA <= -shift) && (valA > -shift2))) {
1873           ierr = DMLabelGetValue(depthLabel, closure[cl], &dep);CHKERRQ(ierr);
1874           ierr = DMLabelClearValue(label, closure[cl], valA);CHKERRQ(ierr);
1875           ierr = DMLabelSetValue(label, closure[cl], dep);CHKERRQ(ierr);
1876         }
1877       }
1878       ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1879     }
1880   }
1881   ierr = ISRestoreIndices(dimIS, &points);CHKERRQ(ierr);
1882   ierr = ISDestroy(&dimIS);CHKERRQ(ierr);
1883   ierr = PetscFree(pMax);CHKERRQ(ierr);
1884   PetscFunctionReturn(0);
1885 }
1886 
1887 #undef __FUNCT__
1888 #define __FUNCT__ "DMPlexCreateHybridMesh"
1889 /*@C
1890   DMPlexCreateHybridMesh - Create a mesh with hybrid cells along an internal interface
1891 
1892   Collective on dm
1893 
1894   Input Parameters:
1895 + dm - The original DM
1896 - labelName - The label specifying the interface vertices
1897 
1898   Output Parameters:
1899 + hybridLabel - The label fully marking the interface
1900 - dmHybrid - The new DM
1901 
1902   Level: developer
1903 
1904 .seealso: DMPlexConstructCohesiveCells(), DMPlexLabelCohesiveComplete(), DMCreate()
1905 @*/
1906 PetscErrorCode DMPlexCreateHybridMesh(DM dm, DMLabel label, DMLabel *hybridLabel, DM *dmHybrid)
1907 {
1908   DM             idm;
1909   DMLabel        subpointMap, hlabel;
1910   PetscInt       dim;
1911   PetscErrorCode ierr;
1912 
1913   PetscFunctionBegin;
1914   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1915   if (hybridLabel) PetscValidPointer(hybridLabel, 3);
1916   PetscValidPointer(dmHybrid, 4);
1917   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
1918   ierr = DMPlexCreateSubmesh(dm, label, 1, &idm);CHKERRQ(ierr);
1919   ierr = DMPlexOrient(idm);CHKERRQ(ierr);
1920   ierr = DMPlexGetSubpointMap(idm, &subpointMap);CHKERRQ(ierr);
1921   ierr = DMLabelDuplicate(subpointMap, &hlabel);CHKERRQ(ierr);
1922   ierr = DMLabelClearStratum(hlabel, dim);CHKERRQ(ierr);
1923   ierr = DMPlexLabelCohesiveComplete(dm, hlabel, NULL, PETSC_FALSE, idm);CHKERRQ(ierr);
1924   ierr = DMDestroy(&idm);CHKERRQ(ierr);
1925   ierr = DMPlexConstructCohesiveCells(dm, hlabel, dmHybrid);CHKERRQ(ierr);
1926   if (hybridLabel) *hybridLabel = hlabel;
1927   else             {ierr = DMLabelDestroy(&hlabel);CHKERRQ(ierr);}
1928   PetscFunctionReturn(0);
1929 }
1930 
1931 #undef __FUNCT__
1932 #define __FUNCT__ "DMPlexMarkSubmesh_Uninterpolated"
1933 /* Here we need the explicit assumption that:
1934 
1935      For any marked cell, the marked vertices constitute a single face
1936 */
1937 static PetscErrorCode DMPlexMarkSubmesh_Uninterpolated(DM dm, DMLabel vertexLabel, PetscInt value, DMLabel subpointMap, PetscInt *numFaces, PetscInt *nFV, DM subdm)
1938 {
1939   IS               subvertexIS = NULL;
1940   const PetscInt  *subvertices;
1941   PetscInt        *pStart, *pEnd, *pMax, pSize;
1942   PetscInt         depth, dim, d, numSubVerticesInitial = 0, v;
1943   PetscErrorCode   ierr;
1944 
1945   PetscFunctionBegin;
1946   *numFaces = 0;
1947   *nFV      = 0;
1948   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
1949   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
1950   pSize = PetscMax(depth, dim) + 1;
1951   ierr = PetscMalloc3(pSize,&pStart,pSize,&pEnd,pSize,&pMax);CHKERRQ(ierr);
1952   ierr = DMPlexGetHybridBounds(dm, depth >= 0 ? &pMax[depth] : NULL, depth>1 ? &pMax[depth-1] : NULL, depth>2 ? &pMax[1] : NULL, &pMax[0]);CHKERRQ(ierr);
1953   for (d = 0; d <= depth; ++d) {
1954     ierr = DMPlexGetDepthStratum(dm, d, &pStart[d], &pEnd[d]);CHKERRQ(ierr);
1955     if (pMax[d] >= 0) pEnd[d] = PetscMin(pEnd[d], pMax[d]);
1956   }
1957   /* Loop over initial vertices and mark all faces in the collective star() */
1958   if (vertexLabel) {ierr = DMLabelGetStratumIS(vertexLabel, value, &subvertexIS);CHKERRQ(ierr);}
1959   if (subvertexIS) {
1960     ierr = ISGetSize(subvertexIS, &numSubVerticesInitial);CHKERRQ(ierr);
1961     ierr = ISGetIndices(subvertexIS, &subvertices);CHKERRQ(ierr);
1962   }
1963   for (v = 0; v < numSubVerticesInitial; ++v) {
1964     const PetscInt vertex = subvertices[v];
1965     PetscInt      *star   = NULL;
1966     PetscInt       starSize, s, numCells = 0, c;
1967 
1968     ierr = DMPlexGetTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star);CHKERRQ(ierr);
1969     for (s = 0; s < starSize*2; s += 2) {
1970       const PetscInt point = star[s];
1971       if ((point >= pStart[depth]) && (point < pEnd[depth])) star[numCells++] = point;
1972     }
1973     for (c = 0; c < numCells; ++c) {
1974       const PetscInt cell    = star[c];
1975       PetscInt      *closure = NULL;
1976       PetscInt       closureSize, cl;
1977       PetscInt       cellLoc, numCorners = 0, faceSize = 0;
1978 
1979       ierr = DMLabelGetValue(subpointMap, cell, &cellLoc);CHKERRQ(ierr);
1980       if (cellLoc == 2) continue;
1981       if (cellLoc >= 0) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Cell %d has dimension %d in the surface label", cell, cellLoc);
1982       ierr = DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1983       for (cl = 0; cl < closureSize*2; cl += 2) {
1984         const PetscInt point = closure[cl];
1985         PetscInt       vertexLoc;
1986 
1987         if ((point >= pStart[0]) && (point < pEnd[0])) {
1988           ++numCorners;
1989           ierr = DMLabelGetValue(vertexLabel, point, &vertexLoc);CHKERRQ(ierr);
1990           if (vertexLoc == value) closure[faceSize++] = point;
1991         }
1992       }
1993       if (!(*nFV)) {ierr = DMPlexGetNumFaceVertices(dm, dim, numCorners, nFV);CHKERRQ(ierr);}
1994       if (faceSize > *nFV) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Invalid submesh: Too many vertices %d of an element on the surface", faceSize);
1995       if (faceSize == *nFV) {
1996         const PetscInt *cells = NULL;
1997         PetscInt        numCells, nc;
1998 
1999         ++(*numFaces);
2000         for (cl = 0; cl < faceSize; ++cl) {
2001           ierr = DMLabelSetValue(subpointMap, closure[cl], 0);CHKERRQ(ierr);
2002         }
2003         ierr = DMPlexGetJoin(dm, faceSize, closure, &numCells, &cells);CHKERRQ(ierr);
2004         for (nc = 0; nc < numCells; ++nc) {
2005           ierr = DMLabelSetValue(subpointMap, cells[nc], 2);CHKERRQ(ierr);
2006         }
2007         ierr = DMPlexRestoreJoin(dm, faceSize, closure, &numCells, &cells);CHKERRQ(ierr);
2008       }
2009       ierr = DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
2010     }
2011     ierr = DMPlexRestoreTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star);CHKERRQ(ierr);
2012   }
2013   if (subvertexIS) {
2014     ierr = ISRestoreIndices(subvertexIS, &subvertices);CHKERRQ(ierr);
2015   }
2016   ierr = ISDestroy(&subvertexIS);CHKERRQ(ierr);
2017   ierr = PetscFree3(pStart,pEnd,pMax);CHKERRQ(ierr);
2018   PetscFunctionReturn(0);
2019 }
2020 
2021 #undef __FUNCT__
2022 #define __FUNCT__ "DMPlexMarkSubmesh_Interpolated"
2023 static PetscErrorCode DMPlexMarkSubmesh_Interpolated(DM dm, DMLabel vertexLabel, PetscInt value, DMLabel subpointMap, DM subdm)
2024 {
2025   IS               subvertexIS = NULL;
2026   const PetscInt  *subvertices;
2027   PetscInt        *pStart, *pEnd, *pMax;
2028   PetscInt         dim, d, numSubVerticesInitial = 0, v;
2029   PetscErrorCode   ierr;
2030 
2031   PetscFunctionBegin;
2032   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
2033   ierr = PetscMalloc3(dim+1,&pStart,dim+1,&pEnd,dim+1,&pMax);CHKERRQ(ierr);
2034   ierr = DMPlexGetHybridBounds(dm, &pMax[dim], dim>1 ? &pMax[dim-1] : NULL, dim > 2 ? &pMax[1] : NULL, &pMax[0]);CHKERRQ(ierr);
2035   for (d = 0; d <= dim; ++d) {
2036     ierr = DMPlexGetDepthStratum(dm, d, &pStart[d], &pEnd[d]);CHKERRQ(ierr);
2037     if (pMax[d] >= 0) pEnd[d] = PetscMin(pEnd[d], pMax[d]);
2038   }
2039   /* Loop over initial vertices and mark all faces in the collective star() */
2040   if (vertexLabel) {
2041     ierr = DMLabelGetStratumIS(vertexLabel, value, &subvertexIS);CHKERRQ(ierr);
2042     if (subvertexIS) {
2043       ierr = ISGetSize(subvertexIS, &numSubVerticesInitial);CHKERRQ(ierr);
2044       ierr = ISGetIndices(subvertexIS, &subvertices);CHKERRQ(ierr);
2045     }
2046   }
2047   for (v = 0; v < numSubVerticesInitial; ++v) {
2048     const PetscInt vertex = subvertices[v];
2049     PetscInt      *star   = NULL;
2050     PetscInt       starSize, s, numFaces = 0, f;
2051 
2052     ierr = DMPlexGetTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star);CHKERRQ(ierr);
2053     for (s = 0; s < starSize*2; s += 2) {
2054       const PetscInt point = star[s];
2055       if ((point >= pStart[dim-1]) && (point < pEnd[dim-1])) star[numFaces++] = point;
2056     }
2057     for (f = 0; f < numFaces; ++f) {
2058       const PetscInt face    = star[f];
2059       PetscInt      *closure = NULL;
2060       PetscInt       closureSize, c;
2061       PetscInt       faceLoc;
2062 
2063       ierr = DMLabelGetValue(subpointMap, face, &faceLoc);CHKERRQ(ierr);
2064       if (faceLoc == dim-1) continue;
2065       if (faceLoc >= 0) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Face %d has dimension %d in the surface label", face, faceLoc);
2066       ierr = DMPlexGetTransitiveClosure(dm, face, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
2067       for (c = 0; c < closureSize*2; c += 2) {
2068         const PetscInt point = closure[c];
2069         PetscInt       vertexLoc;
2070 
2071         if ((point >= pStart[0]) && (point < pEnd[0])) {
2072           ierr = DMLabelGetValue(vertexLabel, point, &vertexLoc);CHKERRQ(ierr);
2073           if (vertexLoc != value) break;
2074         }
2075       }
2076       if (c == closureSize*2) {
2077         const PetscInt *support;
2078         PetscInt        supportSize, s;
2079 
2080         for (c = 0; c < closureSize*2; c += 2) {
2081           const PetscInt point = closure[c];
2082 
2083           for (d = 0; d < dim; ++d) {
2084             if ((point >= pStart[d]) && (point < pEnd[d])) {
2085               ierr = DMLabelSetValue(subpointMap, point, d);CHKERRQ(ierr);
2086               break;
2087             }
2088           }
2089         }
2090         ierr = DMPlexGetSupportSize(dm, face, &supportSize);CHKERRQ(ierr);
2091         ierr = DMPlexGetSupport(dm, face, &support);CHKERRQ(ierr);
2092         for (s = 0; s < supportSize; ++s) {
2093           ierr = DMLabelSetValue(subpointMap, support[s], dim);CHKERRQ(ierr);
2094         }
2095       }
2096       ierr = DMPlexRestoreTransitiveClosure(dm, face, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
2097     }
2098     ierr = DMPlexRestoreTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star);CHKERRQ(ierr);
2099   }
2100   if (subvertexIS) {ierr = ISRestoreIndices(subvertexIS, &subvertices);CHKERRQ(ierr);}
2101   ierr = ISDestroy(&subvertexIS);CHKERRQ(ierr);
2102   ierr = PetscFree3(pStart,pEnd,pMax);CHKERRQ(ierr);
2103   PetscFunctionReturn(0);
2104 }
2105 
2106 #undef __FUNCT__
2107 #define __FUNCT__ "DMPlexMarkCohesiveSubmesh_Uninterpolated"
2108 static PetscErrorCode DMPlexMarkCohesiveSubmesh_Uninterpolated(DM dm, PetscBool hasLagrange, const char labelname[], PetscInt value, DMLabel subpointMap, PetscInt *numFaces, PetscInt *nFV, PetscInt *subCells[], DM subdm)
2109 {
2110   DMLabel         label = NULL;
2111   const PetscInt *cone;
2112   PetscInt        dim, cMax, cEnd, c, subc = 0, p, coneSize;
2113   PetscErrorCode  ierr;
2114 
2115   PetscFunctionBegin;
2116   *numFaces = 0;
2117   *nFV = 0;
2118   if (labelname) {ierr = DMGetLabel(dm, labelname, &label);CHKERRQ(ierr);}
2119   *subCells = NULL;
2120   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
2121   ierr = DMPlexGetHeightStratum(dm, 0, NULL, &cEnd);CHKERRQ(ierr);
2122   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
2123   if (cMax < 0) PetscFunctionReturn(0);
2124   if (label) {
2125     for (c = cMax; c < cEnd; ++c) {
2126       PetscInt val;
2127 
2128       ierr = DMLabelGetValue(label, c, &val);CHKERRQ(ierr);
2129       if (val == value) {
2130         ++(*numFaces);
2131         ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
2132       }
2133     }
2134   } else {
2135     *numFaces = cEnd - cMax;
2136     ierr = DMPlexGetConeSize(dm, cMax, &coneSize);CHKERRQ(ierr);
2137   }
2138   *nFV = hasLagrange ? coneSize/3 : coneSize/2;
2139   ierr = PetscMalloc1(*numFaces *2, subCells);CHKERRQ(ierr);
2140   for (c = cMax; c < cEnd; ++c) {
2141     const PetscInt *cells;
2142     PetscInt        numCells;
2143 
2144     if (label) {
2145       PetscInt val;
2146 
2147       ierr = DMLabelGetValue(label, c, &val);CHKERRQ(ierr);
2148       if (val != value) continue;
2149     }
2150     ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
2151     for (p = 0; p < *nFV; ++p) {
2152       ierr = DMLabelSetValue(subpointMap, cone[p], 0);CHKERRQ(ierr);
2153     }
2154     /* Negative face */
2155     ierr = DMPlexGetJoin(dm, *nFV, cone, &numCells, &cells);CHKERRQ(ierr);
2156     /* Not true in parallel
2157     if (numCells != 2) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive cells should separate two cells"); */
2158     for (p = 0; p < numCells; ++p) {
2159       ierr = DMLabelSetValue(subpointMap, cells[p], 2);CHKERRQ(ierr);
2160       (*subCells)[subc++] = cells[p];
2161     }
2162     ierr = DMPlexRestoreJoin(dm, *nFV, cone, &numCells, &cells);CHKERRQ(ierr);
2163     /* Positive face is not included */
2164   }
2165   PetscFunctionReturn(0);
2166 }
2167 
2168 #undef __FUNCT__
2169 #define __FUNCT__ "DMPlexMarkCohesiveSubmesh_Interpolated"
2170 static PetscErrorCode DMPlexMarkCohesiveSubmesh_Interpolated(DM dm, DMLabel label, PetscInt value, DMLabel subpointMap, DM subdm)
2171 {
2172   PetscInt      *pStart, *pEnd;
2173   PetscInt       dim, cMax, cEnd, c, d;
2174   PetscErrorCode ierr;
2175 
2176   PetscFunctionBegin;
2177   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
2178   ierr = DMPlexGetHeightStratum(dm, 0, NULL, &cEnd);CHKERRQ(ierr);
2179   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
2180   if (cMax < 0) PetscFunctionReturn(0);
2181   ierr = PetscMalloc2(dim+1,&pStart,dim+1,&pEnd);CHKERRQ(ierr);
2182   for (d = 0; d <= dim; ++d) {ierr = DMPlexGetDepthStratum(dm, d, &pStart[d], &pEnd[d]);CHKERRQ(ierr);}
2183   for (c = cMax; c < cEnd; ++c) {
2184     const PetscInt *cone;
2185     PetscInt       *closure = NULL;
2186     PetscInt        fconeSize, coneSize, closureSize, cl, val;
2187 
2188     if (label) {
2189       ierr = DMLabelGetValue(label, c, &val);CHKERRQ(ierr);
2190       if (val != value) continue;
2191     }
2192     ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
2193     ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
2194     ierr = DMPlexGetConeSize(dm, cone[0], &fconeSize);CHKERRQ(ierr);
2195     if (coneSize != (fconeSize ? fconeSize : 1) + 2) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive cells should separate two cells");
2196     /* Negative face */
2197     ierr = DMPlexGetTransitiveClosure(dm, cone[0], PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
2198     for (cl = 0; cl < closureSize*2; cl += 2) {
2199       const PetscInt point = closure[cl];
2200 
2201       for (d = 0; d <= dim; ++d) {
2202         if ((point >= pStart[d]) && (point < pEnd[d])) {
2203           ierr = DMLabelSetValue(subpointMap, point, d);CHKERRQ(ierr);
2204           break;
2205         }
2206       }
2207     }
2208     ierr = DMPlexRestoreTransitiveClosure(dm, cone[0], PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
2209     /* Cells -- positive face is not included */
2210     for (cl = 0; cl < 1; ++cl) {
2211       const PetscInt *support;
2212       PetscInt        supportSize, s;
2213 
2214       ierr = DMPlexGetSupportSize(dm, cone[cl], &supportSize);CHKERRQ(ierr);
2215       /* if (supportSize != 2) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive faces should separate two cells"); */
2216       ierr = DMPlexGetSupport(dm, cone[cl], &support);CHKERRQ(ierr);
2217       for (s = 0; s < supportSize; ++s) {
2218         ierr = DMLabelSetValue(subpointMap, support[s], dim);CHKERRQ(ierr);
2219       }
2220     }
2221   }
2222   ierr = PetscFree2(pStart, pEnd);CHKERRQ(ierr);
2223   PetscFunctionReturn(0);
2224 }
2225 
2226 #undef __FUNCT__
2227 #define __FUNCT__ "DMPlexGetFaceOrientation"
2228 PetscErrorCode DMPlexGetFaceOrientation(DM dm, PetscInt cell, PetscInt numCorners, PetscInt indices[], PetscInt oppositeVertex, PetscInt origVertices[], PetscInt faceVertices[], PetscBool *posOriented)
2229 {
2230   MPI_Comm       comm;
2231   PetscBool      posOrient = PETSC_FALSE;
2232   const PetscInt debug     = 0;
2233   PetscInt       cellDim, faceSize, f;
2234   PetscErrorCode ierr;
2235 
2236   PetscFunctionBegin;
2237   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
2238   ierr = DMGetDimension(dm, &cellDim);CHKERRQ(ierr);
2239   if (debug) {PetscPrintf(comm, "cellDim: %d numCorners: %d\n", cellDim, numCorners);CHKERRQ(ierr);}
2240 
2241   if (cellDim == 1 && numCorners == 2) {
2242     /* Triangle */
2243     faceSize  = numCorners-1;
2244     posOrient = !(oppositeVertex%2) ? PETSC_TRUE : PETSC_FALSE;
2245   } else if (cellDim == 2 && numCorners == 3) {
2246     /* Triangle */
2247     faceSize  = numCorners-1;
2248     posOrient = !(oppositeVertex%2) ? PETSC_TRUE : PETSC_FALSE;
2249   } else if (cellDim == 3 && numCorners == 4) {
2250     /* Tetrahedron */
2251     faceSize  = numCorners-1;
2252     posOrient = (oppositeVertex%2) ? PETSC_TRUE : PETSC_FALSE;
2253   } else if (cellDim == 1 && numCorners == 3) {
2254     /* Quadratic line */
2255     faceSize  = 1;
2256     posOrient = PETSC_TRUE;
2257   } else if (cellDim == 2 && numCorners == 4) {
2258     /* Quads */
2259     faceSize = 2;
2260     if ((indices[1] > indices[0]) && (indices[1] - indices[0] == 1)) {
2261       posOrient = PETSC_TRUE;
2262     } else if ((indices[0] == 3) && (indices[1] == 0)) {
2263       posOrient = PETSC_TRUE;
2264     } else {
2265       if (((indices[0] > indices[1]) && (indices[0] - indices[1] == 1)) || ((indices[0] == 0) && (indices[1] == 3))) {
2266         posOrient = PETSC_FALSE;
2267       } else SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid quad crossedge");
2268     }
2269   } else if (cellDim == 2 && numCorners == 6) {
2270     /* Quadratic triangle (I hate this) */
2271     /* Edges are determined by the first 2 vertices (corners of edges) */
2272     const PetscInt faceSizeTri = 3;
2273     PetscInt       sortedIndices[3], i, iFace;
2274     PetscBool      found                    = PETSC_FALSE;
2275     PetscInt       faceVerticesTriSorted[9] = {
2276       0, 3,  4, /* bottom */
2277       1, 4,  5, /* right */
2278       2, 3,  5, /* left */
2279     };
2280     PetscInt       faceVerticesTri[9] = {
2281       0, 3,  4, /* bottom */
2282       1, 4,  5, /* right */
2283       2, 5,  3, /* left */
2284     };
2285 
2286     faceSize = faceSizeTri;
2287     for (i = 0; i < faceSizeTri; ++i) sortedIndices[i] = indices[i];
2288     ierr = PetscSortInt(faceSizeTri, sortedIndices);CHKERRQ(ierr);
2289     for (iFace = 0; iFace < 3; ++iFace) {
2290       const PetscInt ii = iFace*faceSizeTri;
2291       PetscInt       fVertex, cVertex;
2292 
2293       if ((sortedIndices[0] == faceVerticesTriSorted[ii+0]) &&
2294           (sortedIndices[1] == faceVerticesTriSorted[ii+1])) {
2295         for (fVertex = 0; fVertex < faceSizeTri; ++fVertex) {
2296           for (cVertex = 0; cVertex < faceSizeTri; ++cVertex) {
2297             if (indices[cVertex] == faceVerticesTri[ii+fVertex]) {
2298               faceVertices[fVertex] = origVertices[cVertex];
2299               break;
2300             }
2301           }
2302         }
2303         found = PETSC_TRUE;
2304         break;
2305       }
2306     }
2307     if (!found) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid tri crossface");
2308     if (posOriented) *posOriented = PETSC_TRUE;
2309     PetscFunctionReturn(0);
2310   } else if (cellDim == 2 && numCorners == 9) {
2311     /* Quadratic quad (I hate this) */
2312     /* Edges are determined by the first 2 vertices (corners of edges) */
2313     const PetscInt faceSizeQuad = 3;
2314     PetscInt       sortedIndices[3], i, iFace;
2315     PetscBool      found                      = PETSC_FALSE;
2316     PetscInt       faceVerticesQuadSorted[12] = {
2317       0, 1,  4, /* bottom */
2318       1, 2,  5, /* right */
2319       2, 3,  6, /* top */
2320       0, 3,  7, /* left */
2321     };
2322     PetscInt       faceVerticesQuad[12] = {
2323       0, 1,  4, /* bottom */
2324       1, 2,  5, /* right */
2325       2, 3,  6, /* top */
2326       3, 0,  7, /* left */
2327     };
2328 
2329     faceSize = faceSizeQuad;
2330     for (i = 0; i < faceSizeQuad; ++i) sortedIndices[i] = indices[i];
2331     ierr = PetscSortInt(faceSizeQuad, sortedIndices);CHKERRQ(ierr);
2332     for (iFace = 0; iFace < 4; ++iFace) {
2333       const PetscInt ii = iFace*faceSizeQuad;
2334       PetscInt       fVertex, cVertex;
2335 
2336       if ((sortedIndices[0] == faceVerticesQuadSorted[ii+0]) &&
2337           (sortedIndices[1] == faceVerticesQuadSorted[ii+1])) {
2338         for (fVertex = 0; fVertex < faceSizeQuad; ++fVertex) {
2339           for (cVertex = 0; cVertex < faceSizeQuad; ++cVertex) {
2340             if (indices[cVertex] == faceVerticesQuad[ii+fVertex]) {
2341               faceVertices[fVertex] = origVertices[cVertex];
2342               break;
2343             }
2344           }
2345         }
2346         found = PETSC_TRUE;
2347         break;
2348       }
2349     }
2350     if (!found) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid quad crossface");
2351     if (posOriented) *posOriented = PETSC_TRUE;
2352     PetscFunctionReturn(0);
2353   } else if (cellDim == 3 && numCorners == 8) {
2354     /* Hexes
2355        A hex is two oriented quads with the normal of the first
2356        pointing up at the second.
2357 
2358           7---6
2359          /|  /|
2360         4---5 |
2361         | 1-|-2
2362         |/  |/
2363         0---3
2364 
2365         Faces are determined by the first 4 vertices (corners of faces) */
2366     const PetscInt faceSizeHex = 4;
2367     PetscInt       sortedIndices[4], i, iFace;
2368     PetscBool      found                     = PETSC_FALSE;
2369     PetscInt       faceVerticesHexSorted[24] = {
2370       0, 1, 2, 3,  /* bottom */
2371       4, 5, 6, 7,  /* top */
2372       0, 3, 4, 5,  /* front */
2373       2, 3, 5, 6,  /* right */
2374       1, 2, 6, 7,  /* back */
2375       0, 1, 4, 7,  /* left */
2376     };
2377     PetscInt       faceVerticesHex[24] = {
2378       1, 2, 3, 0,  /* bottom */
2379       4, 5, 6, 7,  /* top */
2380       0, 3, 5, 4,  /* front */
2381       3, 2, 6, 5,  /* right */
2382       2, 1, 7, 6,  /* back */
2383       1, 0, 4, 7,  /* left */
2384     };
2385 
2386     faceSize = faceSizeHex;
2387     for (i = 0; i < faceSizeHex; ++i) sortedIndices[i] = indices[i];
2388     ierr = PetscSortInt(faceSizeHex, sortedIndices);CHKERRQ(ierr);
2389     for (iFace = 0; iFace < 6; ++iFace) {
2390       const PetscInt ii = iFace*faceSizeHex;
2391       PetscInt       fVertex, cVertex;
2392 
2393       if ((sortedIndices[0] == faceVerticesHexSorted[ii+0]) &&
2394           (sortedIndices[1] == faceVerticesHexSorted[ii+1]) &&
2395           (sortedIndices[2] == faceVerticesHexSorted[ii+2]) &&
2396           (sortedIndices[3] == faceVerticesHexSorted[ii+3])) {
2397         for (fVertex = 0; fVertex < faceSizeHex; ++fVertex) {
2398           for (cVertex = 0; cVertex < faceSizeHex; ++cVertex) {
2399             if (indices[cVertex] == faceVerticesHex[ii+fVertex]) {
2400               faceVertices[fVertex] = origVertices[cVertex];
2401               break;
2402             }
2403           }
2404         }
2405         found = PETSC_TRUE;
2406         break;
2407       }
2408     }
2409     if (!found) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid hex crossface");
2410     if (posOriented) *posOriented = PETSC_TRUE;
2411     PetscFunctionReturn(0);
2412   } else if (cellDim == 3 && numCorners == 10) {
2413     /* Quadratic tet */
2414     /* Faces are determined by the first 3 vertices (corners of faces) */
2415     const PetscInt faceSizeTet = 6;
2416     PetscInt       sortedIndices[6], i, iFace;
2417     PetscBool      found                     = PETSC_FALSE;
2418     PetscInt       faceVerticesTetSorted[24] = {
2419       0, 1, 2,  6, 7, 8, /* bottom */
2420       0, 3, 4,  6, 7, 9,  /* front */
2421       1, 4, 5,  7, 8, 9,  /* right */
2422       2, 3, 5,  6, 8, 9,  /* left */
2423     };
2424     PetscInt       faceVerticesTet[24] = {
2425       0, 1, 2,  6, 7, 8, /* bottom */
2426       0, 4, 3,  6, 7, 9,  /* front */
2427       1, 5, 4,  7, 8, 9,  /* right */
2428       2, 3, 5,  8, 6, 9,  /* left */
2429     };
2430 
2431     faceSize = faceSizeTet;
2432     for (i = 0; i < faceSizeTet; ++i) sortedIndices[i] = indices[i];
2433     ierr = PetscSortInt(faceSizeTet, sortedIndices);CHKERRQ(ierr);
2434     for (iFace=0; iFace < 4; ++iFace) {
2435       const PetscInt ii = iFace*faceSizeTet;
2436       PetscInt       fVertex, cVertex;
2437 
2438       if ((sortedIndices[0] == faceVerticesTetSorted[ii+0]) &&
2439           (sortedIndices[1] == faceVerticesTetSorted[ii+1]) &&
2440           (sortedIndices[2] == faceVerticesTetSorted[ii+2]) &&
2441           (sortedIndices[3] == faceVerticesTetSorted[ii+3])) {
2442         for (fVertex = 0; fVertex < faceSizeTet; ++fVertex) {
2443           for (cVertex = 0; cVertex < faceSizeTet; ++cVertex) {
2444             if (indices[cVertex] == faceVerticesTet[ii+fVertex]) {
2445               faceVertices[fVertex] = origVertices[cVertex];
2446               break;
2447             }
2448           }
2449         }
2450         found = PETSC_TRUE;
2451         break;
2452       }
2453     }
2454     if (!found) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid tet crossface");
2455     if (posOriented) *posOriented = PETSC_TRUE;
2456     PetscFunctionReturn(0);
2457   } else if (cellDim == 3 && numCorners == 27) {
2458     /* Quadratic hexes (I hate this)
2459        A hex is two oriented quads with the normal of the first
2460        pointing up at the second.
2461 
2462          7---6
2463         /|  /|
2464        4---5 |
2465        | 3-|-2
2466        |/  |/
2467        0---1
2468 
2469        Faces are determined by the first 4 vertices (corners of faces) */
2470     const PetscInt faceSizeQuadHex = 9;
2471     PetscInt       sortedIndices[9], i, iFace;
2472     PetscBool      found                         = PETSC_FALSE;
2473     PetscInt       faceVerticesQuadHexSorted[54] = {
2474       0, 1, 2, 3,  8, 9, 10, 11,  24, /* bottom */
2475       4, 5, 6, 7,  12, 13, 14, 15,  25, /* top */
2476       0, 1, 4, 5,  8, 12, 16, 17,  22, /* front */
2477       1, 2, 5, 6,  9, 13, 17, 18,  21, /* right */
2478       2, 3, 6, 7,  10, 14, 18, 19,  23, /* back */
2479       0, 3, 4, 7,  11, 15, 16, 19,  20, /* left */
2480     };
2481     PetscInt       faceVerticesQuadHex[54] = {
2482       3, 2, 1, 0,  10, 9, 8, 11,  24, /* bottom */
2483       4, 5, 6, 7,  12, 13, 14, 15,  25, /* top */
2484       0, 1, 5, 4,  8, 17, 12, 16,  22, /* front */
2485       1, 2, 6, 5,  9, 18, 13, 17,  21, /* right */
2486       2, 3, 7, 6,  10, 19, 14, 18,  23, /* back */
2487       3, 0, 4, 7,  11, 16, 15, 19,  20 /* left */
2488     };
2489 
2490     faceSize = faceSizeQuadHex;
2491     for (i = 0; i < faceSizeQuadHex; ++i) sortedIndices[i] = indices[i];
2492     ierr = PetscSortInt(faceSizeQuadHex, sortedIndices);CHKERRQ(ierr);
2493     for (iFace = 0; iFace < 6; ++iFace) {
2494       const PetscInt ii = iFace*faceSizeQuadHex;
2495       PetscInt       fVertex, cVertex;
2496 
2497       if ((sortedIndices[0] == faceVerticesQuadHexSorted[ii+0]) &&
2498           (sortedIndices[1] == faceVerticesQuadHexSorted[ii+1]) &&
2499           (sortedIndices[2] == faceVerticesQuadHexSorted[ii+2]) &&
2500           (sortedIndices[3] == faceVerticesQuadHexSorted[ii+3])) {
2501         for (fVertex = 0; fVertex < faceSizeQuadHex; ++fVertex) {
2502           for (cVertex = 0; cVertex < faceSizeQuadHex; ++cVertex) {
2503             if (indices[cVertex] == faceVerticesQuadHex[ii+fVertex]) {
2504               faceVertices[fVertex] = origVertices[cVertex];
2505               break;
2506             }
2507           }
2508         }
2509         found = PETSC_TRUE;
2510         break;
2511       }
2512     }
2513     if (!found) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid hex crossface");
2514     if (posOriented) *posOriented = PETSC_TRUE;
2515     PetscFunctionReturn(0);
2516   } else SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Unknown cell type for faceOrientation().");
2517   if (!posOrient) {
2518     if (debug) {ierr = PetscPrintf(comm, "  Reversing initial face orientation\n");CHKERRQ(ierr);}
2519     for (f = 0; f < faceSize; ++f) faceVertices[f] = origVertices[faceSize-1 - f];
2520   } else {
2521     if (debug) {ierr = PetscPrintf(comm, "  Keeping initial face orientation\n");CHKERRQ(ierr);}
2522     for (f = 0; f < faceSize; ++f) faceVertices[f] = origVertices[f];
2523   }
2524   if (posOriented) *posOriented = posOrient;
2525   PetscFunctionReturn(0);
2526 }
2527 
2528 #undef __FUNCT__
2529 #define __FUNCT__ "DMPlexGetOrientedFace"
2530 /*
2531     Given a cell and a face, as a set of vertices,
2532       return the oriented face, as a set of vertices, in faceVertices
2533     The orientation is such that the face normal points out of the cell
2534 */
2535 PetscErrorCode DMPlexGetOrientedFace(DM dm, PetscInt cell, PetscInt faceSize, const PetscInt face[], PetscInt numCorners, PetscInt indices[], PetscInt origVertices[], PetscInt faceVertices[], PetscBool *posOriented)
2536 {
2537   const PetscInt *cone = NULL;
2538   PetscInt        coneSize, v, f, v2;
2539   PetscInt        oppositeVertex = -1;
2540   PetscErrorCode  ierr;
2541 
2542   PetscFunctionBegin;
2543   ierr = DMPlexGetConeSize(dm, cell, &coneSize);CHKERRQ(ierr);
2544   ierr = DMPlexGetCone(dm, cell, &cone);CHKERRQ(ierr);
2545   for (v = 0, v2 = 0; v < coneSize; ++v) {
2546     PetscBool found = PETSC_FALSE;
2547 
2548     for (f = 0; f < faceSize; ++f) {
2549       if (face[f] == cone[v]) {
2550         found = PETSC_TRUE; break;
2551       }
2552     }
2553     if (found) {
2554       indices[v2]      = v;
2555       origVertices[v2] = cone[v];
2556       ++v2;
2557     } else {
2558       oppositeVertex = v;
2559     }
2560   }
2561   ierr = DMPlexGetFaceOrientation(dm, cell, numCorners, indices, oppositeVertex, origVertices, faceVertices, posOriented);CHKERRQ(ierr);
2562   PetscFunctionReturn(0);
2563 }
2564 
2565 #undef __FUNCT__
2566 #define __FUNCT__ "DMPlexInsertFace_Internal"
2567 /*
2568   DMPlexInsertFace_Internal - Puts a face into the mesh
2569 
2570   Not collective
2571 
2572   Input Parameters:
2573   + dm              - The DMPlex
2574   . numFaceVertex   - The number of vertices in the face
2575   . faceVertices    - The vertices in the face for dm
2576   . subfaceVertices - The vertices in the face for subdm
2577   . numCorners      - The number of vertices in the cell
2578   . cell            - A cell in dm containing the face
2579   . subcell         - A cell in subdm containing the face
2580   . firstFace       - First face in the mesh
2581   - newFacePoint    - Next face in the mesh
2582 
2583   Output Parameters:
2584   . newFacePoint - Contains next face point number on input, updated on output
2585 
2586   Level: developer
2587 */
2588 static PetscErrorCode DMPlexInsertFace_Internal(DM dm, DM subdm, PetscInt numFaceVertices, const PetscInt faceVertices[], const PetscInt subfaceVertices[], PetscInt numCorners, PetscInt cell, PetscInt subcell, PetscInt firstFace, PetscInt *newFacePoint)
2589 {
2590   MPI_Comm        comm;
2591   DM_Plex        *submesh = (DM_Plex*) subdm->data;
2592   const PetscInt *faces;
2593   PetscInt        numFaces, coneSize;
2594   PetscErrorCode  ierr;
2595 
2596   PetscFunctionBegin;
2597   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
2598   ierr = DMPlexGetConeSize(subdm, subcell, &coneSize);CHKERRQ(ierr);
2599   if (coneSize != 1) SETERRQ2(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cone size of cell %d is %d != 1", cell, coneSize);
2600 #if 0
2601   /* Cannot use this because support() has not been constructed yet */
2602   ierr = DMPlexGetJoin(subdm, numFaceVertices, subfaceVertices, &numFaces, &faces);CHKERRQ(ierr);
2603 #else
2604   {
2605     PetscInt f;
2606 
2607     numFaces = 0;
2608     ierr     = DMGetWorkArray(subdm, 1, PETSC_INT, (void **) &faces);CHKERRQ(ierr);
2609     for (f = firstFace; f < *newFacePoint; ++f) {
2610       PetscInt dof, off, d;
2611 
2612       ierr = PetscSectionGetDof(submesh->coneSection, f, &dof);CHKERRQ(ierr);
2613       ierr = PetscSectionGetOffset(submesh->coneSection, f, &off);CHKERRQ(ierr);
2614       /* Yes, I know this is quadratic, but I expect the sizes to be <5 */
2615       for (d = 0; d < dof; ++d) {
2616         const PetscInt p = submesh->cones[off+d];
2617         PetscInt       v;
2618 
2619         for (v = 0; v < numFaceVertices; ++v) {
2620           if (subfaceVertices[v] == p) break;
2621         }
2622         if (v == numFaceVertices) break;
2623       }
2624       if (d == dof) {
2625         numFaces               = 1;
2626         ((PetscInt*) faces)[0] = f;
2627       }
2628     }
2629   }
2630 #endif
2631   if (numFaces > 1) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Vertex set had %d faces, not one", numFaces);
2632   else if (numFaces == 1) {
2633     /* Add the other cell neighbor for this face */
2634     ierr = DMPlexSetCone(subdm, subcell, faces);CHKERRQ(ierr);
2635   } else {
2636     PetscInt *indices, *origVertices, *orientedVertices, *orientedSubVertices, v, ov;
2637     PetscBool posOriented;
2638 
2639     ierr                = DMGetWorkArray(subdm, 4*numFaceVertices * sizeof(PetscInt), PETSC_INT, &orientedVertices);CHKERRQ(ierr);
2640     origVertices        = &orientedVertices[numFaceVertices];
2641     indices             = &orientedVertices[numFaceVertices*2];
2642     orientedSubVertices = &orientedVertices[numFaceVertices*3];
2643     ierr                = DMPlexGetOrientedFace(dm, cell, numFaceVertices, faceVertices, numCorners, indices, origVertices, orientedVertices, &posOriented);CHKERRQ(ierr);
2644     /* TODO: I know that routine should return a permutation, not the indices */
2645     for (v = 0; v < numFaceVertices; ++v) {
2646       const PetscInt vertex = faceVertices[v], subvertex = subfaceVertices[v];
2647       for (ov = 0; ov < numFaceVertices; ++ov) {
2648         if (orientedVertices[ov] == vertex) {
2649           orientedSubVertices[ov] = subvertex;
2650           break;
2651         }
2652       }
2653       if (ov == numFaceVertices) SETERRQ1(comm, PETSC_ERR_PLIB, "Could not find face vertex %d in orientated set", vertex);
2654     }
2655     ierr = DMPlexSetCone(subdm, *newFacePoint, orientedSubVertices);CHKERRQ(ierr);
2656     ierr = DMPlexSetCone(subdm, subcell, newFacePoint);CHKERRQ(ierr);
2657     ierr = DMRestoreWorkArray(subdm, 4*numFaceVertices * sizeof(PetscInt), PETSC_INT, &orientedVertices);CHKERRQ(ierr);
2658     ++(*newFacePoint);
2659   }
2660 #if 0
2661   ierr = DMPlexRestoreJoin(subdm, numFaceVertices, subfaceVertices, &numFaces, &faces);CHKERRQ(ierr);
2662 #else
2663   ierr = DMRestoreWorkArray(subdm, 1, PETSC_INT, (void **) &faces);CHKERRQ(ierr);
2664 #endif
2665   PetscFunctionReturn(0);
2666 }
2667 
2668 #undef __FUNCT__
2669 #define __FUNCT__ "DMPlexCreateSubmesh_Uninterpolated"
2670 static PetscErrorCode DMPlexCreateSubmesh_Uninterpolated(DM dm, DMLabel vertexLabel, PetscInt value, DM subdm)
2671 {
2672   MPI_Comm        comm;
2673   DMLabel         subpointMap;
2674   IS              subvertexIS,  subcellIS;
2675   const PetscInt *subVertices, *subCells;
2676   PetscInt        numSubVertices, firstSubVertex, numSubCells;
2677   PetscInt       *subface, maxConeSize, numSubFaces = 0, firstSubFace, newFacePoint, nFV = 0;
2678   PetscInt        vStart, vEnd, c, f;
2679   PetscErrorCode  ierr;
2680 
2681   PetscFunctionBegin;
2682   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
2683   /* Create subpointMap which marks the submesh */
2684   ierr = DMLabelCreate("subpoint_map", &subpointMap);CHKERRQ(ierr);
2685   ierr = DMPlexSetSubpointMap(subdm, subpointMap);CHKERRQ(ierr);
2686   ierr = DMLabelDestroy(&subpointMap);CHKERRQ(ierr);
2687   if (vertexLabel) {ierr = DMPlexMarkSubmesh_Uninterpolated(dm, vertexLabel, value, subpointMap, &numSubFaces, &nFV, subdm);CHKERRQ(ierr);}
2688   /* Setup chart */
2689   ierr = DMLabelGetStratumSize(subpointMap, 0, &numSubVertices);CHKERRQ(ierr);
2690   ierr = DMLabelGetStratumSize(subpointMap, 2, &numSubCells);CHKERRQ(ierr);
2691   ierr = DMPlexSetChart(subdm, 0, numSubCells+numSubFaces+numSubVertices);CHKERRQ(ierr);
2692   ierr = DMPlexSetVTKCellHeight(subdm, 1);CHKERRQ(ierr);
2693   /* Set cone sizes */
2694   firstSubVertex = numSubCells;
2695   firstSubFace   = numSubCells+numSubVertices;
2696   newFacePoint   = firstSubFace;
2697   ierr = DMLabelGetStratumIS(subpointMap, 0, &subvertexIS);CHKERRQ(ierr);
2698   if (subvertexIS) {ierr = ISGetIndices(subvertexIS, &subVertices);CHKERRQ(ierr);}
2699   ierr = DMLabelGetStratumIS(subpointMap, 2, &subcellIS);CHKERRQ(ierr);
2700   if (subcellIS) {ierr = ISGetIndices(subcellIS, &subCells);CHKERRQ(ierr);}
2701   for (c = 0; c < numSubCells; ++c) {
2702     ierr = DMPlexSetConeSize(subdm, c, 1);CHKERRQ(ierr);
2703   }
2704   for (f = firstSubFace; f < firstSubFace+numSubFaces; ++f) {
2705     ierr = DMPlexSetConeSize(subdm, f, nFV);CHKERRQ(ierr);
2706   }
2707   ierr = DMSetUp(subdm);CHKERRQ(ierr);
2708   /* Create face cones */
2709   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
2710   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, NULL);CHKERRQ(ierr);
2711   ierr = DMGetWorkArray(subdm, maxConeSize, PETSC_INT, (void**) &subface);CHKERRQ(ierr);
2712   for (c = 0; c < numSubCells; ++c) {
2713     const PetscInt cell    = subCells[c];
2714     const PetscInt subcell = c;
2715     PetscInt      *closure = NULL;
2716     PetscInt       closureSize, cl, numCorners = 0, faceSize = 0;
2717 
2718     ierr = DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
2719     for (cl = 0; cl < closureSize*2; cl += 2) {
2720       const PetscInt point = closure[cl];
2721       PetscInt       subVertex;
2722 
2723       if ((point >= vStart) && (point < vEnd)) {
2724         ++numCorners;
2725         ierr = PetscFindInt(point, numSubVertices, subVertices, &subVertex);CHKERRQ(ierr);
2726         if (subVertex >= 0) {
2727           closure[faceSize] = point;
2728           subface[faceSize] = firstSubVertex+subVertex;
2729           ++faceSize;
2730         }
2731       }
2732     }
2733     if (faceSize > nFV) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Invalid submesh: Too many vertices %d of an element on the surface", faceSize);
2734     if (faceSize == nFV) {
2735       ierr = DMPlexInsertFace_Internal(dm, subdm, faceSize, closure, subface, numCorners, cell, subcell, firstSubFace, &newFacePoint);CHKERRQ(ierr);
2736     }
2737     ierr = DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
2738   }
2739   ierr = DMRestoreWorkArray(subdm, maxConeSize, PETSC_INT, (void**) &subface);CHKERRQ(ierr);
2740   ierr = DMPlexSymmetrize(subdm);CHKERRQ(ierr);
2741   ierr = DMPlexStratify(subdm);CHKERRQ(ierr);
2742   /* Build coordinates */
2743   {
2744     PetscSection coordSection, subCoordSection;
2745     Vec          coordinates, subCoordinates;
2746     PetscScalar *coords, *subCoords;
2747     PetscInt     numComp, coordSize, v;
2748     const char  *name;
2749 
2750     ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
2751     ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
2752     ierr = DMGetCoordinateSection(subdm, &subCoordSection);CHKERRQ(ierr);
2753     ierr = PetscSectionSetNumFields(subCoordSection, 1);CHKERRQ(ierr);
2754     ierr = PetscSectionGetFieldComponents(coordSection, 0, &numComp);CHKERRQ(ierr);
2755     ierr = PetscSectionSetFieldComponents(subCoordSection, 0, numComp);CHKERRQ(ierr);
2756     ierr = PetscSectionSetChart(subCoordSection, firstSubVertex, firstSubVertex+numSubVertices);CHKERRQ(ierr);
2757     for (v = 0; v < numSubVertices; ++v) {
2758       const PetscInt vertex    = subVertices[v];
2759       const PetscInt subvertex = firstSubVertex+v;
2760       PetscInt       dof;
2761 
2762       ierr = PetscSectionGetDof(coordSection, vertex, &dof);CHKERRQ(ierr);
2763       ierr = PetscSectionSetDof(subCoordSection, subvertex, dof);CHKERRQ(ierr);
2764       ierr = PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof);CHKERRQ(ierr);
2765     }
2766     ierr = PetscSectionSetUp(subCoordSection);CHKERRQ(ierr);
2767     ierr = PetscSectionGetStorageSize(subCoordSection, &coordSize);CHKERRQ(ierr);
2768     ierr = VecCreate(comm, &subCoordinates);CHKERRQ(ierr);
2769     ierr = PetscObjectGetName((PetscObject)coordinates,&name);CHKERRQ(ierr);
2770     ierr = PetscObjectSetName((PetscObject)subCoordinates,name);CHKERRQ(ierr);
2771     ierr = VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
2772     ierr = VecSetType(subCoordinates,VECSTANDARD);CHKERRQ(ierr);
2773     if (coordSize) {
2774       ierr = VecGetArray(coordinates,    &coords);CHKERRQ(ierr);
2775       ierr = VecGetArray(subCoordinates, &subCoords);CHKERRQ(ierr);
2776       for (v = 0; v < numSubVertices; ++v) {
2777         const PetscInt vertex    = subVertices[v];
2778         const PetscInt subvertex = firstSubVertex+v;
2779         PetscInt       dof, off, sdof, soff, d;
2780 
2781         ierr = PetscSectionGetDof(coordSection, vertex, &dof);CHKERRQ(ierr);
2782         ierr = PetscSectionGetOffset(coordSection, vertex, &off);CHKERRQ(ierr);
2783         ierr = PetscSectionGetDof(subCoordSection, subvertex, &sdof);CHKERRQ(ierr);
2784         ierr = PetscSectionGetOffset(subCoordSection, subvertex, &soff);CHKERRQ(ierr);
2785         if (dof != sdof) SETERRQ4(comm, PETSC_ERR_PLIB, "Coordinate dimension %d on subvertex %d, vertex %d should be %d", sdof, subvertex, vertex, dof);
2786         for (d = 0; d < dof; ++d) subCoords[soff+d] = coords[off+d];
2787       }
2788       ierr = VecRestoreArray(coordinates,    &coords);CHKERRQ(ierr);
2789       ierr = VecRestoreArray(subCoordinates, &subCoords);CHKERRQ(ierr);
2790     }
2791     ierr = DMSetCoordinatesLocal(subdm, subCoordinates);CHKERRQ(ierr);
2792     ierr = VecDestroy(&subCoordinates);CHKERRQ(ierr);
2793   }
2794   /* Cleanup */
2795   if (subvertexIS) {ierr = ISRestoreIndices(subvertexIS, &subVertices);CHKERRQ(ierr);}
2796   ierr = ISDestroy(&subvertexIS);CHKERRQ(ierr);
2797   if (subcellIS) {ierr = ISRestoreIndices(subcellIS, &subCells);CHKERRQ(ierr);}
2798   ierr = ISDestroy(&subcellIS);CHKERRQ(ierr);
2799   PetscFunctionReturn(0);
2800 }
2801 
2802 #undef __FUNCT__
2803 #define __FUNCT__ "DMPlexFilterPoint_Internal"
2804 PETSC_STATIC_INLINE PetscInt DMPlexFilterPoint_Internal(PetscInt point, PetscInt firstSubPoint, PetscInt numSubPoints, const PetscInt subPoints[])
2805 {
2806   PetscInt       subPoint;
2807   PetscErrorCode ierr;
2808 
2809   ierr = PetscFindInt(point, numSubPoints, subPoints, &subPoint); if (ierr < 0) return ierr;
2810   return subPoint < 0 ? subPoint : firstSubPoint+subPoint;
2811 }
2812 
2813 #undef __FUNCT__
2814 #define __FUNCT__ "DMPlexCreateSubmeshGeneric_Interpolated"
2815 static PetscErrorCode DMPlexCreateSubmeshGeneric_Interpolated(DM dm, DMLabel label, PetscInt value, PetscBool isCohesive, PetscInt cellHeight, DM subdm)
2816 {
2817   MPI_Comm         comm;
2818   DMLabel          subpointMap;
2819   IS              *subpointIS;
2820   const PetscInt **subpoints;
2821   PetscInt        *numSubPoints, *firstSubPoint, *coneNew, *orntNew;
2822   PetscInt         totSubPoints = 0, maxConeSize, dim, p, d, v;
2823   PetscErrorCode   ierr;
2824 
2825   PetscFunctionBegin;
2826   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
2827   /* Create subpointMap which marks the submesh */
2828   ierr = DMLabelCreate("subpoint_map", &subpointMap);CHKERRQ(ierr);
2829   ierr = DMPlexSetSubpointMap(subdm, subpointMap);CHKERRQ(ierr);
2830   if (cellHeight) {
2831     if (isCohesive) {ierr = DMPlexMarkCohesiveSubmesh_Interpolated(dm, label, value, subpointMap, subdm);CHKERRQ(ierr);}
2832     else            {ierr = DMPlexMarkSubmesh_Interpolated(dm, label, value, subpointMap, subdm);CHKERRQ(ierr);}
2833   } else {
2834     DMLabel         depth;
2835     IS              pointIS;
2836     const PetscInt *points;
2837     PetscInt        numPoints;
2838 
2839     ierr = DMPlexGetDepthLabel(dm, &depth);CHKERRQ(ierr);
2840     ierr = DMLabelGetStratumSize(label, value, &numPoints);CHKERRQ(ierr);
2841     ierr = DMLabelGetStratumIS(label, value, &pointIS);CHKERRQ(ierr);
2842     ierr = ISGetIndices(pointIS, &points);CHKERRQ(ierr);
2843     for (p = 0; p < numPoints; ++p) {
2844       PetscInt *closure = NULL;
2845       PetscInt  closureSize, c, pdim;
2846 
2847       ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
2848       for (c = 0; c < closureSize*2; c += 2) {
2849         ierr = DMLabelGetValue(depth, closure[c], &pdim);CHKERRQ(ierr);
2850         ierr = DMLabelSetValue(subpointMap, closure[c], pdim);CHKERRQ(ierr);
2851       }
2852       ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
2853     }
2854     ierr = ISRestoreIndices(pointIS, &points);CHKERRQ(ierr);
2855     ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
2856   }
2857   ierr = DMLabelDestroy(&subpointMap);CHKERRQ(ierr);
2858   /* Setup chart */
2859   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
2860   ierr = PetscMalloc4(dim+1,&numSubPoints,dim+1,&firstSubPoint,dim+1,&subpointIS,dim+1,&subpoints);CHKERRQ(ierr);
2861   for (d = 0; d <= dim; ++d) {
2862     ierr = DMLabelGetStratumSize(subpointMap, d, &numSubPoints[d]);CHKERRQ(ierr);
2863     totSubPoints += numSubPoints[d];
2864   }
2865   ierr = DMPlexSetChart(subdm, 0, totSubPoints);CHKERRQ(ierr);
2866   ierr = DMPlexSetVTKCellHeight(subdm, cellHeight);CHKERRQ(ierr);
2867   /* Set cone sizes */
2868   firstSubPoint[dim] = 0;
2869   firstSubPoint[0]   = firstSubPoint[dim] + numSubPoints[dim];
2870   if (dim > 1) {firstSubPoint[dim-1] = firstSubPoint[0]     + numSubPoints[0];}
2871   if (dim > 2) {firstSubPoint[dim-2] = firstSubPoint[dim-1] + numSubPoints[dim-1];}
2872   for (d = 0; d <= dim; ++d) {
2873     ierr = DMLabelGetStratumIS(subpointMap, d, &subpointIS[d]);CHKERRQ(ierr);
2874     if (subpointIS[d]) {ierr = ISGetIndices(subpointIS[d], &subpoints[d]);CHKERRQ(ierr);}
2875   }
2876   for (d = 0; d <= dim; ++d) {
2877     for (p = 0; p < numSubPoints[d]; ++p) {
2878       const PetscInt  point    = subpoints[d][p];
2879       const PetscInt  subpoint = firstSubPoint[d] + p;
2880       const PetscInt *cone;
2881       PetscInt        coneSize, coneSizeNew, c, val;
2882 
2883       ierr = DMPlexGetConeSize(dm, point, &coneSize);CHKERRQ(ierr);
2884       ierr = DMPlexSetConeSize(subdm, subpoint, coneSize);CHKERRQ(ierr);
2885       if (cellHeight && (d == dim)) {
2886         ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
2887         for (c = 0, coneSizeNew = 0; c < coneSize; ++c) {
2888           ierr = DMLabelGetValue(subpointMap, cone[c], &val);CHKERRQ(ierr);
2889           if (val >= 0) coneSizeNew++;
2890         }
2891         ierr = DMPlexSetConeSize(subdm, subpoint, coneSizeNew);CHKERRQ(ierr);
2892       }
2893     }
2894   }
2895   ierr = DMSetUp(subdm);CHKERRQ(ierr);
2896   /* Set cones */
2897   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, NULL);CHKERRQ(ierr);
2898   ierr = PetscMalloc2(maxConeSize,&coneNew,maxConeSize,&orntNew);CHKERRQ(ierr);
2899   for (d = 0; d <= dim; ++d) {
2900     for (p = 0; p < numSubPoints[d]; ++p) {
2901       const PetscInt  point    = subpoints[d][p];
2902       const PetscInt  subpoint = firstSubPoint[d] + p;
2903       const PetscInt *cone, *ornt;
2904       PetscInt        coneSize, subconeSize, coneSizeNew, c, subc;
2905 
2906       ierr = DMPlexGetConeSize(dm, point, &coneSize);CHKERRQ(ierr);
2907       ierr = DMPlexGetConeSize(subdm, subpoint, &subconeSize);CHKERRQ(ierr);
2908       ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
2909       ierr = DMPlexGetConeOrientation(dm, point, &ornt);CHKERRQ(ierr);
2910       for (c = 0, coneSizeNew = 0; c < coneSize; ++c) {
2911         ierr = PetscFindInt(cone[c], numSubPoints[d-1], subpoints[d-1], &subc);CHKERRQ(ierr);
2912         if (subc >= 0) {
2913           coneNew[coneSizeNew] = firstSubPoint[d-1] + subc;
2914           orntNew[coneSizeNew] = ornt[c];
2915           ++coneSizeNew;
2916         }
2917       }
2918       if (coneSizeNew != subconeSize) SETERRQ2(comm, PETSC_ERR_PLIB, "Number of cone points located %d does not match subcone size %d", coneSizeNew, subconeSize);
2919       ierr = DMPlexSetCone(subdm, subpoint, coneNew);CHKERRQ(ierr);
2920       ierr = DMPlexSetConeOrientation(subdm, subpoint, orntNew);CHKERRQ(ierr);
2921     }
2922   }
2923   ierr = PetscFree2(coneNew,orntNew);CHKERRQ(ierr);
2924   ierr = DMPlexSymmetrize(subdm);CHKERRQ(ierr);
2925   ierr = DMPlexStratify(subdm);CHKERRQ(ierr);
2926   /* Build coordinates */
2927   {
2928     PetscSection coordSection, subCoordSection;
2929     Vec          coordinates, subCoordinates;
2930     PetscScalar *coords, *subCoords;
2931     PetscInt     numComp, coordSize;
2932     const char  *name;
2933 
2934     ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
2935     ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
2936     ierr = DMGetCoordinateSection(subdm, &subCoordSection);CHKERRQ(ierr);
2937     ierr = PetscSectionSetNumFields(subCoordSection, 1);CHKERRQ(ierr);
2938     ierr = PetscSectionGetFieldComponents(coordSection, 0, &numComp);CHKERRQ(ierr);
2939     ierr = PetscSectionSetFieldComponents(subCoordSection, 0, numComp);CHKERRQ(ierr);
2940     ierr = PetscSectionSetChart(subCoordSection, firstSubPoint[0], firstSubPoint[0]+numSubPoints[0]);CHKERRQ(ierr);
2941     for (v = 0; v < numSubPoints[0]; ++v) {
2942       const PetscInt vertex    = subpoints[0][v];
2943       const PetscInt subvertex = firstSubPoint[0]+v;
2944       PetscInt       dof;
2945 
2946       ierr = PetscSectionGetDof(coordSection, vertex, &dof);CHKERRQ(ierr);
2947       ierr = PetscSectionSetDof(subCoordSection, subvertex, dof);CHKERRQ(ierr);
2948       ierr = PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof);CHKERRQ(ierr);
2949     }
2950     ierr = PetscSectionSetUp(subCoordSection);CHKERRQ(ierr);
2951     ierr = PetscSectionGetStorageSize(subCoordSection, &coordSize);CHKERRQ(ierr);
2952     ierr = VecCreate(comm, &subCoordinates);CHKERRQ(ierr);
2953     ierr = PetscObjectGetName((PetscObject)coordinates,&name);CHKERRQ(ierr);
2954     ierr = PetscObjectSetName((PetscObject)subCoordinates,name);CHKERRQ(ierr);
2955     ierr = VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
2956     ierr = VecSetType(subCoordinates,VECSTANDARD);CHKERRQ(ierr);
2957     ierr = VecGetArray(coordinates,    &coords);CHKERRQ(ierr);
2958     ierr = VecGetArray(subCoordinates, &subCoords);CHKERRQ(ierr);
2959     for (v = 0; v < numSubPoints[0]; ++v) {
2960       const PetscInt vertex    = subpoints[0][v];
2961       const PetscInt subvertex = firstSubPoint[0]+v;
2962       PetscInt dof, off, sdof, soff, d;
2963 
2964       ierr = PetscSectionGetDof(coordSection, vertex, &dof);CHKERRQ(ierr);
2965       ierr = PetscSectionGetOffset(coordSection, vertex, &off);CHKERRQ(ierr);
2966       ierr = PetscSectionGetDof(subCoordSection, subvertex, &sdof);CHKERRQ(ierr);
2967       ierr = PetscSectionGetOffset(subCoordSection, subvertex, &soff);CHKERRQ(ierr);
2968       if (dof != sdof) SETERRQ4(comm, PETSC_ERR_PLIB, "Coordinate dimension %d on subvertex %d, vertex %d should be %d", sdof, subvertex, vertex, dof);
2969       for (d = 0; d < dof; ++d) subCoords[soff+d] = coords[off+d];
2970     }
2971     ierr = VecRestoreArray(coordinates,    &coords);CHKERRQ(ierr);
2972     ierr = VecRestoreArray(subCoordinates, &subCoords);CHKERRQ(ierr);
2973     ierr = DMSetCoordinatesLocal(subdm, subCoordinates);CHKERRQ(ierr);
2974     ierr = VecDestroy(&subCoordinates);CHKERRQ(ierr);
2975   }
2976   /* Build SF: We need this complexity because subpoints might not be selected on the owning process */
2977   {
2978     PetscSF            sfPoint, sfPointSub;
2979     IS                 subpIS;
2980     const PetscSFNode *remotePoints;
2981     PetscSFNode       *sremotePoints, *newLocalPoints, *newOwners;
2982     const PetscInt    *localPoints, *subpoints;
2983     PetscInt          *slocalPoints;
2984     PetscInt           numRoots, numLeaves, numSubpoints = 0, numSubroots, numSubleaves = 0, l, sl, ll, pStart, pEnd, p;
2985     PetscMPIInt        rank;
2986 
2987     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRQ(ierr);
2988     ierr = DMGetPointSF(dm, &sfPoint);CHKERRQ(ierr);
2989     ierr = DMGetPointSF(subdm, &sfPointSub);CHKERRQ(ierr);
2990     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
2991     ierr = DMPlexGetChart(subdm, NULL, &numSubroots);CHKERRQ(ierr);
2992     ierr = DMPlexCreateSubpointIS(subdm, &subpIS);CHKERRQ(ierr);
2993     if (subpIS) {
2994       ierr = ISGetIndices(subpIS, &subpoints);CHKERRQ(ierr);
2995       ierr = ISGetLocalSize(subpIS, &numSubpoints);CHKERRQ(ierr);
2996     }
2997     ierr = PetscSFGetGraph(sfPoint, &numRoots, &numLeaves, &localPoints, &remotePoints);CHKERRQ(ierr);
2998     if (numRoots >= 0) {
2999       ierr = PetscMalloc2(pEnd-pStart,&newLocalPoints,numRoots,&newOwners);CHKERRQ(ierr);
3000       for (p = 0; p < pEnd-pStart; ++p) {
3001         newLocalPoints[p].rank  = -2;
3002         newLocalPoints[p].index = -2;
3003       }
3004       /* Set subleaves */
3005       for (l = 0; l < numLeaves; ++l) {
3006         const PetscInt point    = localPoints[l];
3007         const PetscInt subpoint = DMPlexFilterPoint_Internal(point, 0, numSubpoints, subpoints);
3008 
3009         if (subpoint < 0) continue;
3010         newLocalPoints[point-pStart].rank  = rank;
3011         newLocalPoints[point-pStart].index = subpoint;
3012         ++numSubleaves;
3013       }
3014       /* Must put in owned subpoints */
3015       for (p = pStart; p < pEnd; ++p) {
3016         const PetscInt subpoint = DMPlexFilterPoint_Internal(p, 0, numSubpoints, subpoints);
3017 
3018         if (subpoint < 0) {
3019           newOwners[p-pStart].rank  = -3;
3020           newOwners[p-pStart].index = -3;
3021         } else {
3022           newOwners[p-pStart].rank  = rank;
3023           newOwners[p-pStart].index = subpoint;
3024         }
3025       }
3026       ierr = PetscSFReduceBegin(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC);CHKERRQ(ierr);
3027       ierr = PetscSFReduceEnd(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC);CHKERRQ(ierr);
3028       ierr = PetscSFBcastBegin(sfPoint, MPIU_2INT, newOwners, newLocalPoints);CHKERRQ(ierr);
3029       ierr = PetscSFBcastEnd(sfPoint, MPIU_2INT, newOwners, newLocalPoints);CHKERRQ(ierr);
3030       ierr = PetscMalloc1(numSubleaves, &slocalPoints);CHKERRQ(ierr);
3031       ierr = PetscMalloc1(numSubleaves, &sremotePoints);CHKERRQ(ierr);
3032       for (l = 0, sl = 0, ll = 0; l < numLeaves; ++l) {
3033         const PetscInt point    = localPoints[l];
3034         const PetscInt subpoint = DMPlexFilterPoint_Internal(point, 0, numSubpoints, subpoints);
3035 
3036         if (subpoint < 0) continue;
3037         if (newLocalPoints[point].rank == rank) {++ll; continue;}
3038         slocalPoints[sl]        = subpoint;
3039         sremotePoints[sl].rank  = newLocalPoints[point].rank;
3040         sremotePoints[sl].index = newLocalPoints[point].index;
3041         if (sremotePoints[sl].rank  < 0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote rank for local point %d", point);
3042         if (sremotePoints[sl].index < 0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote subpoint for local point %d", point);
3043         ++sl;
3044       }
3045       if (sl + ll != numSubleaves) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatch in number of subleaves %d + %d != %d", sl, ll, numSubleaves);
3046       ierr = PetscFree2(newLocalPoints,newOwners);CHKERRQ(ierr);
3047       ierr = PetscSFSetGraph(sfPointSub, numSubroots, sl, slocalPoints, PETSC_OWN_POINTER, sremotePoints, PETSC_OWN_POINTER);CHKERRQ(ierr);
3048     }
3049     if (subpIS) {
3050       ierr = ISRestoreIndices(subpIS, &subpoints);CHKERRQ(ierr);
3051       ierr = ISDestroy(&subpIS);CHKERRQ(ierr);
3052     }
3053   }
3054   /* Cleanup */
3055   for (d = 0; d <= dim; ++d) {
3056     if (subpointIS[d]) {ierr = ISRestoreIndices(subpointIS[d], &subpoints[d]);CHKERRQ(ierr);}
3057     ierr = ISDestroy(&subpointIS[d]);CHKERRQ(ierr);
3058   }
3059   ierr = PetscFree4(numSubPoints,firstSubPoint,subpointIS,subpoints);CHKERRQ(ierr);
3060   PetscFunctionReturn(0);
3061 }
3062 
3063 #undef __FUNCT__
3064 #define __FUNCT__ "DMPlexCreateSubmesh_Interpolated"
3065 static PetscErrorCode DMPlexCreateSubmesh_Interpolated(DM dm, DMLabel vertexLabel, PetscInt value, DM subdm)
3066 {
3067   PetscErrorCode ierr;
3068 
3069   PetscFunctionBegin;
3070   ierr = DMPlexCreateSubmeshGeneric_Interpolated(dm, vertexLabel, value, PETSC_FALSE, 1, subdm);CHKERRQ(ierr);
3071   PetscFunctionReturn(0);
3072 }
3073 
3074 #undef __FUNCT__
3075 #define __FUNCT__ "DMPlexCreateSubmesh"
3076 /*@
3077   DMPlexCreateSubmesh - Extract a hypersurface from the mesh using vertices defined by a label
3078 
3079   Input Parameters:
3080 + dm           - The original mesh
3081 . vertexLabel  - The DMLabel marking vertices contained in the surface
3082 - value        - The label value to use
3083 
3084   Output Parameter:
3085 . subdm - The surface mesh
3086 
3087   Note: This function produces a DMLabel mapping original points in the submesh to their depth. This can be obtained using DMPlexGetSubpointMap().
3088 
3089   Level: developer
3090 
3091 .seealso: DMPlexGetSubpointMap(), DMGetLabel(), DMLabelSetValue()
3092 @*/
3093 PetscErrorCode DMPlexCreateSubmesh(DM dm, DMLabel vertexLabel, PetscInt value, DM *subdm)
3094 {
3095   PetscInt       dim, depth;
3096   PetscErrorCode ierr;
3097 
3098   PetscFunctionBegin;
3099   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3100   PetscValidPointer(subdm, 3);
3101   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
3102   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3103   ierr = DMCreate(PetscObjectComm((PetscObject)dm), subdm);CHKERRQ(ierr);
3104   ierr = DMSetType(*subdm, DMPLEX);CHKERRQ(ierr);
3105   ierr = DMSetDimension(*subdm, dim-1);CHKERRQ(ierr);
3106   if (depth == dim) {
3107     ierr = DMPlexCreateSubmesh_Interpolated(dm, vertexLabel, value, *subdm);CHKERRQ(ierr);
3108   } else {
3109     ierr = DMPlexCreateSubmesh_Uninterpolated(dm, vertexLabel, value, *subdm);CHKERRQ(ierr);
3110   }
3111   PetscFunctionReturn(0);
3112 }
3113 
3114 #undef __FUNCT__
3115 #define __FUNCT__ "DMPlexCreateCohesiveSubmesh_Uninterpolated"
3116 static PetscErrorCode DMPlexCreateCohesiveSubmesh_Uninterpolated(DM dm, PetscBool hasLagrange, const char label[], PetscInt value, DM subdm)
3117 {
3118   MPI_Comm        comm;
3119   DMLabel         subpointMap;
3120   IS              subvertexIS;
3121   const PetscInt *subVertices;
3122   PetscInt        numSubVertices, firstSubVertex, numSubCells, *subCells = NULL;
3123   PetscInt       *subface, maxConeSize, numSubFaces, firstSubFace, newFacePoint, nFV;
3124   PetscInt        cMax, c, f;
3125   PetscErrorCode  ierr;
3126 
3127   PetscFunctionBegin;
3128   ierr = PetscObjectGetComm((PetscObject)dm, &comm);CHKERRQ(ierr);
3129   /* Create subpointMap which marks the submesh */
3130   ierr = DMLabelCreate("subpoint_map", &subpointMap);CHKERRQ(ierr);
3131   ierr = DMPlexSetSubpointMap(subdm, subpointMap);CHKERRQ(ierr);
3132   ierr = DMLabelDestroy(&subpointMap);CHKERRQ(ierr);
3133   ierr = DMPlexMarkCohesiveSubmesh_Uninterpolated(dm, hasLagrange, label, value, subpointMap, &numSubFaces, &nFV, &subCells, subdm);CHKERRQ(ierr);
3134   /* Setup chart */
3135   ierr = DMLabelGetStratumSize(subpointMap, 0, &numSubVertices);CHKERRQ(ierr);
3136   ierr = DMLabelGetStratumSize(subpointMap, 2, &numSubCells);CHKERRQ(ierr);
3137   ierr = DMPlexSetChart(subdm, 0, numSubCells+numSubFaces+numSubVertices);CHKERRQ(ierr);
3138   ierr = DMPlexSetVTKCellHeight(subdm, 1);CHKERRQ(ierr);
3139   /* Set cone sizes */
3140   firstSubVertex = numSubCells;
3141   firstSubFace   = numSubCells+numSubVertices;
3142   newFacePoint   = firstSubFace;
3143   ierr = DMLabelGetStratumIS(subpointMap, 0, &subvertexIS);CHKERRQ(ierr);
3144   if (subvertexIS) {ierr = ISGetIndices(subvertexIS, &subVertices);CHKERRQ(ierr);}
3145   for (c = 0; c < numSubCells; ++c) {
3146     ierr = DMPlexSetConeSize(subdm, c, 1);CHKERRQ(ierr);
3147   }
3148   for (f = firstSubFace; f < firstSubFace+numSubFaces; ++f) {
3149     ierr = DMPlexSetConeSize(subdm, f, nFV);CHKERRQ(ierr);
3150   }
3151   ierr = DMSetUp(subdm);CHKERRQ(ierr);
3152   /* Create face cones */
3153   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, NULL);CHKERRQ(ierr);
3154   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
3155   ierr = DMGetWorkArray(subdm, maxConeSize, PETSC_INT, (void**) &subface);CHKERRQ(ierr);
3156   for (c = 0; c < numSubCells; ++c) {
3157     const PetscInt  cell    = subCells[c];
3158     const PetscInt  subcell = c;
3159     const PetscInt *cone, *cells;
3160     PetscInt        numCells, subVertex, p, v;
3161 
3162     if (cell < cMax) continue;
3163     ierr = DMPlexGetCone(dm, cell, &cone);CHKERRQ(ierr);
3164     for (v = 0; v < nFV; ++v) {
3165       ierr = PetscFindInt(cone[v], numSubVertices, subVertices, &subVertex);CHKERRQ(ierr);
3166       subface[v] = firstSubVertex+subVertex;
3167     }
3168     ierr = DMPlexSetCone(subdm, newFacePoint, subface);CHKERRQ(ierr);
3169     ierr = DMPlexSetCone(subdm, subcell, &newFacePoint);CHKERRQ(ierr);
3170     ierr = DMPlexGetJoin(dm, nFV, cone, &numCells, &cells);CHKERRQ(ierr);
3171     /* Not true in parallel
3172     if (numCells != 2) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive cells should separate two cells"); */
3173     for (p = 0; p < numCells; ++p) {
3174       PetscInt negsubcell;
3175 
3176       if (cells[p] >= cMax) continue;
3177       /* I know this is a crap search */
3178       for (negsubcell = 0; negsubcell < numSubCells; ++negsubcell) {
3179         if (subCells[negsubcell] == cells[p]) break;
3180       }
3181       if (negsubcell == numSubCells) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not find negative face neighbor for cohesive cell %d", cell);
3182       ierr = DMPlexSetCone(subdm, negsubcell, &newFacePoint);CHKERRQ(ierr);
3183     }
3184     ierr = DMPlexRestoreJoin(dm, nFV, cone, &numCells, &cells);CHKERRQ(ierr);
3185     ++newFacePoint;
3186   }
3187   ierr = DMRestoreWorkArray(subdm, maxConeSize, PETSC_INT, (void**) &subface);CHKERRQ(ierr);
3188   ierr = DMPlexSymmetrize(subdm);CHKERRQ(ierr);
3189   ierr = DMPlexStratify(subdm);CHKERRQ(ierr);
3190   /* Build coordinates */
3191   {
3192     PetscSection coordSection, subCoordSection;
3193     Vec          coordinates, subCoordinates;
3194     PetscScalar *coords, *subCoords;
3195     PetscInt     numComp, coordSize, v;
3196     const char  *name;
3197 
3198     ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
3199     ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
3200     ierr = DMGetCoordinateSection(subdm, &subCoordSection);CHKERRQ(ierr);
3201     ierr = PetscSectionSetNumFields(subCoordSection, 1);CHKERRQ(ierr);
3202     ierr = PetscSectionGetFieldComponents(coordSection, 0, &numComp);CHKERRQ(ierr);
3203     ierr = PetscSectionSetFieldComponents(subCoordSection, 0, numComp);CHKERRQ(ierr);
3204     ierr = PetscSectionSetChart(subCoordSection, firstSubVertex, firstSubVertex+numSubVertices);CHKERRQ(ierr);
3205     for (v = 0; v < numSubVertices; ++v) {
3206       const PetscInt vertex    = subVertices[v];
3207       const PetscInt subvertex = firstSubVertex+v;
3208       PetscInt       dof;
3209 
3210       ierr = PetscSectionGetDof(coordSection, vertex, &dof);CHKERRQ(ierr);
3211       ierr = PetscSectionSetDof(subCoordSection, subvertex, dof);CHKERRQ(ierr);
3212       ierr = PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof);CHKERRQ(ierr);
3213     }
3214     ierr = PetscSectionSetUp(subCoordSection);CHKERRQ(ierr);
3215     ierr = PetscSectionGetStorageSize(subCoordSection, &coordSize);CHKERRQ(ierr);
3216     ierr = VecCreate(comm, &subCoordinates);CHKERRQ(ierr);
3217     ierr = PetscObjectGetName((PetscObject)coordinates,&name);CHKERRQ(ierr);
3218     ierr = PetscObjectSetName((PetscObject)subCoordinates,name);CHKERRQ(ierr);
3219     ierr = VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
3220     ierr = VecSetType(subCoordinates,VECSTANDARD);CHKERRQ(ierr);
3221     ierr = VecGetArray(coordinates,    &coords);CHKERRQ(ierr);
3222     ierr = VecGetArray(subCoordinates, &subCoords);CHKERRQ(ierr);
3223     for (v = 0; v < numSubVertices; ++v) {
3224       const PetscInt vertex    = subVertices[v];
3225       const PetscInt subvertex = firstSubVertex+v;
3226       PetscInt       dof, off, sdof, soff, d;
3227 
3228       ierr = PetscSectionGetDof(coordSection, vertex, &dof);CHKERRQ(ierr);
3229       ierr = PetscSectionGetOffset(coordSection, vertex, &off);CHKERRQ(ierr);
3230       ierr = PetscSectionGetDof(subCoordSection, subvertex, &sdof);CHKERRQ(ierr);
3231       ierr = PetscSectionGetOffset(subCoordSection, subvertex, &soff);CHKERRQ(ierr);
3232       if (dof != sdof) SETERRQ4(comm, PETSC_ERR_PLIB, "Coordinate dimension %d on subvertex %d, vertex %d should be %d", sdof, subvertex, vertex, dof);
3233       for (d = 0; d < dof; ++d) subCoords[soff+d] = coords[off+d];
3234     }
3235     ierr = VecRestoreArray(coordinates,    &coords);CHKERRQ(ierr);
3236     ierr = VecRestoreArray(subCoordinates, &subCoords);CHKERRQ(ierr);
3237     ierr = DMSetCoordinatesLocal(subdm, subCoordinates);CHKERRQ(ierr);
3238     ierr = VecDestroy(&subCoordinates);CHKERRQ(ierr);
3239   }
3240   /* Build SF */
3241   CHKMEMQ;
3242   {
3243     PetscSF            sfPoint, sfPointSub;
3244     const PetscSFNode *remotePoints;
3245     PetscSFNode       *sremotePoints, *newLocalPoints, *newOwners;
3246     const PetscInt    *localPoints;
3247     PetscInt          *slocalPoints;
3248     PetscInt           numRoots, numLeaves, numSubRoots = numSubCells+numSubFaces+numSubVertices, numSubLeaves = 0, l, sl, ll, pStart, pEnd, p, vStart, vEnd;
3249     PetscMPIInt        rank;
3250 
3251     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRQ(ierr);
3252     ierr = DMGetPointSF(dm, &sfPoint);CHKERRQ(ierr);
3253     ierr = DMGetPointSF(subdm, &sfPointSub);CHKERRQ(ierr);
3254     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
3255     ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
3256     ierr = PetscSFGetGraph(sfPoint, &numRoots, &numLeaves, &localPoints, &remotePoints);CHKERRQ(ierr);
3257     if (numRoots >= 0) {
3258       /* Only vertices should be shared */
3259       ierr = PetscMalloc2(pEnd-pStart,&newLocalPoints,numRoots,&newOwners);CHKERRQ(ierr);
3260       for (p = 0; p < pEnd-pStart; ++p) {
3261         newLocalPoints[p].rank  = -2;
3262         newLocalPoints[p].index = -2;
3263       }
3264       /* Set subleaves */
3265       for (l = 0; l < numLeaves; ++l) {
3266         const PetscInt point    = localPoints[l];
3267         const PetscInt subPoint = DMPlexFilterPoint_Internal(point, firstSubVertex, numSubVertices, subVertices);
3268 
3269         if ((point < vStart) && (point >= vEnd)) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Should not be mapping anything but vertices, %d", point);
3270         if (subPoint < 0) continue;
3271         newLocalPoints[point-pStart].rank  = rank;
3272         newLocalPoints[point-pStart].index = subPoint;
3273         ++numSubLeaves;
3274       }
3275       /* Must put in owned subpoints */
3276       for (p = pStart; p < pEnd; ++p) {
3277         const PetscInt subPoint = DMPlexFilterPoint_Internal(p, firstSubVertex, numSubVertices, subVertices);
3278 
3279         if (subPoint < 0) {
3280           newOwners[p-pStart].rank  = -3;
3281           newOwners[p-pStart].index = -3;
3282         } else {
3283           newOwners[p-pStart].rank  = rank;
3284           newOwners[p-pStart].index = subPoint;
3285         }
3286       }
3287       ierr = PetscSFReduceBegin(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC);CHKERRQ(ierr);
3288       ierr = PetscSFReduceEnd(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC);CHKERRQ(ierr);
3289       ierr = PetscSFBcastBegin(sfPoint, MPIU_2INT, newOwners, newLocalPoints);CHKERRQ(ierr);
3290       ierr = PetscSFBcastEnd(sfPoint, MPIU_2INT, newOwners, newLocalPoints);CHKERRQ(ierr);
3291       ierr = PetscMalloc1(numSubLeaves,    &slocalPoints);CHKERRQ(ierr);
3292       ierr = PetscMalloc1(numSubLeaves, &sremotePoints);CHKERRQ(ierr);
3293       for (l = 0, sl = 0, ll = 0; l < numLeaves; ++l) {
3294         const PetscInt point    = localPoints[l];
3295         const PetscInt subPoint = DMPlexFilterPoint_Internal(point, firstSubVertex, numSubVertices, subVertices);
3296 
3297         if (subPoint < 0) continue;
3298         if (newLocalPoints[point].rank == rank) {++ll; continue;}
3299         slocalPoints[sl]        = subPoint;
3300         sremotePoints[sl].rank  = newLocalPoints[point].rank;
3301         sremotePoints[sl].index = newLocalPoints[point].index;
3302         if (sremotePoints[sl].rank  < 0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote rank for local point %d", point);
3303         if (sremotePoints[sl].index < 0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote subpoint for local point %d", point);
3304         ++sl;
3305       }
3306       ierr = PetscFree2(newLocalPoints,newOwners);CHKERRQ(ierr);
3307       if (sl + ll != numSubLeaves) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatch in number of subleaves %d + %d != %d", sl, ll, numSubLeaves);
3308       ierr = PetscSFSetGraph(sfPointSub, numSubRoots, sl, slocalPoints, PETSC_OWN_POINTER, sremotePoints, PETSC_OWN_POINTER);CHKERRQ(ierr);
3309     }
3310   }
3311   CHKMEMQ;
3312   /* Cleanup */
3313   if (subvertexIS) {ierr = ISRestoreIndices(subvertexIS, &subVertices);CHKERRQ(ierr);}
3314   ierr = ISDestroy(&subvertexIS);CHKERRQ(ierr);
3315   ierr = PetscFree(subCells);CHKERRQ(ierr);
3316   PetscFunctionReturn(0);
3317 }
3318 
3319 #undef __FUNCT__
3320 #define __FUNCT__ "DMPlexCreateCohesiveSubmesh_Interpolated"
3321 static PetscErrorCode DMPlexCreateCohesiveSubmesh_Interpolated(DM dm, const char labelname[], PetscInt value, DM subdm)
3322 {
3323   DMLabel        label = NULL;
3324   PetscErrorCode ierr;
3325 
3326   PetscFunctionBegin;
3327   if (labelname) {ierr = DMGetLabel(dm, labelname, &label);CHKERRQ(ierr);}
3328   ierr = DMPlexCreateSubmeshGeneric_Interpolated(dm, label, value, PETSC_TRUE, 1, subdm);CHKERRQ(ierr);
3329   PetscFunctionReturn(0);
3330 }
3331 
3332 #undef __FUNCT__
3333 #define __FUNCT__ "DMPlexCreateCohesiveSubmesh"
3334 /*
3335   DMPlexCreateCohesiveSubmesh - Extract from a mesh with cohesive cells the hypersurface defined by one face of the cells. Optionally, a Label an be given to restrict the cells.
3336 
3337   Input Parameters:
3338 + dm          - The original mesh
3339 . hasLagrange - The mesh has Lagrange unknowns in the cohesive cells
3340 . label       - A label name, or NULL
3341 - value  - A label value
3342 
3343   Output Parameter:
3344 . subdm - The surface mesh
3345 
3346   Note: This function produces a DMLabel mapping original points in the submesh to their depth. This can be obtained using DMPlexGetSubpointMap().
3347 
3348   Level: developer
3349 
3350 .seealso: DMPlexGetSubpointMap(), DMPlexCreateSubmesh()
3351 */
3352 PetscErrorCode DMPlexCreateCohesiveSubmesh(DM dm, PetscBool hasLagrange, const char label[], PetscInt value, DM *subdm)
3353 {
3354   PetscInt       dim, depth;
3355   PetscErrorCode ierr;
3356 
3357   PetscFunctionBegin;
3358   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3359   PetscValidPointer(subdm, 5);
3360   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
3361   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3362   ierr = DMCreate(PetscObjectComm((PetscObject)dm), subdm);CHKERRQ(ierr);
3363   ierr = DMSetType(*subdm, DMPLEX);CHKERRQ(ierr);
3364   ierr = DMSetDimension(*subdm, dim-1);CHKERRQ(ierr);
3365   if (depth == dim) {
3366     ierr = DMPlexCreateCohesiveSubmesh_Interpolated(dm, label, value, *subdm);CHKERRQ(ierr);
3367   } else {
3368     ierr = DMPlexCreateCohesiveSubmesh_Uninterpolated(dm, hasLagrange, label, value, *subdm);CHKERRQ(ierr);
3369   }
3370   PetscFunctionReturn(0);
3371 }
3372 
3373 #undef __FUNCT__
3374 #define __FUNCT__ "DMPlexFilter"
3375 /*@
3376   DMPlexFilter - Extract a subset of mesh cells defined by a label as a separate mesh
3377 
3378   Input Parameters:
3379 + dm        - The original mesh
3380 . cellLabel - The DMLabel marking cells contained in the new mesh
3381 - value     - The label value to use
3382 
3383   Output Parameter:
3384 . subdm - The new mesh
3385 
3386   Note: This function produces a DMLabel mapping original points in the submesh to their depth. This can be obtained using DMPlexGetSubpointMap().
3387 
3388   Level: developer
3389 
3390 .seealso: DMPlexGetSubpointMap(), DMGetLabel(), DMLabelSetValue()
3391 @*/
3392 PetscErrorCode DMPlexFilter(DM dm, DMLabel cellLabel, PetscInt value, DM *subdm)
3393 {
3394   PetscInt       dim;
3395   PetscErrorCode ierr;
3396 
3397   PetscFunctionBegin;
3398   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3399   PetscValidPointer(subdm, 3);
3400   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
3401   ierr = DMCreate(PetscObjectComm((PetscObject) dm), subdm);CHKERRQ(ierr);
3402   ierr = DMSetType(*subdm, DMPLEX);CHKERRQ(ierr);
3403   ierr = DMSetDimension(*subdm, dim);CHKERRQ(ierr);
3404   /* Extract submesh in place, could be empty on some procs, could have inconsistency if procs do not both extract a shared cell */
3405   ierr = DMPlexCreateSubmeshGeneric_Interpolated(dm, cellLabel, value, PETSC_FALSE, 0, *subdm);CHKERRQ(ierr);
3406   PetscFunctionReturn(0);
3407 }
3408 
3409 #undef __FUNCT__
3410 #define __FUNCT__ "DMPlexGetSubpointMap"
3411 /*@
3412   DMPlexGetSubpointMap - Returns a DMLabel with point dimension as values
3413 
3414   Input Parameter:
3415 . dm - The submesh DM
3416 
3417   Output Parameter:
3418 . subpointMap - The DMLabel of all the points from the original mesh in this submesh, or NULL if this is not a submesh
3419 
3420   Level: developer
3421 
3422 .seealso: DMPlexCreateSubmesh(), DMPlexCreateSubpointIS()
3423 @*/
3424 PetscErrorCode DMPlexGetSubpointMap(DM dm, DMLabel *subpointMap)
3425 {
3426   PetscFunctionBegin;
3427   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3428   PetscValidPointer(subpointMap, 2);
3429   *subpointMap = ((DM_Plex*) dm->data)->subpointMap;
3430   PetscFunctionReturn(0);
3431 }
3432 
3433 #undef __FUNCT__
3434 #define __FUNCT__ "DMPlexSetSubpointMap"
3435 /* Note: Should normally not be called by the user, since it is set in DMPlexCreateSubmesh() */
3436 PetscErrorCode DMPlexSetSubpointMap(DM dm, DMLabel subpointMap)
3437 {
3438   DM_Plex       *mesh = (DM_Plex *) dm->data;
3439   DMLabel        tmp;
3440   PetscErrorCode ierr;
3441 
3442   PetscFunctionBegin;
3443   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3444   tmp  = mesh->subpointMap;
3445   mesh->subpointMap = subpointMap;
3446   ++mesh->subpointMap->refct;
3447   ierr = DMLabelDestroy(&tmp);CHKERRQ(ierr);
3448   PetscFunctionReturn(0);
3449 }
3450 
3451 #undef __FUNCT__
3452 #define __FUNCT__ "DMPlexCreateSubpointIS"
3453 /*@
3454   DMPlexCreateSubpointIS - Creates an IS covering the entire subdm chart with the original points as data
3455 
3456   Input Parameter:
3457 . dm - The submesh DM
3458 
3459   Output Parameter:
3460 . subpointIS - The IS of all the points from the original mesh in this submesh, or NULL if this is not a submesh
3461 
3462   Note: This IS is guaranteed to be sorted by the construction of the submesh
3463 
3464   Level: developer
3465 
3466 .seealso: DMPlexCreateSubmesh(), DMPlexGetSubpointMap()
3467 @*/
3468 PetscErrorCode DMPlexCreateSubpointIS(DM dm, IS *subpointIS)
3469 {
3470   MPI_Comm        comm;
3471   DMLabel         subpointMap;
3472   IS              is;
3473   const PetscInt *opoints;
3474   PetscInt       *points, *depths;
3475   PetscInt        depth, depStart, depEnd, d, pStart, pEnd, p, n, off;
3476   PetscErrorCode  ierr;
3477 
3478   PetscFunctionBegin;
3479   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3480   PetscValidPointer(subpointIS, 2);
3481   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
3482   *subpointIS = NULL;
3483   ierr = DMPlexGetSubpointMap(dm, &subpointMap);CHKERRQ(ierr);
3484   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3485   if (subpointMap && depth >= 0) {
3486     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
3487     if (pStart) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Submeshes must start the point numbering at 0, not %d", pStart);
3488     ierr = DMGetWorkArray(dm, depth+1, PETSC_INT, &depths);CHKERRQ(ierr);
3489     depths[0] = depth;
3490     depths[1] = 0;
3491     for(d = 2; d <= depth; ++d) {depths[d] = depth+1 - d;}
3492     ierr = PetscMalloc1(pEnd, &points);CHKERRQ(ierr);
3493     for(d = 0, off = 0; d <= depth; ++d) {
3494       const PetscInt dep = depths[d];
3495 
3496       ierr = DMPlexGetDepthStratum(dm, dep, &depStart, &depEnd);CHKERRQ(ierr);
3497       ierr = DMLabelGetStratumSize(subpointMap, dep, &n);CHKERRQ(ierr);
3498       if (((d < 2) && (depth > 1)) || (d == 1)) { /* Only check vertices and cells for now since the map is broken for others */
3499         if (n != depEnd-depStart) SETERRQ3(comm, PETSC_ERR_ARG_WRONG, "The number of mapped submesh points %d at depth %d should be %d", n, dep, depEnd-depStart);
3500       } else {
3501         if (!n) {
3502           if (d == 0) {
3503             /* Missing cells */
3504             for(p = 0; p < depEnd-depStart; ++p, ++off) points[off] = -1;
3505           } else {
3506             /* Missing faces */
3507             for(p = 0; p < depEnd-depStart; ++p, ++off) points[off] = PETSC_MAX_INT;
3508           }
3509         }
3510       }
3511       if (n) {
3512         ierr = DMLabelGetStratumIS(subpointMap, dep, &is);CHKERRQ(ierr);
3513         ierr = ISGetIndices(is, &opoints);CHKERRQ(ierr);
3514         for(p = 0; p < n; ++p, ++off) points[off] = opoints[p];
3515         ierr = ISRestoreIndices(is, &opoints);CHKERRQ(ierr);
3516         ierr = ISDestroy(&is);CHKERRQ(ierr);
3517       }
3518     }
3519     ierr = DMRestoreWorkArray(dm, depth+1, PETSC_INT, &depths);CHKERRQ(ierr);
3520     if (off != pEnd) SETERRQ2(comm, PETSC_ERR_ARG_WRONG, "The number of mapped submesh points %d should be %d", off, pEnd);
3521     ierr = ISCreateGeneral(PETSC_COMM_SELF, pEnd, points, PETSC_OWN_POINTER, subpointIS);CHKERRQ(ierr);
3522   }
3523   PetscFunctionReturn(0);
3524 }
3525