xref: /petsc/src/dm/impls/plex/plexsubmesh.c (revision ef0bb6c736604ce380bf8bea4ebd4a7bda431d97) !
1 #include <petsc/private/dmpleximpl.h>    /*I      "petscdmplex.h"    I*/
2 #include <petsc/private/dmlabelimpl.h>   /*I      "petscdmlabel.h"   I*/
3 #include <petscsf.h>
4 
5 static PetscErrorCode DMPlexMarkBoundaryFaces_Internal(DM dm, PetscInt val, PetscInt cellHeight, DMLabel label)
6 {
7   PetscInt       fStart, fEnd, f;
8   PetscErrorCode ierr;
9 
10   PetscFunctionBegin;
11   ierr = DMPlexGetHeightStratum(dm, cellHeight+1, &fStart, &fEnd);CHKERRQ(ierr);
12   for (f = fStart; f < fEnd; ++f) {
13     PetscInt supportSize;
14 
15     ierr = DMPlexGetSupportSize(dm, f, &supportSize);CHKERRQ(ierr);
16     if (supportSize == 1) {
17       if (val < 0) {
18         PetscInt *closure = NULL;
19         PetscInt  clSize, cl, cval;
20 
21         ierr = DMPlexGetTransitiveClosure(dm, f, PETSC_TRUE, &clSize, &closure);CHKERRQ(ierr);
22         for (cl = 0; cl < clSize*2; cl += 2) {
23           ierr = DMLabelGetValue(label, closure[cl], &cval);CHKERRQ(ierr);
24           if (cval < 0) continue;
25           ierr = DMLabelSetValue(label, f, cval);CHKERRQ(ierr);
26           break;
27         }
28         if (cl == clSize*2) {ierr = DMLabelSetValue(label, f, 1);CHKERRQ(ierr);}
29         ierr = DMPlexRestoreTransitiveClosure(dm, f, PETSC_TRUE, &clSize, &closure);CHKERRQ(ierr);
30       } else {
31         ierr = DMLabelSetValue(label, f, val);CHKERRQ(ierr);
32       }
33     }
34   }
35   PetscFunctionReturn(0);
36 }
37 
38 /*@
39   DMPlexMarkBoundaryFaces - Mark all faces on the boundary
40 
41   Not Collective
42 
43   Input Parameter:
44 + dm - The original DM
45 - val - The marker value, or PETSC_DETERMINE to use some value in the closure (or 1 if none are found)
46 
47   Output Parameter:
48 . label - The DMLabel marking boundary faces with the given value
49 
50   Level: developer
51 
52 .seealso: DMLabelCreate(), DMCreateLabel()
53 @*/
54 PetscErrorCode DMPlexMarkBoundaryFaces(DM dm, PetscInt val, DMLabel label)
55 {
56   DMPlexInterpolatedFlag  flg;
57   PetscErrorCode          ierr;
58 
59   PetscFunctionBegin;
60   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
61   ierr = DMPlexIsInterpolated(dm, &flg);CHKERRQ(ierr);
62   if (flg != DMPLEX_INTERPOLATED_FULL) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM is not fully interpolated on this rank");
63   ierr = DMPlexMarkBoundaryFaces_Internal(dm, val, 0, label);CHKERRQ(ierr);
64   PetscFunctionReturn(0);
65 }
66 
67 static PetscErrorCode DMPlexLabelComplete_Internal(DM dm, DMLabel label, PetscBool completeCells)
68 {
69   IS              valueIS;
70   PetscSF         sfPoint;
71   const PetscInt *values;
72   PetscInt        numValues, v, cStart, cEnd, nroots;
73   PetscErrorCode  ierr;
74 
75   PetscFunctionBegin;
76   ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
77   ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
78   ierr = DMPlexGetHeightStratum(dm,0,&cStart,&cEnd);CHKERRQ(ierr);
79   ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
80   for (v = 0; v < numValues; ++v) {
81     IS              pointIS;
82     const PetscInt *points;
83     PetscInt        numPoints, p;
84 
85     ierr = DMLabelGetStratumSize(label, values[v], &numPoints);CHKERRQ(ierr);
86     ierr = DMLabelGetStratumIS(label, values[v], &pointIS);CHKERRQ(ierr);
87     ierr = ISGetIndices(pointIS, &points);CHKERRQ(ierr);
88     for (p = 0; p < numPoints; ++p) {
89       PetscInt  q = points[p];
90       PetscInt *closure = NULL;
91       PetscInt  closureSize, c;
92 
93       if (cStart <= q && q < cEnd && !completeCells) { /* skip cells */
94         continue;
95       }
96       ierr = DMPlexGetTransitiveClosure(dm, q, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
97       for (c = 0; c < closureSize*2; c += 2) {
98         ierr = DMLabelSetValue(label, closure[c], values[v]);CHKERRQ(ierr);
99       }
100       ierr = DMPlexRestoreTransitiveClosure(dm, q, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
101     }
102     ierr = ISRestoreIndices(pointIS, &points);CHKERRQ(ierr);
103     ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
104   }
105   ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
106   ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
107   ierr = DMGetPointSF(dm, &sfPoint);CHKERRQ(ierr);
108   ierr = PetscSFGetGraph(sfPoint, &nroots, NULL, NULL, NULL);CHKERRQ(ierr);
109   if (nroots >= 0) {
110     DMLabel         lblRoots, lblLeaves;
111     IS              valueIS, pointIS;
112     const PetscInt *values;
113     PetscInt        numValues, v;
114     PetscErrorCode  ierr;
115 
116     ierr = DMGetPointSF(dm, &sfPoint);CHKERRQ(ierr);
117     /* Pull point contributions from remote leaves into local roots */
118     ierr = DMLabelGather(label, sfPoint, &lblLeaves);CHKERRQ(ierr);
119     ierr = DMLabelGetValueIS(lblLeaves, &valueIS);CHKERRQ(ierr);
120     ierr = ISGetLocalSize(valueIS, &numValues);CHKERRQ(ierr);
121     ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
122     for (v = 0; v < numValues; ++v) {
123       const PetscInt value = values[v];
124 
125       ierr = DMLabelGetStratumIS(lblLeaves, value, &pointIS);CHKERRQ(ierr);
126       ierr = DMLabelInsertIS(label, pointIS, value);CHKERRQ(ierr);
127       ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
128     }
129     ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
130     ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
131     ierr = DMLabelDestroy(&lblLeaves);CHKERRQ(ierr);
132     /* Push point contributions from roots into remote leaves */
133     ierr = DMLabelDistribute(label, sfPoint, &lblRoots);CHKERRQ(ierr);
134     ierr = DMLabelGetValueIS(lblRoots, &valueIS);CHKERRQ(ierr);
135     ierr = ISGetLocalSize(valueIS, &numValues);CHKERRQ(ierr);
136     ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
137     for (v = 0; v < numValues; ++v) {
138       const PetscInt value = values[v];
139 
140       ierr = DMLabelGetStratumIS(lblRoots, value, &pointIS);CHKERRQ(ierr);
141       ierr = DMLabelInsertIS(label, pointIS, value);CHKERRQ(ierr);
142       ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
143     }
144     ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
145     ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
146     ierr = DMLabelDestroy(&lblRoots);CHKERRQ(ierr);
147   }
148   PetscFunctionReturn(0);
149 }
150 
151 /*@
152   DMPlexLabelComplete - Starting with a label marking points on a surface, we add the transitive closure to the surface
153 
154   Input Parameters:
155 + dm - The DM
156 - label - A DMLabel marking the surface points
157 
158   Output Parameter:
159 . label - A DMLabel marking all surface points in the transitive closure
160 
161   Level: developer
162 
163 .seealso: DMPlexLabelCohesiveComplete()
164 @*/
165 PetscErrorCode DMPlexLabelComplete(DM dm, DMLabel label)
166 {
167   PetscErrorCode ierr;
168 
169   PetscFunctionBegin;
170   ierr = DMPlexLabelComplete_Internal(dm, label, PETSC_TRUE);CHKERRQ(ierr);
171   PetscFunctionReturn(0);
172 }
173 
174 /*@
175   DMPlexLabelAddCells - Starting with a label marking points on a surface, we add a cell for each point
176 
177   Input Parameters:
178 + dm - The DM
179 - label - A DMLabel marking the surface points
180 
181   Output Parameter:
182 . label - A DMLabel incorporating cells
183 
184   Level: developer
185 
186   Note: The cells allow FEM boundary conditions to be applied using the cell geometry
187 
188 .seealso: DMPlexLabelAddFaceCells(), DMPlexLabelComplete(), DMPlexLabelCohesiveComplete()
189 @*/
190 PetscErrorCode DMPlexLabelAddCells(DM dm, DMLabel label)
191 {
192   IS              valueIS;
193   const PetscInt *values;
194   PetscInt        numValues, v, cStart, cEnd;
195   PetscErrorCode  ierr;
196 
197   PetscFunctionBegin;
198   ierr = DMPlexGetInteriorCellStratum(dm, &cStart, &cEnd);CHKERRQ(ierr);
199   ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
200   ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
201   ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
202   for (v = 0; v < numValues; ++v) {
203     IS              pointIS;
204     const PetscInt *points;
205     PetscInt        numPoints, p;
206 
207     ierr = DMLabelGetStratumSize(label, values[v], &numPoints);CHKERRQ(ierr);
208     ierr = DMLabelGetStratumIS(label, values[v], &pointIS);CHKERRQ(ierr);
209     ierr = ISGetIndices(pointIS, &points);CHKERRQ(ierr);
210     for (p = 0; p < numPoints; ++p) {
211       PetscInt *closure = NULL;
212       PetscInt  closureSize, cl;
213 
214       ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_FALSE, &closureSize, &closure);CHKERRQ(ierr);
215       for (cl = closureSize-1; cl > 0; --cl) {
216         const PetscInt cell = closure[cl*2];
217         if ((cell >= cStart) && (cell < cEnd)) {ierr = DMLabelSetValue(label, cell, values[v]);CHKERRQ(ierr); break;}
218       }
219       ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_FALSE, &closureSize, &closure);CHKERRQ(ierr);
220     }
221     ierr = ISRestoreIndices(pointIS, &points);CHKERRQ(ierr);
222     ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
223   }
224   ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
225   ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
226   PetscFunctionReturn(0);
227 }
228 
229 /*@
230   DMPlexLabelAddFaceCells - Starting with a label marking faces on a surface, we add a cell for each face
231 
232   Input Parameters:
233 + dm - The DM
234 - label - A DMLabel marking the surface points
235 
236   Output Parameter:
237 . label - A DMLabel incorporating cells
238 
239   Level: developer
240 
241   Note: The cells allow FEM boundary conditions to be applied using the cell geometry
242 
243 .seealso: DMPlexLabelAddCells(), DMPlexLabelComplete(), DMPlexLabelCohesiveComplete()
244 @*/
245 PetscErrorCode DMPlexLabelAddFaceCells(DM dm, DMLabel label)
246 {
247   IS              valueIS;
248   const PetscInt *values;
249   PetscInt        numValues, v, cStart, cEnd, fStart, fEnd;
250   PetscErrorCode  ierr;
251 
252   PetscFunctionBegin;
253   ierr = DMPlexGetInteriorCellStratum(dm, &cStart, &cEnd);CHKERRQ(ierr);
254   ierr = DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd);CHKERRQ(ierr);
255   ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
256   ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
257   ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
258   for (v = 0; v < numValues; ++v) {
259     IS              pointIS;
260     const PetscInt *points;
261     PetscInt        numPoints, p;
262 
263     ierr = DMLabelGetStratumSize(label, values[v], &numPoints);CHKERRQ(ierr);
264     ierr = DMLabelGetStratumIS(label, values[v], &pointIS);CHKERRQ(ierr);
265     ierr = ISGetIndices(pointIS, &points);CHKERRQ(ierr);
266     for (p = 0; p < numPoints; ++p) {
267       const PetscInt face = points[p];
268       PetscInt      *closure = NULL;
269       PetscInt       closureSize, cl;
270 
271       if ((face < fStart) || (face >= fEnd)) continue;
272       ierr = DMPlexGetTransitiveClosure(dm, face, PETSC_FALSE, &closureSize, &closure);CHKERRQ(ierr);
273       for (cl = closureSize-1; cl > 0; --cl) {
274         const PetscInt cell = closure[cl*2];
275         if ((cell >= cStart) && (cell < cEnd)) {ierr = DMLabelSetValue(label, cell, values[v]);CHKERRQ(ierr); break;}
276       }
277       ierr = DMPlexRestoreTransitiveClosure(dm, face, PETSC_FALSE, &closureSize, &closure);CHKERRQ(ierr);
278     }
279     ierr = ISRestoreIndices(pointIS, &points);CHKERRQ(ierr);
280     ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
281   }
282   ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
283   ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
284   PetscFunctionReturn(0);
285 }
286 
287 /*@
288   DMPlexLabelClearCells - Remove cells from a label
289 
290   Input Parameters:
291 + dm - The DM
292 - label - A DMLabel marking surface points and their adjacent cells
293 
294   Output Parameter:
295 . label - A DMLabel without cells
296 
297   Level: developer
298 
299   Note: This undoes DMPlexLabelAddCells() or DMPlexLabelAddFaceCells()
300 
301 .seealso: DMPlexLabelComplete(), DMPlexLabelCohesiveComplete(), DMPlexLabelAddCells()
302 @*/
303 PetscErrorCode DMPlexLabelClearCells(DM dm, DMLabel label)
304 {
305   IS              valueIS;
306   const PetscInt *values;
307   PetscInt        numValues, v, cStart, cEnd;
308   PetscErrorCode  ierr;
309 
310   PetscFunctionBegin;
311   ierr = DMPlexGetInteriorCellStratum(dm, &cStart, &cEnd);CHKERRQ(ierr);
312   ierr = DMLabelGetNumValues(label, &numValues);CHKERRQ(ierr);
313   ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
314   ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
315   for (v = 0; v < numValues; ++v) {
316     IS              pointIS;
317     const PetscInt *points;
318     PetscInt        numPoints, p;
319 
320     ierr = DMLabelGetStratumSize(label, values[v], &numPoints);CHKERRQ(ierr);
321     ierr = DMLabelGetStratumIS(label, values[v], &pointIS);CHKERRQ(ierr);
322     ierr = ISGetIndices(pointIS, &points);CHKERRQ(ierr);
323     for (p = 0; p < numPoints; ++p) {
324       PetscInt point = points[p];
325 
326       if (point >= cStart && point < cEnd) {
327         ierr = DMLabelClearValue(label,point,values[v]);CHKERRQ(ierr);
328       }
329     }
330     ierr = ISRestoreIndices(pointIS, &points);CHKERRQ(ierr);
331     ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
332   }
333   ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
334   ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
335   PetscFunctionReturn(0);
336 }
337 
338 /* take (oldEnd, added) pairs, ordered by height and convert them to (oldstart, newstart) pairs, ordered by ascending
339  * index (skipping first, which is (0,0)) */
340 PETSC_STATIC_INLINE PetscErrorCode DMPlexShiftPointSetUp_Internal(PetscInt depth, PetscInt depthShift[])
341 {
342   PetscInt d, off = 0;
343 
344   PetscFunctionBegin;
345   /* sort by (oldend): yes this is an O(n^2) sort, we expect depth <= 3 */
346   for (d = 0; d < depth; d++) {
347     PetscInt firstd = d;
348     PetscInt firstStart = depthShift[2*d];
349     PetscInt e;
350 
351     for (e = d+1; e <= depth; e++) {
352       if (depthShift[2*e] < firstStart) {
353         firstd = e;
354         firstStart = depthShift[2*d];
355       }
356     }
357     if (firstd != d) {
358       PetscInt swap[2];
359 
360       e = firstd;
361       swap[0] = depthShift[2*d];
362       swap[1] = depthShift[2*d+1];
363       depthShift[2*d]   = depthShift[2*e];
364       depthShift[2*d+1] = depthShift[2*e+1];
365       depthShift[2*e]   = swap[0];
366       depthShift[2*e+1] = swap[1];
367     }
368   }
369   /* convert (oldstart, added) to (oldstart, newstart) */
370   for (d = 0; d <= depth; d++) {
371     off += depthShift[2*d+1];
372     depthShift[2*d+1] = depthShift[2*d] + off;
373   }
374   PetscFunctionReturn(0);
375 }
376 
377 /* depthShift is a list of (old, new) pairs */
378 PETSC_STATIC_INLINE PetscInt DMPlexShiftPoint_Internal(PetscInt p, PetscInt depth, PetscInt depthShift[])
379 {
380   PetscInt d;
381   PetscInt newOff = 0;
382 
383   for (d = 0; d <= depth; d++) {
384     if (p < depthShift[2*d]) return p + newOff;
385     else newOff = depthShift[2*d+1] - depthShift[2*d];
386   }
387   return p + newOff;
388 }
389 
390 /* depthShift is a list of (old, new) pairs */
391 PETSC_STATIC_INLINE PetscInt DMPlexShiftPointInverse_Internal(PetscInt p, PetscInt depth, PetscInt depthShift[])
392 {
393   PetscInt d;
394   PetscInt newOff = 0;
395 
396   for (d = 0; d <= depth; d++) {
397     if (p < depthShift[2*d+1]) return p + newOff;
398     else newOff = depthShift[2*d] - depthShift[2*d+1];
399   }
400   return p + newOff;
401 }
402 
403 static PetscErrorCode DMPlexShiftSizes_Internal(DM dm, PetscInt depthShift[], DM dmNew)
404 {
405   PetscInt       depth = 0, d, pStart, pEnd, p;
406   DMLabel        depthLabel;
407   PetscErrorCode ierr;
408 
409   PetscFunctionBegin;
410   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
411   if (depth < 0) PetscFunctionReturn(0);
412   /* Step 1: Expand chart */
413   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
414   pEnd = DMPlexShiftPoint_Internal(pEnd,depth,depthShift);
415   ierr = DMPlexSetChart(dmNew, pStart, pEnd);CHKERRQ(ierr);
416   ierr = DMCreateLabel(dmNew,"depth");CHKERRQ(ierr);
417   ierr = DMPlexGetDepthLabel(dmNew,&depthLabel);CHKERRQ(ierr);
418   /* Step 2: Set cone and support sizes */
419   for (d = 0; d <= depth; ++d) {
420     PetscInt pStartNew, pEndNew;
421     IS pIS;
422 
423     ierr = DMPlexGetDepthStratum(dm, d, &pStart, &pEnd);CHKERRQ(ierr);
424     pStartNew = DMPlexShiftPoint_Internal(pStart, depth, depthShift);
425     pEndNew = DMPlexShiftPoint_Internal(pEnd, depth, depthShift);
426     ierr = ISCreateStride(PETSC_COMM_SELF, pEndNew - pStartNew, pStartNew, 1, &pIS);CHKERRQ(ierr);
427     ierr = DMLabelSetStratumIS(depthLabel, d, pIS);CHKERRQ(ierr);
428     ierr = ISDestroy(&pIS);CHKERRQ(ierr);
429     for (p = pStart; p < pEnd; ++p) {
430       PetscInt newp = DMPlexShiftPoint_Internal(p, depth, depthShift);
431       PetscInt size;
432 
433       ierr = DMPlexGetConeSize(dm, p, &size);CHKERRQ(ierr);
434       ierr = DMPlexSetConeSize(dmNew, newp, size);CHKERRQ(ierr);
435       ierr = DMPlexGetSupportSize(dm, p, &size);CHKERRQ(ierr);
436       ierr = DMPlexSetSupportSize(dmNew, newp, size);CHKERRQ(ierr);
437     }
438   }
439   PetscFunctionReturn(0);
440 }
441 
442 static PetscErrorCode DMPlexShiftPoints_Internal(DM dm, PetscInt depthShift[], DM dmNew)
443 {
444   PetscInt      *newpoints;
445   PetscInt       depth = 0, maxConeSize, maxSupportSize, maxConeSizeNew, maxSupportSizeNew, pStart, pEnd, p;
446   PetscErrorCode ierr;
447 
448   PetscFunctionBegin;
449   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
450   if (depth < 0) PetscFunctionReturn(0);
451   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);CHKERRQ(ierr);
452   ierr = DMPlexGetMaxSizes(dmNew, &maxConeSizeNew, &maxSupportSizeNew);CHKERRQ(ierr);
453   ierr = PetscMalloc1(PetscMax(PetscMax(maxConeSize, maxSupportSize), PetscMax(maxConeSizeNew, maxSupportSizeNew)),&newpoints);CHKERRQ(ierr);
454   /* Step 5: Set cones and supports */
455   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
456   for (p = pStart; p < pEnd; ++p) {
457     const PetscInt *points = NULL, *orientations = NULL;
458     PetscInt        size,sizeNew, i, newp = DMPlexShiftPoint_Internal(p, depth, depthShift);
459 
460     ierr = DMPlexGetConeSize(dm, p, &size);CHKERRQ(ierr);
461     ierr = DMPlexGetCone(dm, p, &points);CHKERRQ(ierr);
462     ierr = DMPlexGetConeOrientation(dm, p, &orientations);CHKERRQ(ierr);
463     for (i = 0; i < size; ++i) {
464       newpoints[i] = DMPlexShiftPoint_Internal(points[i], depth, depthShift);
465     }
466     ierr = DMPlexSetCone(dmNew, newp, newpoints);CHKERRQ(ierr);
467     ierr = DMPlexSetConeOrientation(dmNew, newp, orientations);CHKERRQ(ierr);
468     ierr = DMPlexGetSupportSize(dm, p, &size);CHKERRQ(ierr);
469     ierr = DMPlexGetSupportSize(dmNew, newp, &sizeNew);CHKERRQ(ierr);
470     ierr = DMPlexGetSupport(dm, p, &points);CHKERRQ(ierr);
471     for (i = 0; i < size; ++i) {
472       newpoints[i] = DMPlexShiftPoint_Internal(points[i], depth, depthShift);
473     }
474     for (i = size; i < sizeNew; ++i) newpoints[i] = 0;
475     ierr = DMPlexSetSupport(dmNew, newp, newpoints);CHKERRQ(ierr);
476   }
477   ierr = PetscFree(newpoints);CHKERRQ(ierr);
478   PetscFunctionReturn(0);
479 }
480 
481 static PetscErrorCode DMPlexShiftCoordinates_Internal(DM dm, PetscInt depthShift[], DM dmNew)
482 {
483   PetscSection   coordSection, newCoordSection;
484   Vec            coordinates, newCoordinates;
485   PetscScalar   *coords, *newCoords;
486   PetscInt       coordSize, sStart, sEnd;
487   PetscInt       dim, depth = 0, cStart, cEnd, cStartNew, cEndNew, c, vStart, vEnd, vStartNew, vEndNew, v;
488   PetscBool      hasCells;
489   PetscErrorCode ierr;
490 
491   PetscFunctionBegin;
492   ierr = DMGetCoordinateDim(dm, &dim);CHKERRQ(ierr);
493   ierr = DMSetCoordinateDim(dmNew, dim);CHKERRQ(ierr);
494   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
495   /* Step 8: Convert coordinates */
496   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
497   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
498   ierr = DMPlexGetDepthStratum(dmNew, 0, &vStartNew, &vEndNew);CHKERRQ(ierr);
499   ierr = DMPlexGetHeightStratum(dmNew, 0, &cStartNew, &cEndNew);CHKERRQ(ierr);
500   ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
501   ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dm), &newCoordSection);CHKERRQ(ierr);
502   ierr = PetscSectionSetNumFields(newCoordSection, 1);CHKERRQ(ierr);
503   ierr = PetscSectionSetFieldComponents(newCoordSection, 0, dim);CHKERRQ(ierr);
504   ierr = PetscSectionGetChart(coordSection, &sStart, &sEnd);CHKERRQ(ierr);
505   hasCells = sStart == cStart ? PETSC_TRUE : PETSC_FALSE;
506   ierr = PetscSectionSetChart(newCoordSection, hasCells ? cStartNew : vStartNew, vEndNew);CHKERRQ(ierr);
507   if (hasCells) {
508     for (c = cStart; c < cEnd; ++c) {
509       PetscInt cNew = DMPlexShiftPoint_Internal(c, depth, depthShift), dof;
510 
511       ierr = PetscSectionGetDof(coordSection, c, &dof);CHKERRQ(ierr);
512       ierr = PetscSectionSetDof(newCoordSection, cNew, dof);CHKERRQ(ierr);
513       ierr = PetscSectionSetFieldDof(newCoordSection, cNew, 0, dof);CHKERRQ(ierr);
514     }
515   }
516   for (v = vStartNew; v < vEndNew; ++v) {
517     ierr = PetscSectionSetDof(newCoordSection, v, dim);CHKERRQ(ierr);
518     ierr = PetscSectionSetFieldDof(newCoordSection, v, 0, dim);CHKERRQ(ierr);
519   }
520   ierr = PetscSectionSetUp(newCoordSection);CHKERRQ(ierr);
521   ierr = DMSetCoordinateSection(dmNew, PETSC_DETERMINE, newCoordSection);CHKERRQ(ierr);
522   ierr = PetscSectionGetStorageSize(newCoordSection, &coordSize);CHKERRQ(ierr);
523   ierr = VecCreate(PETSC_COMM_SELF, &newCoordinates);CHKERRQ(ierr);
524   ierr = PetscObjectSetName((PetscObject) newCoordinates, "coordinates");CHKERRQ(ierr);
525   ierr = VecSetSizes(newCoordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
526   ierr = VecSetBlockSize(newCoordinates, dim);CHKERRQ(ierr);
527   ierr = VecSetType(newCoordinates,VECSTANDARD);CHKERRQ(ierr);
528   ierr = DMSetCoordinatesLocal(dmNew, newCoordinates);CHKERRQ(ierr);
529   ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
530   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
531   ierr = VecGetArray(newCoordinates, &newCoords);CHKERRQ(ierr);
532   if (hasCells) {
533     for (c = cStart; c < cEnd; ++c) {
534       PetscInt cNew = DMPlexShiftPoint_Internal(c, depth, depthShift), dof, off, noff, d;
535 
536       ierr = PetscSectionGetDof(coordSection, c, &dof);CHKERRQ(ierr);
537       ierr = PetscSectionGetOffset(coordSection, c, &off);CHKERRQ(ierr);
538       ierr = PetscSectionGetOffset(newCoordSection, cNew, &noff);CHKERRQ(ierr);
539       for (d = 0; d < dof; ++d) newCoords[noff+d] = coords[off+d];
540     }
541   }
542   for (v = vStart; v < vEnd; ++v) {
543     PetscInt dof, off, noff, d;
544 
545     ierr = PetscSectionGetDof(coordSection, v, &dof);CHKERRQ(ierr);
546     ierr = PetscSectionGetOffset(coordSection, v, &off);CHKERRQ(ierr);
547     ierr = PetscSectionGetOffset(newCoordSection, DMPlexShiftPoint_Internal(v, depth, depthShift), &noff);CHKERRQ(ierr);
548     for (d = 0; d < dof; ++d) newCoords[noff+d] = coords[off+d];
549   }
550   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
551   ierr = VecRestoreArray(newCoordinates, &newCoords);CHKERRQ(ierr);
552   ierr = VecDestroy(&newCoordinates);CHKERRQ(ierr);
553   ierr = PetscSectionDestroy(&newCoordSection);CHKERRQ(ierr);
554   PetscFunctionReturn(0);
555 }
556 
557 static PetscErrorCode DMPlexShiftSF_Internal(DM dm, PetscInt depthShift[], DM dmNew)
558 {
559   PetscInt           depth = 0;
560   PetscSF            sfPoint, sfPointNew;
561   const PetscSFNode *remotePoints;
562   PetscSFNode       *gremotePoints;
563   const PetscInt    *localPoints;
564   PetscInt          *glocalPoints, *newLocation, *newRemoteLocation;
565   PetscInt           numRoots, numLeaves, l, pStart, pEnd, totShift = 0;
566   PetscErrorCode     ierr;
567 
568   PetscFunctionBegin;
569   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
570   /* Step 9: Convert pointSF */
571   ierr = DMGetPointSF(dm, &sfPoint);CHKERRQ(ierr);
572   ierr = DMGetPointSF(dmNew, &sfPointNew);CHKERRQ(ierr);
573   ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
574   ierr = PetscSFGetGraph(sfPoint, &numRoots, &numLeaves, &localPoints, &remotePoints);CHKERRQ(ierr);
575   totShift = DMPlexShiftPoint_Internal(pEnd,depth,depthShift) - pEnd;
576   if (numRoots >= 0) {
577     ierr = PetscMalloc2(numRoots,&newLocation,pEnd-pStart,&newRemoteLocation);CHKERRQ(ierr);
578     for (l=0; l<numRoots; l++) newLocation[l] = DMPlexShiftPoint_Internal(l, depth, depthShift);
579     ierr = PetscSFBcastBegin(sfPoint, MPIU_INT, newLocation, newRemoteLocation);CHKERRQ(ierr);
580     ierr = PetscSFBcastEnd(sfPoint, MPIU_INT, newLocation, newRemoteLocation);CHKERRQ(ierr);
581     ierr = PetscMalloc1(numLeaves,    &glocalPoints);CHKERRQ(ierr);
582     ierr = PetscMalloc1(numLeaves, &gremotePoints);CHKERRQ(ierr);
583     for (l = 0; l < numLeaves; ++l) {
584       glocalPoints[l]        = DMPlexShiftPoint_Internal(localPoints[l], depth, depthShift);
585       gremotePoints[l].rank  = remotePoints[l].rank;
586       gremotePoints[l].index = newRemoteLocation[localPoints[l]];
587     }
588     ierr = PetscFree2(newLocation,newRemoteLocation);CHKERRQ(ierr);
589     ierr = PetscSFSetGraph(sfPointNew, numRoots + totShift, numLeaves, glocalPoints, PETSC_OWN_POINTER, gremotePoints, PETSC_OWN_POINTER);CHKERRQ(ierr);
590   }
591   PetscFunctionReturn(0);
592 }
593 
594 static PetscErrorCode DMPlexShiftLabels_Internal(DM dm, PetscInt depthShift[], DM dmNew)
595 {
596   PetscSF            sfPoint;
597   DMLabel            vtkLabel, ghostLabel;
598   const PetscSFNode *leafRemote;
599   const PetscInt    *leafLocal;
600   PetscInt           depth = 0, numLeaves, numLabels, l, cStart, cEnd, c, fStart, fEnd, f;
601   PetscMPIInt        rank;
602   PetscErrorCode     ierr;
603 
604   PetscFunctionBegin;
605   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
606   /* Step 10: Convert labels */
607   ierr = DMGetNumLabels(dm, &numLabels);CHKERRQ(ierr);
608   for (l = 0; l < numLabels; ++l) {
609     DMLabel         label, newlabel;
610     const char     *lname;
611     PetscBool       isDepth, isDim;
612     IS              valueIS;
613     const PetscInt *values;
614     PetscInt        numValues, val;
615 
616     ierr = DMGetLabelName(dm, l, &lname);CHKERRQ(ierr);
617     ierr = PetscStrcmp(lname, "depth", &isDepth);CHKERRQ(ierr);
618     if (isDepth) continue;
619     ierr = PetscStrcmp(lname, "dim", &isDim);CHKERRQ(ierr);
620     if (isDim) continue;
621     ierr = DMCreateLabel(dmNew, lname);CHKERRQ(ierr);
622     ierr = DMGetLabel(dm, lname, &label);CHKERRQ(ierr);
623     ierr = DMGetLabel(dmNew, lname, &newlabel);CHKERRQ(ierr);
624     ierr = DMLabelGetDefaultValue(label,&val);CHKERRQ(ierr);
625     ierr = DMLabelSetDefaultValue(newlabel,val);CHKERRQ(ierr);
626     ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
627     ierr = ISGetLocalSize(valueIS, &numValues);CHKERRQ(ierr);
628     ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
629     for (val = 0; val < numValues; ++val) {
630       IS              pointIS;
631       const PetscInt *points;
632       PetscInt        numPoints, p;
633 
634       ierr = DMLabelGetStratumIS(label, values[val], &pointIS);CHKERRQ(ierr);
635       ierr = ISGetLocalSize(pointIS, &numPoints);CHKERRQ(ierr);
636       ierr = ISGetIndices(pointIS, &points);CHKERRQ(ierr);
637       for (p = 0; p < numPoints; ++p) {
638         const PetscInt newpoint = DMPlexShiftPoint_Internal(points[p], depth, depthShift);
639 
640         ierr = DMLabelSetValue(newlabel, newpoint, values[val]);CHKERRQ(ierr);
641       }
642       ierr = ISRestoreIndices(pointIS, &points);CHKERRQ(ierr);
643       ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
644     }
645     ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
646     ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
647   }
648   /* Step 11: Make label for output (vtk) and to mark ghost points (ghost) */
649   ierr = MPI_Comm_rank(PetscObjectComm((PetscObject)dm), &rank);CHKERRQ(ierr);
650   ierr = DMGetPointSF(dm, &sfPoint);CHKERRQ(ierr);
651   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
652   ierr = PetscSFGetGraph(sfPoint, NULL, &numLeaves, &leafLocal, &leafRemote);CHKERRQ(ierr);
653   ierr = DMCreateLabel(dmNew, "vtk");CHKERRQ(ierr);
654   ierr = DMCreateLabel(dmNew, "ghost");CHKERRQ(ierr);
655   ierr = DMGetLabel(dmNew, "vtk", &vtkLabel);CHKERRQ(ierr);
656   ierr = DMGetLabel(dmNew, "ghost", &ghostLabel);CHKERRQ(ierr);
657   for (l = 0, c = cStart; l < numLeaves && c < cEnd; ++l, ++c) {
658     for (; c < leafLocal[l] && c < cEnd; ++c) {
659       ierr = DMLabelSetValue(vtkLabel, c, 1);CHKERRQ(ierr);
660     }
661     if (leafLocal[l] >= cEnd) break;
662     if (leafRemote[l].rank == rank) {
663       ierr = DMLabelSetValue(vtkLabel, c, 1);CHKERRQ(ierr);
664     } else {
665       ierr = DMLabelSetValue(ghostLabel, c, 2);CHKERRQ(ierr);
666     }
667   }
668   for (; c < cEnd; ++c) {
669     ierr = DMLabelSetValue(vtkLabel, c, 1);CHKERRQ(ierr);
670   }
671   ierr = DMPlexGetHeightStratum(dmNew, 1, &fStart, &fEnd);CHKERRQ(ierr);
672   for (f = fStart; f < fEnd; ++f) {
673     PetscInt numCells;
674 
675     ierr = DMPlexGetSupportSize(dmNew, f, &numCells);CHKERRQ(ierr);
676     if (numCells < 2) {
677       ierr = DMLabelSetValue(ghostLabel, f, 1);CHKERRQ(ierr);
678     } else {
679       const PetscInt *cells = NULL;
680       PetscInt        vA, vB;
681 
682       ierr = DMPlexGetSupport(dmNew, f, &cells);CHKERRQ(ierr);
683       ierr = DMLabelGetValue(vtkLabel, cells[0], &vA);CHKERRQ(ierr);
684       ierr = DMLabelGetValue(vtkLabel, cells[1], &vB);CHKERRQ(ierr);
685       if (vA != 1 && vB != 1) {ierr = DMLabelSetValue(ghostLabel, f, 1);CHKERRQ(ierr);}
686     }
687   }
688   PetscFunctionReturn(0);
689 }
690 
691 static PetscErrorCode DMPlexShiftTree_Internal(DM dm, PetscInt depthShift[], DM dmNew)
692 {
693   DM             refTree;
694   PetscSection   pSec;
695   PetscInt       *parents, *childIDs;
696   PetscErrorCode ierr;
697 
698   PetscFunctionBegin;
699   ierr = DMPlexGetReferenceTree(dm,&refTree);CHKERRQ(ierr);
700   ierr = DMPlexSetReferenceTree(dmNew,refTree);CHKERRQ(ierr);
701   ierr = DMPlexGetTree(dm,&pSec,&parents,&childIDs,NULL,NULL);CHKERRQ(ierr);
702   if (pSec) {
703     PetscInt p, pStart, pEnd, *parentsShifted, pStartShifted, pEndShifted, depth;
704     PetscInt *childIDsShifted;
705     PetscSection pSecShifted;
706 
707     ierr = PetscSectionGetChart(pSec,&pStart,&pEnd);CHKERRQ(ierr);
708     ierr = DMPlexGetDepth(dm,&depth);CHKERRQ(ierr);
709     pStartShifted = DMPlexShiftPoint_Internal(pStart,depth,depthShift);
710     pEndShifted   = DMPlexShiftPoint_Internal(pEnd,depth,depthShift);
711     ierr = PetscMalloc2(pEndShifted - pStartShifted,&parentsShifted,pEndShifted-pStartShifted,&childIDsShifted);CHKERRQ(ierr);
712     ierr = PetscSectionCreate(PetscObjectComm((PetscObject)dmNew),&pSecShifted);CHKERRQ(ierr);
713     ierr = PetscSectionSetChart(pSecShifted,pStartShifted,pEndShifted);CHKERRQ(ierr);
714     for (p = pStartShifted; p < pEndShifted; p++) {
715       /* start off assuming no children */
716       ierr = PetscSectionSetDof(pSecShifted,p,0);CHKERRQ(ierr);
717     }
718     for (p = pStart; p < pEnd; p++) {
719       PetscInt dof;
720       PetscInt pNew = DMPlexShiftPoint_Internal(p,depth,depthShift);
721 
722       ierr = PetscSectionGetDof(pSec,p,&dof);CHKERRQ(ierr);
723       ierr = PetscSectionSetDof(pSecShifted,pNew,dof);CHKERRQ(ierr);
724     }
725     ierr = PetscSectionSetUp(pSecShifted);CHKERRQ(ierr);
726     for (p = pStart; p < pEnd; p++) {
727       PetscInt dof;
728       PetscInt pNew = DMPlexShiftPoint_Internal(p,depth,depthShift);
729 
730       ierr = PetscSectionGetDof(pSec,p,&dof);CHKERRQ(ierr);
731       if (dof) {
732         PetscInt off, offNew;
733 
734         ierr = PetscSectionGetOffset(pSec,p,&off);CHKERRQ(ierr);
735         ierr = PetscSectionGetOffset(pSecShifted,pNew,&offNew);CHKERRQ(ierr);
736         parentsShifted[offNew] = DMPlexShiftPoint_Internal(parents[off],depth,depthShift);
737         childIDsShifted[offNew] = childIDs[off];
738       }
739     }
740     ierr = DMPlexSetTree(dmNew,pSecShifted,parentsShifted,childIDsShifted);CHKERRQ(ierr);
741     ierr = PetscFree2(parentsShifted,childIDsShifted);CHKERRQ(ierr);
742     ierr = PetscSectionDestroy(&pSecShifted);CHKERRQ(ierr);
743   }
744   PetscFunctionReturn(0);
745 }
746 
747 static PetscErrorCode DMPlexConstructGhostCells_Internal(DM dm, DMLabel label, PetscInt *numGhostCells, DM gdm)
748 {
749   PetscSF               sf;
750   IS                    valueIS;
751   const PetscInt       *values, *leaves;
752   PetscInt             *depthShift;
753   PetscInt              d, depth = 0, nleaves, loc, Ng, numFS, fs, fStart, fEnd, ghostCell, cEnd, c;
754   PetscBool             isper;
755   const PetscReal      *maxCell, *L;
756   const DMBoundaryType *bd;
757   PetscErrorCode        ierr;
758 
759   PetscFunctionBegin;
760   ierr = DMGetPointSF(dm, &sf);CHKERRQ(ierr);
761   ierr = PetscSFGetGraph(sf, NULL, &nleaves, &leaves, NULL);CHKERRQ(ierr);
762   nleaves = PetscMax(0, nleaves);
763   ierr = DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd);CHKERRQ(ierr);
764   /* Count ghost cells */
765   ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
766   ierr = ISGetLocalSize(valueIS, &numFS);CHKERRQ(ierr);
767   ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
768   Ng   = 0;
769   for (fs = 0; fs < numFS; ++fs) {
770     IS              faceIS;
771     const PetscInt *faces;
772     PetscInt        numFaces, f, numBdFaces = 0;
773 
774     ierr = DMLabelGetStratumIS(label, values[fs], &faceIS);CHKERRQ(ierr);
775     ierr = ISGetLocalSize(faceIS, &numFaces);CHKERRQ(ierr);
776     ierr = ISGetIndices(faceIS, &faces);CHKERRQ(ierr);
777     for (f = 0; f < numFaces; ++f) {
778       PetscInt numChildren;
779 
780       ierr = PetscFindInt(faces[f], nleaves, leaves, &loc);CHKERRQ(ierr);
781       ierr = DMPlexGetTreeChildren(dm,faces[f],&numChildren,NULL);CHKERRQ(ierr);
782       /* non-local and ancestors points don't get to register ghosts */
783       if (loc >= 0 || numChildren) continue;
784       if ((faces[f] >= fStart) && (faces[f] < fEnd)) ++numBdFaces;
785     }
786     Ng += numBdFaces;
787     ierr = ISDestroy(&faceIS);CHKERRQ(ierr);
788   }
789   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
790   ierr = PetscMalloc1(2*(depth+1), &depthShift);CHKERRQ(ierr);
791   for (d = 0; d <= depth; d++) {
792     PetscInt dEnd;
793 
794     ierr = DMPlexGetDepthStratum(dm,d,NULL,&dEnd);CHKERRQ(ierr);
795     depthShift[2*d]   = dEnd;
796     depthShift[2*d+1] = 0;
797   }
798   if (depth >= 0) depthShift[2*depth+1] = Ng;
799   ierr = DMPlexShiftPointSetUp_Internal(depth,depthShift);CHKERRQ(ierr);
800   ierr = DMPlexShiftSizes_Internal(dm, depthShift, gdm);CHKERRQ(ierr);
801   /* Step 3: Set cone/support sizes for new points */
802   ierr = DMPlexGetHeightStratum(dm, 0, NULL, &cEnd);CHKERRQ(ierr);
803   ierr = DMPlexSetGhostCellStratum(gdm, cEnd, PETSC_DETERMINE);CHKERRQ(ierr);
804   for (c = cEnd; c < cEnd + Ng; ++c) {
805     ierr = DMPlexSetConeSize(gdm, c, 1);CHKERRQ(ierr);
806   }
807   for (fs = 0; fs < numFS; ++fs) {
808     IS              faceIS;
809     const PetscInt *faces;
810     PetscInt        numFaces, f;
811 
812     ierr = DMLabelGetStratumIS(label, values[fs], &faceIS);CHKERRQ(ierr);
813     ierr = ISGetLocalSize(faceIS, &numFaces);CHKERRQ(ierr);
814     ierr = ISGetIndices(faceIS, &faces);CHKERRQ(ierr);
815     for (f = 0; f < numFaces; ++f) {
816       PetscInt size, numChildren;
817 
818       ierr = PetscFindInt(faces[f], nleaves, leaves, &loc);CHKERRQ(ierr);
819       ierr = DMPlexGetTreeChildren(dm,faces[f],&numChildren,NULL);CHKERRQ(ierr);
820       if (loc >= 0 || numChildren) continue;
821       if ((faces[f] < fStart) || (faces[f] >= fEnd)) continue;
822       ierr = DMPlexGetSupportSize(dm, faces[f], &size);CHKERRQ(ierr);
823       if (size != 1) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "DM has boundary face %d with %d support cells", faces[f], size);
824       ierr = DMPlexSetSupportSize(gdm, faces[f] + Ng, 2);CHKERRQ(ierr);
825     }
826     ierr = ISRestoreIndices(faceIS, &faces);CHKERRQ(ierr);
827     ierr = ISDestroy(&faceIS);CHKERRQ(ierr);
828   }
829   /* Step 4: Setup ghosted DM */
830   ierr = DMSetUp(gdm);CHKERRQ(ierr);
831   ierr = DMPlexShiftPoints_Internal(dm, depthShift, gdm);CHKERRQ(ierr);
832   /* Step 6: Set cones and supports for new points */
833   ghostCell = cEnd;
834   for (fs = 0; fs < numFS; ++fs) {
835     IS              faceIS;
836     const PetscInt *faces;
837     PetscInt        numFaces, f;
838 
839     ierr = DMLabelGetStratumIS(label, values[fs], &faceIS);CHKERRQ(ierr);
840     ierr = ISGetLocalSize(faceIS, &numFaces);CHKERRQ(ierr);
841     ierr = ISGetIndices(faceIS, &faces);CHKERRQ(ierr);
842     for (f = 0; f < numFaces; ++f) {
843       PetscInt newFace = faces[f] + Ng, numChildren;
844 
845       ierr = PetscFindInt(faces[f], nleaves, leaves, &loc);CHKERRQ(ierr);
846       ierr = DMPlexGetTreeChildren(dm,faces[f],&numChildren,NULL);CHKERRQ(ierr);
847       if (loc >= 0 || numChildren) continue;
848       if ((faces[f] < fStart) || (faces[f] >= fEnd)) continue;
849       ierr = DMPlexSetCone(gdm, ghostCell, &newFace);CHKERRQ(ierr);
850       ierr = DMPlexInsertSupport(gdm, newFace, 1, ghostCell);CHKERRQ(ierr);
851       ++ghostCell;
852     }
853     ierr = ISRestoreIndices(faceIS, &faces);CHKERRQ(ierr);
854     ierr = ISDestroy(&faceIS);CHKERRQ(ierr);
855   }
856   ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
857   ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
858   ierr = DMPlexShiftCoordinates_Internal(dm, depthShift, gdm);CHKERRQ(ierr);
859   ierr = DMPlexShiftSF_Internal(dm, depthShift, gdm);CHKERRQ(ierr);
860   ierr = DMPlexShiftLabels_Internal(dm, depthShift, gdm);CHKERRQ(ierr);
861   ierr = DMPlexShiftTree_Internal(dm, depthShift, gdm);CHKERRQ(ierr);
862   ierr = PetscFree(depthShift);CHKERRQ(ierr);
863   /* Step 7: Periodicity */
864   ierr = DMGetPeriodicity(dm, &isper, &maxCell, &L, &bd);CHKERRQ(ierr);
865   ierr = DMSetPeriodicity(gdm, isper, maxCell,  L,  bd);CHKERRQ(ierr);
866   if (numGhostCells) *numGhostCells = Ng;
867   PetscFunctionReturn(0);
868 }
869 
870 /*@C
871   DMPlexConstructGhostCells - Construct ghost cells which connect to every boundary face
872 
873   Collective on dm
874 
875   Input Parameters:
876 + dm - The original DM
877 - labelName - The label specifying the boundary faces, or "Face Sets" if this is NULL
878 
879   Output Parameters:
880 + numGhostCells - The number of ghost cells added to the DM
881 - dmGhosted - The new DM
882 
883   Note: If no label exists of that name, one will be created marking all boundary faces
884 
885   Level: developer
886 
887 .seealso: DMCreate()
888 @*/
889 PetscErrorCode DMPlexConstructGhostCells(DM dm, const char labelName[], PetscInt *numGhostCells, DM *dmGhosted)
890 {
891   DM             gdm;
892   DMLabel        label;
893   const char    *name = labelName ? labelName : "Face Sets";
894   PetscInt       dim, Ng = 0, cMax, fMax, eMax, vMax;
895   PetscBool      useCone, useClosure;
896   PetscErrorCode ierr;
897 
898   PetscFunctionBegin;
899   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
900   if (numGhostCells) PetscValidPointer(numGhostCells, 3);
901   PetscValidPointer(dmGhosted, 4);
902   ierr = DMCreate(PetscObjectComm((PetscObject)dm), &gdm);CHKERRQ(ierr);
903   ierr = DMSetType(gdm, DMPLEX);CHKERRQ(ierr);
904   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
905   ierr = DMSetDimension(gdm, dim);CHKERRQ(ierr);
906   ierr = DMGetBasicAdjacency(dm, &useCone, &useClosure);CHKERRQ(ierr);
907   ierr = DMSetBasicAdjacency(gdm, useCone,  useClosure);CHKERRQ(ierr);
908   ierr = DMGetLabel(dm, name, &label);CHKERRQ(ierr);
909   if (!label) {
910     /* Get label for boundary faces */
911     ierr = DMCreateLabel(dm, name);CHKERRQ(ierr);
912     ierr = DMGetLabel(dm, name, &label);CHKERRQ(ierr);
913     ierr = DMPlexMarkBoundaryFaces(dm, 1, label);CHKERRQ(ierr);
914   }
915   ierr = DMPlexConstructGhostCells_Internal(dm, label, &Ng, gdm);CHKERRQ(ierr);
916   ierr = DMCopyBoundary(dm, gdm);CHKERRQ(ierr);
917   ierr = DMCopyDisc(dm, gdm);CHKERRQ(ierr);
918   /* Copy the hybrid bounds */
919   ierr = DMPlexGetHybridBounds(dm, &cMax, &fMax, &eMax, &vMax);CHKERRQ(ierr);
920   if (fMax >= 0) fMax += Ng;
921   if (eMax >= 0) eMax += Ng;
922   if (vMax >= 0) vMax += Ng;
923   ierr = DMPlexSetHybridBounds(gdm, cMax, fMax, eMax, vMax);CHKERRQ(ierr);
924   gdm->setfromoptionscalled = dm->setfromoptionscalled;
925   if (numGhostCells) *numGhostCells = Ng;
926   *dmGhosted = gdm;
927   PetscFunctionReturn(0);
928 }
929 
930 /*
931   We are adding three kinds of points here:
932     Replicated:     Copies of points which exist in the mesh, such as vertices identified across a fault
933     Non-replicated: Points which exist on the fault, but are not replicated
934     Hybrid:         Entirely new points, such as cohesive cells
935 
936   When creating subsequent cohesive cells, we shift the old hybrid cells to the end of the numbering at
937   each depth so that the new split/hybrid points can be inserted as a block.
938 */
939 static PetscErrorCode DMPlexConstructCohesiveCells_Internal(DM dm, DMLabel label, DMLabel splitLabel, DM sdm)
940 {
941   MPI_Comm         comm;
942   IS               valueIS;
943   PetscInt         numSP = 0;       /* The number of depths for which we have replicated points */
944   const PetscInt  *values;          /* List of depths for which we have replicated points */
945   IS              *splitIS;
946   IS              *unsplitIS;
947   PetscInt        *numSplitPoints;     /* The number of replicated points at each depth */
948   PetscInt        *numUnsplitPoints;   /* The number of non-replicated points at each depth which still give rise to hybrid points */
949   PetscInt        *numHybridPoints;    /* The number of new hybrid points at each depth */
950   PetscInt        *numHybridPointsOld; /* The number of existing hybrid points at each depth */
951   const PetscInt **splitPoints;        /* Replicated points for each depth */
952   const PetscInt **unsplitPoints;      /* Non-replicated points for each depth */
953   PetscSection     coordSection;
954   Vec              coordinates;
955   PetscScalar     *coords;
956   PetscInt        *depthMax;           /* The first hybrid point at each depth in the original mesh */
957   PetscInt        *depthEnd;           /* The point limit at each depth in the original mesh */
958   PetscInt        *depthShift;         /* Number of replicated+hybrid points at each depth */
959   PetscInt        *pMaxNew;            /* The first replicated point at each depth in the new mesh, hybrids come after this */
960   PetscInt        *coneNew, *coneONew, *supportNew;
961   PetscInt         shift = 100, shift2 = 200, depth = 0, dep, dim, d, sp, maxConeSize, maxSupportSize, maxConeSizeNew, maxSupportSizeNew, numLabels, vStart, vEnd, pEnd, p, v;
962   PetscErrorCode   ierr;
963 
964   PetscFunctionBegin;
965   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
966   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
967   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
968   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
969   /* Count split points and add cohesive cells */
970   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, &maxSupportSize);CHKERRQ(ierr);
971   ierr = PetscMalloc5(depth+1,&depthMax,depth+1,&depthEnd,2*(depth+1),&depthShift,depth+1,&pMaxNew,depth+1,&numHybridPointsOld);CHKERRQ(ierr);
972   ierr = PetscMalloc7(depth+1,&splitIS,depth+1,&unsplitIS,depth+1,&numSplitPoints,depth+1,&numUnsplitPoints,depth+1,&numHybridPoints,depth+1,&splitPoints,depth+1,&unsplitPoints);CHKERRQ(ierr);
973   ierr = DMPlexGetHybridBounds(dm, depth >= 0 ? &depthMax[depth] : NULL, depth>1 ? &depthMax[depth-1] : NULL, depth>2 ? &depthMax[1] : NULL, depth >= 0 ? &depthMax[0] : NULL);CHKERRQ(ierr);
974   for (d = 0; d <= depth; ++d) {
975     ierr = DMPlexGetDepthStratum(dm, d, NULL, &pMaxNew[d]);CHKERRQ(ierr);
976     depthEnd[d]           = pMaxNew[d];
977     depthMax[d]           = depthMax[d] < 0 ? depthEnd[d] : depthMax[d];
978     numSplitPoints[d]     = 0;
979     numUnsplitPoints[d]   = 0;
980     numHybridPoints[d]    = 0;
981     numHybridPointsOld[d] = depthMax[d] < 0 ? 0 : depthEnd[d] - depthMax[d];
982     splitPoints[d]        = NULL;
983     unsplitPoints[d]      = NULL;
984     splitIS[d]            = NULL;
985     unsplitIS[d]          = NULL;
986     /* we are shifting the existing hybrid points with the stratum behind them, so
987      * the split comes at the end of the normal points, i.e., at depthMax[d] */
988     depthShift[2*d]       = depthMax[d];
989     depthShift[2*d+1]     = 0;
990   }
991   if (label) {
992     ierr = DMLabelGetValueIS(label, &valueIS);CHKERRQ(ierr);
993     ierr = ISGetLocalSize(valueIS, &numSP);CHKERRQ(ierr);
994     ierr = ISGetIndices(valueIS, &values);CHKERRQ(ierr);
995   }
996   for (sp = 0; sp < numSP; ++sp) {
997     const PetscInt dep = values[sp];
998 
999     if ((dep < 0) || (dep > depth)) continue;
1000     ierr = DMLabelGetStratumIS(label, dep, &splitIS[dep]);CHKERRQ(ierr);
1001     if (splitIS[dep]) {
1002       ierr = ISGetLocalSize(splitIS[dep], &numSplitPoints[dep]);CHKERRQ(ierr);
1003       ierr = ISGetIndices(splitIS[dep], &splitPoints[dep]);CHKERRQ(ierr);
1004     }
1005     ierr = DMLabelGetStratumIS(label, shift2+dep, &unsplitIS[dep]);CHKERRQ(ierr);
1006     if (unsplitIS[dep]) {
1007       ierr = ISGetLocalSize(unsplitIS[dep], &numUnsplitPoints[dep]);CHKERRQ(ierr);
1008       ierr = ISGetIndices(unsplitIS[dep], &unsplitPoints[dep]);CHKERRQ(ierr);
1009     }
1010   }
1011   /* Calculate number of hybrid points */
1012   for (d = 1; d <= depth; ++d) numHybridPoints[d]     = numSplitPoints[d-1] + numUnsplitPoints[d-1]; /* There is a hybrid cell/face/edge for every split face/edge/vertex   */
1013   for (d = 0; d <= depth; ++d) depthShift[2*d+1]      = numSplitPoints[d] + numHybridPoints[d];
1014   ierr = DMPlexShiftPointSetUp_Internal(depth,depthShift);CHKERRQ(ierr);
1015   /* the end of the points in this stratum that come before the new points:
1016    * shifting pMaxNew[d] gets the new start of the next stratum, then count back the old hybrid points and the newly
1017    * added points */
1018   for (d = 0; d <= depth; ++d) pMaxNew[d]             = DMPlexShiftPoint_Internal(pMaxNew[d],depth,depthShift) - (numHybridPointsOld[d] + numSplitPoints[d] + numHybridPoints[d]);
1019   ierr = DMPlexShiftSizes_Internal(dm, depthShift, sdm);CHKERRQ(ierr);
1020   /* Step 3: Set cone/support sizes for new points */
1021   for (dep = 0; dep <= depth; ++dep) {
1022     for (p = 0; p < numSplitPoints[dep]; ++p) {
1023       const PetscInt  oldp   = splitPoints[dep][p];
1024       const PetscInt  newp   = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*oldp + depthOffset[dep]*/;
1025       const PetscInt  splitp = p    + pMaxNew[dep];
1026       const PetscInt *support;
1027       PetscInt        coneSize, supportSize, qf, qn, qp, e;
1028 
1029       ierr = DMPlexGetConeSize(dm, oldp, &coneSize);CHKERRQ(ierr);
1030       ierr = DMPlexSetConeSize(sdm, splitp, coneSize);CHKERRQ(ierr);
1031       ierr = DMPlexGetSupportSize(dm, oldp, &supportSize);CHKERRQ(ierr);
1032       ierr = DMPlexSetSupportSize(sdm, splitp, supportSize);CHKERRQ(ierr);
1033       if (dep == depth-1) {
1034         const PetscInt hybcell = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1035 
1036         /* Add cohesive cells, they are prisms */
1037         ierr = DMPlexSetConeSize(sdm, hybcell, 2 + coneSize);CHKERRQ(ierr);
1038       } else if (dep == 0) {
1039         const PetscInt hybedge = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1040 
1041         ierr = DMPlexGetSupport(dm, oldp, &support);CHKERRQ(ierr);
1042         for (e = 0, qn = 0, qp = 0, qf = 0; e < supportSize; ++e) {
1043           PetscInt val;
1044 
1045           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
1046           if (val == 1) ++qf;
1047           if ((val == 1) || (val ==  (shift + 1))) ++qn;
1048           if ((val == 1) || (val == -(shift + 1))) ++qp;
1049         }
1050         /* Split old vertex: Edges into original vertex and new cohesive edge */
1051         ierr = DMPlexSetSupportSize(sdm, newp, qn+1);CHKERRQ(ierr);
1052         /* Split new vertex: Edges into split vertex and new cohesive edge */
1053         ierr = DMPlexSetSupportSize(sdm, splitp, qp+1);CHKERRQ(ierr);
1054         /* Add hybrid edge */
1055         ierr = DMPlexSetConeSize(sdm, hybedge, 2);CHKERRQ(ierr);
1056         ierr = DMPlexSetSupportSize(sdm, hybedge, qf);CHKERRQ(ierr);
1057       } else if (dep == dim-2) {
1058         const PetscInt hybface = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1059 
1060         ierr = DMPlexGetSupport(dm, oldp, &support);CHKERRQ(ierr);
1061         for (e = 0, qn = 0, qp = 0, qf = 0; e < supportSize; ++e) {
1062           PetscInt val;
1063 
1064           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
1065           if (val == dim-1) ++qf;
1066           if ((val == dim-1) || (val ==  (shift + dim-1))) ++qn;
1067           if ((val == dim-1) || (val == -(shift + dim-1))) ++qp;
1068         }
1069         /* Split old edge: Faces into original edge and cohesive face (positive side?) */
1070         ierr = DMPlexSetSupportSize(sdm, newp, qn+1);CHKERRQ(ierr);
1071         /* Split new edge: Faces into split edge and cohesive face (negative side?) */
1072         ierr = DMPlexSetSupportSize(sdm, splitp, qp+1);CHKERRQ(ierr);
1073         /* Add hybrid face */
1074         ierr = DMPlexSetConeSize(sdm, hybface, 4);CHKERRQ(ierr);
1075         ierr = DMPlexSetSupportSize(sdm, hybface, qf);CHKERRQ(ierr);
1076       }
1077     }
1078   }
1079   for (dep = 0; dep <= depth; ++dep) {
1080     for (p = 0; p < numUnsplitPoints[dep]; ++p) {
1081       const PetscInt  oldp   = unsplitPoints[dep][p];
1082       const PetscInt  newp   = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*oldp + depthOffset[dep]*/;
1083       const PetscInt *support;
1084       PetscInt        coneSize, supportSize, qf, e, s;
1085 
1086       ierr = DMPlexGetConeSize(dm, oldp, &coneSize);CHKERRQ(ierr);
1087       ierr = DMPlexGetSupportSize(dm, oldp, &supportSize);CHKERRQ(ierr);
1088       ierr = DMPlexGetSupport(dm, oldp, &support);CHKERRQ(ierr);
1089       if (dep == 0) {
1090         const PetscInt hybedge = p + pMaxNew[dep+1] + numSplitPoints[dep+1] + numSplitPoints[dep];
1091 
1092         /* Unsplit vertex: Edges into original vertex, split edges, and new cohesive edge twice */
1093         for (s = 0, qf = 0; s < supportSize; ++s, ++qf) {
1094           ierr = PetscFindInt(support[s], numSplitPoints[dep+1], splitPoints[dep+1], &e);CHKERRQ(ierr);
1095           if (e >= 0) ++qf;
1096         }
1097         ierr = DMPlexSetSupportSize(sdm, newp, qf+2);CHKERRQ(ierr);
1098         /* Add hybrid edge */
1099         ierr = DMPlexSetConeSize(sdm, hybedge, 2);CHKERRQ(ierr);
1100         for (e = 0, qf = 0; e < supportSize; ++e) {
1101           PetscInt val;
1102 
1103           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
1104           /* Split and unsplit edges produce hybrid faces */
1105           if (val == 1) ++qf;
1106           if (val == (shift2 + 1)) ++qf;
1107         }
1108         ierr = DMPlexSetSupportSize(sdm, hybedge, qf);CHKERRQ(ierr);
1109       } else if (dep == dim-2) {
1110         const PetscInt hybface = p + pMaxNew[dep+1] + numSplitPoints[dep+1] + numSplitPoints[dep];
1111         PetscInt       val;
1112 
1113         for (e = 0, qf = 0; e < supportSize; ++e) {
1114           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
1115           if (val == dim-1) qf += 2;
1116           else              ++qf;
1117         }
1118         /* Unsplit edge: Faces into original edge, split face, and cohesive face twice */
1119         ierr = DMPlexSetSupportSize(sdm, newp, qf+2);CHKERRQ(ierr);
1120         /* Add hybrid face */
1121         for (e = 0, qf = 0; e < supportSize; ++e) {
1122           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
1123           if (val == dim-1) ++qf;
1124         }
1125         ierr = DMPlexSetConeSize(sdm, hybface, 4);CHKERRQ(ierr);
1126         ierr = DMPlexSetSupportSize(sdm, hybface, qf);CHKERRQ(ierr);
1127       }
1128     }
1129   }
1130   /* Step 4: Setup split DM */
1131   ierr = DMSetUp(sdm);CHKERRQ(ierr);
1132   ierr = DMPlexShiftPoints_Internal(dm, depthShift, sdm);CHKERRQ(ierr);
1133   ierr = DMPlexGetMaxSizes(sdm, &maxConeSizeNew, &maxSupportSizeNew);CHKERRQ(ierr);
1134   ierr = PetscMalloc3(PetscMax(maxConeSize, maxConeSizeNew)*3,&coneNew,PetscMax(maxConeSize, maxConeSizeNew)*3,&coneONew,PetscMax(maxSupportSize, maxSupportSizeNew),&supportNew);CHKERRQ(ierr);
1135   /* Step 6: Set cones and supports for new points */
1136   for (dep = 0; dep <= depth; ++dep) {
1137     for (p = 0; p < numSplitPoints[dep]; ++p) {
1138       const PetscInt  oldp   = splitPoints[dep][p];
1139       const PetscInt  newp   = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*oldp + depthOffset[dep]*/;
1140       const PetscInt  splitp = p    + pMaxNew[dep];
1141       const PetscInt *cone, *support, *ornt;
1142       PetscInt        coneSize, supportSize, q, qf, qn, qp, v, e, s;
1143 
1144       ierr = DMPlexGetConeSize(dm, oldp, &coneSize);CHKERRQ(ierr);
1145       ierr = DMPlexGetCone(dm, oldp, &cone);CHKERRQ(ierr);
1146       ierr = DMPlexGetConeOrientation(dm, oldp, &ornt);CHKERRQ(ierr);
1147       ierr = DMPlexGetSupportSize(dm, oldp, &supportSize);CHKERRQ(ierr);
1148       ierr = DMPlexGetSupport(dm, oldp, &support);CHKERRQ(ierr);
1149       if (dep == depth-1) {
1150         PetscBool       hasUnsplit = PETSC_FALSE;
1151         const PetscInt  hybcell    = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1152         const PetscInt *supportF;
1153 
1154         /* Split face:       copy in old face to new face to start */
1155         ierr = DMPlexGetSupport(sdm, newp,  &supportF);CHKERRQ(ierr);
1156         ierr = DMPlexSetSupport(sdm, splitp, supportF);CHKERRQ(ierr);
1157         /* Split old face:   old vertices/edges in cone so no change */
1158         /* Split new face:   new vertices/edges in cone */
1159         for (q = 0; q < coneSize; ++q) {
1160           ierr = PetscFindInt(cone[q], numSplitPoints[dep-1], splitPoints[dep-1], &v);CHKERRQ(ierr);
1161           if (v < 0) {
1162             ierr = PetscFindInt(cone[q], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &v);CHKERRQ(ierr);
1163             if (v < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %d in split or unsplit points of depth %d", cone[q], dep-1);
1164             coneNew[2+q] = DMPlexShiftPoint_Internal(cone[q], depth, depthShift) /*cone[q] + depthOffset[dep-1]*/;
1165             hasUnsplit   = PETSC_TRUE;
1166           } else {
1167             coneNew[2+q] = v + pMaxNew[dep-1];
1168             if (dep > 1) {
1169               const PetscInt *econe;
1170               PetscInt        econeSize, r, vs, vu;
1171 
1172               ierr = DMPlexGetConeSize(dm, cone[q], &econeSize);CHKERRQ(ierr);
1173               ierr = DMPlexGetCone(dm, cone[q], &econe);CHKERRQ(ierr);
1174               for (r = 0; r < econeSize; ++r) {
1175                 ierr = PetscFindInt(econe[r], numSplitPoints[dep-2],   splitPoints[dep-2],   &vs);CHKERRQ(ierr);
1176                 ierr = PetscFindInt(econe[r], numUnsplitPoints[dep-2], unsplitPoints[dep-2], &vu);CHKERRQ(ierr);
1177                 if (vs >= 0) continue;
1178                 if (vu < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %d in split or unsplit points of depth %d", econe[r], dep-2);
1179                 hasUnsplit   = PETSC_TRUE;
1180               }
1181             }
1182           }
1183         }
1184         ierr = DMPlexSetCone(sdm, splitp, &coneNew[2]);CHKERRQ(ierr);
1185         ierr = DMPlexSetConeOrientation(sdm, splitp, ornt);CHKERRQ(ierr);
1186         /* Face support */
1187         for (s = 0; s < supportSize; ++s) {
1188           PetscInt val;
1189 
1190           ierr = DMLabelGetValue(label, support[s], &val);CHKERRQ(ierr);
1191           if (val < 0) {
1192             /* Split old face:   Replace negative side cell with cohesive cell */
1193              ierr = DMPlexInsertSupport(sdm, newp, s, hybcell);CHKERRQ(ierr);
1194           } else {
1195             /* Split new face:   Replace positive side cell with cohesive cell */
1196             ierr = DMPlexInsertSupport(sdm, splitp, s, hybcell);CHKERRQ(ierr);
1197             /* Get orientation for cohesive face */
1198             {
1199               const PetscInt *ncone, *nconeO;
1200               PetscInt        nconeSize, nc;
1201 
1202               ierr = DMPlexGetConeSize(dm, support[s], &nconeSize);CHKERRQ(ierr);
1203               ierr = DMPlexGetCone(dm, support[s], &ncone);CHKERRQ(ierr);
1204               ierr = DMPlexGetConeOrientation(dm, support[s], &nconeO);CHKERRQ(ierr);
1205               for (nc = 0; nc < nconeSize; ++nc) {
1206                 if (ncone[nc] == oldp) {
1207                   coneONew[0] = nconeO[nc];
1208                   break;
1209                 }
1210               }
1211               if (nc >= nconeSize) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate face %d in neighboring cell %d", oldp, support[s]);
1212             }
1213           }
1214         }
1215         /* Cohesive cell:    Old and new split face, then new cohesive faces */
1216         coneNew[0]  = newp;   /* Extracted negative side orientation above */
1217         coneNew[1]  = splitp;
1218         coneONew[1] = coneONew[0];
1219         for (q = 0; q < coneSize; ++q) {
1220           ierr = PetscFindInt(cone[q], numSplitPoints[dep-1], splitPoints[dep-1], &v);CHKERRQ(ierr);
1221           if (v < 0) {
1222             ierr = PetscFindInt(cone[q], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &v);CHKERRQ(ierr);
1223             coneNew[2+q]  = v + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep-1];
1224             coneONew[2+q] = 0;
1225           } else {
1226             coneNew[2+q]  = v + pMaxNew[dep] + numSplitPoints[dep];
1227           }
1228           coneONew[2+q] = 0;
1229         }
1230         ierr = DMPlexSetCone(sdm, hybcell, coneNew);CHKERRQ(ierr);
1231         ierr = DMPlexSetConeOrientation(sdm, hybcell, coneONew);CHKERRQ(ierr);
1232         /* Label the hybrid cells on the boundary of the split */
1233         if (hasUnsplit) {ierr = DMLabelSetValue(label, -hybcell, dim);CHKERRQ(ierr);}
1234       } else if (dep == 0) {
1235         const PetscInt hybedge = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1236 
1237         /* Split old vertex: Edges in old split faces and new cohesive edge */
1238         for (e = 0, qn = 0; e < supportSize; ++e) {
1239           PetscInt val;
1240 
1241           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
1242           if ((val == 1) || (val == (shift + 1))) {
1243             supportNew[qn++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1244           }
1245         }
1246         supportNew[qn] = hybedge;
1247         ierr = DMPlexSetSupport(sdm, newp, supportNew);CHKERRQ(ierr);
1248         /* Split new vertex: Edges in new split faces and new cohesive edge */
1249         for (e = 0, qp = 0; e < supportSize; ++e) {
1250           PetscInt val, edge;
1251 
1252           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
1253           if (val == 1) {
1254             ierr = PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &edge);CHKERRQ(ierr);
1255             if (edge < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Edge %d is not a split edge", support[e]);
1256             supportNew[qp++] = edge + pMaxNew[dep+1];
1257           } else if (val == -(shift + 1)) {
1258             supportNew[qp++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1259           }
1260         }
1261         supportNew[qp] = hybedge;
1262         ierr = DMPlexSetSupport(sdm, splitp, supportNew);CHKERRQ(ierr);
1263         /* Hybrid edge:    Old and new split vertex */
1264         coneNew[0] = newp;
1265         coneNew[1] = splitp;
1266         ierr = DMPlexSetCone(sdm, hybedge, coneNew);CHKERRQ(ierr);
1267         for (e = 0, qf = 0; e < supportSize; ++e) {
1268           PetscInt val, edge;
1269 
1270           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
1271           if (val == 1) {
1272             ierr = PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &edge);CHKERRQ(ierr);
1273             if (edge < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Edge %d is not a split edge", support[e]);
1274             supportNew[qf++] = edge + pMaxNew[dep+2] + numSplitPoints[dep+2];
1275           }
1276         }
1277         ierr = DMPlexSetSupport(sdm, hybedge, supportNew);CHKERRQ(ierr);
1278       } else if (dep == dim-2) {
1279         const PetscInt hybface = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1280 
1281         /* Split old edge:   old vertices in cone so no change */
1282         /* Split new edge:   new vertices in cone */
1283         for (q = 0; q < coneSize; ++q) {
1284           ierr = PetscFindInt(cone[q], numSplitPoints[dep-1], splitPoints[dep-1], &v);CHKERRQ(ierr);
1285           if (v < 0) {
1286             ierr = PetscFindInt(cone[q], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &v);CHKERRQ(ierr);
1287             if (v < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %d in split or unsplit points of depth %d", cone[q], dep-1);
1288             coneNew[q] = DMPlexShiftPoint_Internal(cone[q], depth, depthShift) /*cone[q] + depthOffset[dep-1]*/;
1289           } else {
1290             coneNew[q] = v + pMaxNew[dep-1];
1291           }
1292         }
1293         ierr = DMPlexSetCone(sdm, splitp, coneNew);CHKERRQ(ierr);
1294         /* Split old edge: Faces in positive side cells and old split faces */
1295         for (e = 0, q = 0; e < supportSize; ++e) {
1296           PetscInt val;
1297 
1298           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
1299           if (val == dim-1) {
1300             supportNew[q++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1301           } else if (val == (shift + dim-1)) {
1302             supportNew[q++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1303           }
1304         }
1305         supportNew[q++] = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1306         ierr = DMPlexSetSupport(sdm, newp, supportNew);CHKERRQ(ierr);
1307         /* Split new edge: Faces in negative side cells and new split faces */
1308         for (e = 0, q = 0; e < supportSize; ++e) {
1309           PetscInt val, face;
1310 
1311           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
1312           if (val == dim-1) {
1313             ierr = PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &face);CHKERRQ(ierr);
1314             if (face < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Face %d is not a split face", support[e]);
1315             supportNew[q++] = face + pMaxNew[dep+1];
1316           } else if (val == -(shift + dim-1)) {
1317             supportNew[q++] = DMPlexShiftPoint_Internal(support[e], depth, depthShift) /*support[e] + depthOffset[dep+1]*/;
1318           }
1319         }
1320         supportNew[q++] = p + pMaxNew[dep+1] + numSplitPoints[dep+1];
1321         ierr = DMPlexSetSupport(sdm, splitp, supportNew);CHKERRQ(ierr);
1322         /* Hybrid face */
1323         coneNew[0] = newp;
1324         coneNew[1] = splitp;
1325         for (v = 0; v < coneSize; ++v) {
1326           PetscInt vertex;
1327           ierr = PetscFindInt(cone[v], numSplitPoints[dep-1], splitPoints[dep-1], &vertex);CHKERRQ(ierr);
1328           if (vertex < 0) {
1329             ierr = PetscFindInt(cone[v], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &vertex);CHKERRQ(ierr);
1330             if (vertex < 0) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate point %d in split or unsplit points of depth %d", cone[v], dep-1);
1331             coneNew[2+v] = vertex + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep-1];
1332           } else {
1333             coneNew[2+v] = vertex + pMaxNew[dep] + numSplitPoints[dep];
1334           }
1335         }
1336         ierr = DMPlexSetCone(sdm, hybface, coneNew);CHKERRQ(ierr);
1337         for (e = 0, qf = 0; e < supportSize; ++e) {
1338           PetscInt val, face;
1339 
1340           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
1341           if (val == dim-1) {
1342             ierr = PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &face);CHKERRQ(ierr);
1343             if (face < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Face %d is not a split face", support[e]);
1344             supportNew[qf++] = face + pMaxNew[dep+2] + numSplitPoints[dep+2];
1345           }
1346         }
1347         ierr = DMPlexSetSupport(sdm, hybface, supportNew);CHKERRQ(ierr);
1348       }
1349     }
1350   }
1351   for (dep = 0; dep <= depth; ++dep) {
1352     for (p = 0; p < numUnsplitPoints[dep]; ++p) {
1353       const PetscInt  oldp   = unsplitPoints[dep][p];
1354       const PetscInt  newp   = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*oldp + depthOffset[dep]*/;
1355       const PetscInt *cone, *support, *ornt;
1356       PetscInt        coneSize, supportSize, supportSizeNew, q, qf, e, f, s;
1357 
1358       ierr = DMPlexGetConeSize(dm, oldp, &coneSize);CHKERRQ(ierr);
1359       ierr = DMPlexGetCone(dm, oldp, &cone);CHKERRQ(ierr);
1360       ierr = DMPlexGetConeOrientation(dm, oldp, &ornt);CHKERRQ(ierr);
1361       ierr = DMPlexGetSupportSize(dm, oldp, &supportSize);CHKERRQ(ierr);
1362       ierr = DMPlexGetSupport(dm, oldp, &support);CHKERRQ(ierr);
1363       if (dep == 0) {
1364         const PetscInt hybedge = p + pMaxNew[dep+1] + numSplitPoints[dep+1] + numSplitPoints[dep];
1365 
1366         /* Unsplit vertex */
1367         ierr = DMPlexGetSupportSize(sdm, newp, &supportSizeNew);CHKERRQ(ierr);
1368         for (s = 0, q = 0; s < supportSize; ++s) {
1369           supportNew[q++] = DMPlexShiftPoint_Internal(support[s], depth, depthShift) /*support[s] + depthOffset[dep+1]*/;
1370           ierr = PetscFindInt(support[s], numSplitPoints[dep+1], splitPoints[dep+1], &e);CHKERRQ(ierr);
1371           if (e >= 0) {
1372             supportNew[q++] = e + pMaxNew[dep+1];
1373           }
1374         }
1375         supportNew[q++] = hybedge;
1376         supportNew[q++] = hybedge;
1377         if (q != supportSizeNew) SETERRQ3(comm, PETSC_ERR_ARG_WRONG, "Support size %d != %d for vertex %d", q, supportSizeNew, newp);
1378         ierr = DMPlexSetSupport(sdm, newp, supportNew);CHKERRQ(ierr);
1379         /* Hybrid edge */
1380         coneNew[0] = newp;
1381         coneNew[1] = newp;
1382         ierr = DMPlexSetCone(sdm, hybedge, coneNew);CHKERRQ(ierr);
1383         for (e = 0, qf = 0; e < supportSize; ++e) {
1384           PetscInt val, edge;
1385 
1386           ierr = DMLabelGetValue(label, support[e], &val);CHKERRQ(ierr);
1387           if (val == 1) {
1388             ierr = PetscFindInt(support[e], numSplitPoints[dep+1], splitPoints[dep+1], &edge);CHKERRQ(ierr);
1389             if (edge < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Edge %d is not a split edge", support[e]);
1390             supportNew[qf++] = edge + pMaxNew[dep+2] + numSplitPoints[dep+2];
1391           } else if  (val ==  (shift2 + 1)) {
1392             ierr = PetscFindInt(support[e], numUnsplitPoints[dep+1], unsplitPoints[dep+1], &edge);CHKERRQ(ierr);
1393             if (edge < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Edge %d is not a unsplit edge", support[e]);
1394             supportNew[qf++] = edge + pMaxNew[dep+2] + numSplitPoints[dep+2] + numSplitPoints[dep+1];
1395           }
1396         }
1397         ierr = DMPlexSetSupport(sdm, hybedge, supportNew);CHKERRQ(ierr);
1398       } else if (dep == dim-2) {
1399         const PetscInt hybface = p + pMaxNew[dep+1] + numSplitPoints[dep+1] + numSplitPoints[dep];
1400 
1401         /* Unsplit edge: Faces into original edge, split face, and hybrid face twice */
1402         for (f = 0, qf = 0; f < supportSize; ++f) {
1403           PetscInt val, face;
1404 
1405           ierr = DMLabelGetValue(label, support[f], &val);CHKERRQ(ierr);
1406           if (val == dim-1) {
1407             ierr = PetscFindInt(support[f], numSplitPoints[dep+1], splitPoints[dep+1], &face);CHKERRQ(ierr);
1408             if (face < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Face %d is not a split face", support[f]);
1409             supportNew[qf++] = DMPlexShiftPoint_Internal(support[f], depth, depthShift) /*support[f] + depthOffset[dep+1]*/;
1410             supportNew[qf++] = face + pMaxNew[dep+1];
1411           } else {
1412             supportNew[qf++] = DMPlexShiftPoint_Internal(support[f], depth, depthShift) /*support[f] + depthOffset[dep+1]*/;
1413           }
1414         }
1415         supportNew[qf++] = hybface;
1416         supportNew[qf++] = hybface;
1417         ierr = DMPlexGetSupportSize(sdm, newp, &supportSizeNew);CHKERRQ(ierr);
1418         if (qf != supportSizeNew) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Support size for unsplit edge %d is %d != %d\n", newp, qf, supportSizeNew);
1419         ierr = DMPlexSetSupport(sdm, newp, supportNew);CHKERRQ(ierr);
1420         /* Add hybrid face */
1421         coneNew[0] = newp;
1422         coneNew[1] = newp;
1423         ierr = PetscFindInt(cone[0], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &v);CHKERRQ(ierr);
1424         if (v < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Vertex %d is not an unsplit vertex", cone[0]);
1425         coneNew[2] = v + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep-1];
1426         ierr = PetscFindInt(cone[1], numUnsplitPoints[dep-1], unsplitPoints[dep-1], &v);CHKERRQ(ierr);
1427         if (v < 0) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Vertex %d is not an unsplit vertex", cone[1]);
1428         coneNew[3] = v + pMaxNew[dep] + numSplitPoints[dep] + numSplitPoints[dep-1];
1429         ierr = DMPlexSetCone(sdm, hybface, coneNew);CHKERRQ(ierr);
1430         for (f = 0, qf = 0; f < supportSize; ++f) {
1431           PetscInt val, face;
1432 
1433           ierr = DMLabelGetValue(label, support[f], &val);CHKERRQ(ierr);
1434           if (val == dim-1) {
1435             ierr = PetscFindInt(support[f], numSplitPoints[dep+1], splitPoints[dep+1], &face);CHKERRQ(ierr);
1436             supportNew[qf++] = face + pMaxNew[dep+2] + numSplitPoints[dep+2];
1437           }
1438         }
1439         ierr = DMPlexGetSupportSize(sdm, hybface, &supportSizeNew);CHKERRQ(ierr);
1440         if (qf != supportSizeNew) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Support size for hybrid face %d is %d != %d\n", hybface, qf, supportSizeNew);
1441         ierr = DMPlexSetSupport(sdm, hybface, supportNew);CHKERRQ(ierr);
1442       }
1443     }
1444   }
1445   /* Step 6b: Replace split points in negative side cones */
1446   for (sp = 0; sp < numSP; ++sp) {
1447     PetscInt        dep = values[sp];
1448     IS              pIS;
1449     PetscInt        numPoints;
1450     const PetscInt *points;
1451 
1452     if (dep >= 0) continue;
1453     ierr = DMLabelGetStratumIS(label, dep, &pIS);CHKERRQ(ierr);
1454     if (!pIS) continue;
1455     dep  = -dep - shift;
1456     ierr = ISGetLocalSize(pIS, &numPoints);CHKERRQ(ierr);
1457     ierr = ISGetIndices(pIS, &points);CHKERRQ(ierr);
1458     for (p = 0; p < numPoints; ++p) {
1459       const PetscInt  oldp = points[p];
1460       const PetscInt  newp = DMPlexShiftPoint_Internal(oldp, depth, depthShift) /*depthOffset[dep] + oldp*/;
1461       const PetscInt *cone;
1462       PetscInt        coneSize, c;
1463       /* PetscBool       replaced = PETSC_FALSE; */
1464 
1465       /* Negative edge: replace split vertex */
1466       /* Negative cell: replace split face */
1467       ierr = DMPlexGetConeSize(sdm, newp, &coneSize);CHKERRQ(ierr);
1468       ierr = DMPlexGetCone(sdm, newp, &cone);CHKERRQ(ierr);
1469       for (c = 0; c < coneSize; ++c) {
1470         const PetscInt coldp = DMPlexShiftPointInverse_Internal(cone[c],depth,depthShift);
1471         PetscInt       csplitp, cp, val;
1472 
1473         ierr = DMLabelGetValue(label, coldp, &val);CHKERRQ(ierr);
1474         if (val == dep-1) {
1475           ierr = PetscFindInt(coldp, numSplitPoints[dep-1], splitPoints[dep-1], &cp);CHKERRQ(ierr);
1476           if (cp < 0) SETERRQ2(comm, PETSC_ERR_ARG_WRONG, "Point %d is not a split point of dimension %d", oldp, dep-1);
1477           csplitp  = pMaxNew[dep-1] + cp;
1478           ierr     = DMPlexInsertCone(sdm, newp, c, csplitp);CHKERRQ(ierr);
1479           /* replaced = PETSC_TRUE; */
1480         }
1481       }
1482       /* Cells with only a vertex or edge on the submesh have no replacement */
1483       /* if (!replaced) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "The cone of point %d does not contain split points", oldp); */
1484     }
1485     ierr = ISRestoreIndices(pIS, &points);CHKERRQ(ierr);
1486     ierr = ISDestroy(&pIS);CHKERRQ(ierr);
1487   }
1488   /* Step 7: Coordinates */
1489   ierr = DMPlexShiftCoordinates_Internal(dm, depthShift, sdm);CHKERRQ(ierr);
1490   ierr = DMGetCoordinateSection(sdm, &coordSection);CHKERRQ(ierr);
1491   ierr = DMGetCoordinatesLocal(sdm, &coordinates);CHKERRQ(ierr);
1492   ierr = VecGetArray(coordinates, &coords);CHKERRQ(ierr);
1493   for (v = 0; v < (numSplitPoints ? numSplitPoints[0] : 0); ++v) {
1494     const PetscInt newp   = DMPlexShiftPoint_Internal(splitPoints[0][v], depth, depthShift) /*depthOffset[0] + splitPoints[0][v]*/;
1495     const PetscInt splitp = pMaxNew[0] + v;
1496     PetscInt       dof, off, soff, d;
1497 
1498     ierr = PetscSectionGetDof(coordSection, newp, &dof);CHKERRQ(ierr);
1499     ierr = PetscSectionGetOffset(coordSection, newp, &off);CHKERRQ(ierr);
1500     ierr = PetscSectionGetOffset(coordSection, splitp, &soff);CHKERRQ(ierr);
1501     for (d = 0; d < dof; ++d) coords[soff+d] = coords[off+d];
1502   }
1503   ierr = VecRestoreArray(coordinates, &coords);CHKERRQ(ierr);
1504   /* Step 8: SF, if I can figure this out we can split the mesh in parallel */
1505   ierr = DMPlexShiftSF_Internal(dm, depthShift, sdm);CHKERRQ(ierr);
1506   /* Step 9: Labels */
1507   ierr = DMPlexShiftLabels_Internal(dm, depthShift, sdm);CHKERRQ(ierr);
1508   ierr = DMGetNumLabels(sdm, &numLabels);CHKERRQ(ierr);
1509   for (dep = 0; dep <= depth; ++dep) {
1510     for (p = 0; p < numSplitPoints[dep]; ++p) {
1511       const PetscInt newp   = DMPlexShiftPoint_Internal(splitPoints[dep][p], depth, depthShift) /*depthOffset[dep] + splitPoints[dep][p]*/;
1512       const PetscInt splitp = pMaxNew[dep] + p;
1513       PetscInt       l;
1514 
1515       if (splitLabel) {
1516         const PetscInt val = 100 + dep;
1517 
1518         ierr = DMLabelSetValue(splitLabel, newp,    val);CHKERRQ(ierr);
1519         ierr = DMLabelSetValue(splitLabel, splitp, -val);CHKERRQ(ierr);
1520       }
1521       for (l = 0; l < numLabels; ++l) {
1522         DMLabel     mlabel;
1523         const char *lname;
1524         PetscInt    val;
1525         PetscBool   isDepth;
1526 
1527         ierr = DMGetLabelName(sdm, l, &lname);CHKERRQ(ierr);
1528         ierr = PetscStrcmp(lname, "depth", &isDepth);CHKERRQ(ierr);
1529         if (isDepth) continue;
1530         ierr = DMGetLabel(sdm, lname, &mlabel);CHKERRQ(ierr);
1531         ierr = DMLabelGetValue(mlabel, newp, &val);CHKERRQ(ierr);
1532         if (val >= 0) {
1533           ierr = DMLabelSetValue(mlabel, splitp, val);CHKERRQ(ierr);
1534         }
1535       }
1536     }
1537   }
1538   for (sp = 0; sp < numSP; ++sp) {
1539     const PetscInt dep = values[sp];
1540 
1541     if ((dep < 0) || (dep > depth)) continue;
1542     if (splitIS[dep]) {ierr = ISRestoreIndices(splitIS[dep], &splitPoints[dep]);CHKERRQ(ierr);}
1543     ierr = ISDestroy(&splitIS[dep]);CHKERRQ(ierr);
1544     if (unsplitIS[dep]) {ierr = ISRestoreIndices(unsplitIS[dep], &unsplitPoints[dep]);CHKERRQ(ierr);}
1545     ierr = ISDestroy(&unsplitIS[dep]);CHKERRQ(ierr);
1546   }
1547   if (label) {
1548     ierr = ISRestoreIndices(valueIS, &values);CHKERRQ(ierr);
1549     ierr = ISDestroy(&valueIS);CHKERRQ(ierr);
1550   }
1551   for (d = 0; d <= depth; ++d) {
1552     ierr = DMPlexGetDepthStratum(sdm, d, NULL, &pEnd);CHKERRQ(ierr);
1553     pMaxNew[d] = pEnd - numHybridPoints[d] - numHybridPointsOld[d];
1554   }
1555   ierr = DMPlexSetHybridBounds(sdm, depth >= 0 ? pMaxNew[depth] : PETSC_DETERMINE, depth>1 ? pMaxNew[depth-1] : PETSC_DETERMINE, depth>2 ? pMaxNew[1] : PETSC_DETERMINE, depth >= 0 ? pMaxNew[0] : PETSC_DETERMINE);CHKERRQ(ierr);
1556   ierr = PetscFree3(coneNew, coneONew, supportNew);CHKERRQ(ierr);
1557   ierr = PetscFree5(depthMax, depthEnd, depthShift, pMaxNew, numHybridPointsOld);CHKERRQ(ierr);
1558   ierr = PetscFree7(splitIS, unsplitIS, numSplitPoints, numUnsplitPoints, numHybridPoints, splitPoints, unsplitPoints);CHKERRQ(ierr);
1559   PetscFunctionReturn(0);
1560 }
1561 
1562 /*@C
1563   DMPlexConstructCohesiveCells - Construct cohesive cells which split the face along an internal interface
1564 
1565   Collective on dm
1566 
1567   Input Parameters:
1568 + dm - The original DM
1569 - label - The label specifying the boundary faces (this could be auto-generated)
1570 
1571   Output Parameters:
1572 + splitLabel - The label containing the split points, or NULL if no output is desired
1573 - dmSplit - The new DM
1574 
1575   Level: developer
1576 
1577 .seealso: DMCreate(), DMPlexLabelCohesiveComplete()
1578 @*/
1579 PetscErrorCode DMPlexConstructCohesiveCells(DM dm, DMLabel label, DMLabel splitLabel, DM *dmSplit)
1580 {
1581   DM             sdm;
1582   PetscInt       dim;
1583   PetscErrorCode ierr;
1584 
1585   PetscFunctionBegin;
1586   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1587   PetscValidPointer(dmSplit, 3);
1588   ierr = DMCreate(PetscObjectComm((PetscObject)dm), &sdm);CHKERRQ(ierr);
1589   ierr = DMSetType(sdm, DMPLEX);CHKERRQ(ierr);
1590   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
1591   ierr = DMSetDimension(sdm, dim);CHKERRQ(ierr);
1592   switch (dim) {
1593   case 2:
1594   case 3:
1595     ierr = DMPlexConstructCohesiveCells_Internal(dm, label, splitLabel, sdm);CHKERRQ(ierr);
1596     break;
1597   default:
1598     SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Cannot construct cohesive cells for dimension %d", dim);
1599   }
1600   *dmSplit = sdm;
1601   PetscFunctionReturn(0);
1602 }
1603 
1604 /* Returns the side of the surface for a given cell with a face on the surface */
1605 static PetscErrorCode GetSurfaceSide_Static(DM dm, DM subdm, PetscInt numSubpoints, const PetscInt *subpoints, PetscInt cell, PetscInt face, PetscBool *pos)
1606 {
1607   const PetscInt *cone, *ornt;
1608   PetscInt        dim, coneSize, c;
1609   PetscErrorCode  ierr;
1610 
1611   PetscFunctionBegin;
1612   *pos = PETSC_TRUE;
1613   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
1614   ierr = DMPlexGetConeSize(dm, cell, &coneSize);CHKERRQ(ierr);
1615   ierr = DMPlexGetCone(dm, cell, &cone);CHKERRQ(ierr);
1616   ierr = DMPlexGetConeOrientation(dm, cell, &ornt);CHKERRQ(ierr);
1617   for (c = 0; c < coneSize; ++c) {
1618     if (cone[c] == face) {
1619       PetscInt o = ornt[c];
1620 
1621       if (subdm) {
1622         const PetscInt *subcone, *subornt;
1623         PetscInt        subpoint, subface, subconeSize, sc;
1624 
1625         ierr = PetscFindInt(cell, numSubpoints, subpoints, &subpoint);CHKERRQ(ierr);
1626         ierr = PetscFindInt(face, numSubpoints, subpoints, &subface);CHKERRQ(ierr);
1627         ierr = DMPlexGetConeSize(subdm, subpoint, &subconeSize);CHKERRQ(ierr);
1628         ierr = DMPlexGetCone(subdm, subpoint, &subcone);CHKERRQ(ierr);
1629         ierr = DMPlexGetConeOrientation(subdm, subpoint, &subornt);CHKERRQ(ierr);
1630         for (sc = 0; sc < subconeSize; ++sc) {
1631           if (subcone[sc] == subface) {
1632             o = subornt[0];
1633             break;
1634           }
1635         }
1636         if (sc >= subconeSize) SETERRQ4(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not find subpoint %d (%d) in cone for subpoint %d (%d)", subface, face, subpoint, cell);
1637       }
1638       if (o >= 0) *pos = PETSC_TRUE;
1639       else        *pos = PETSC_FALSE;
1640       break;
1641     }
1642   }
1643   if (c == coneSize) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Cell %d in split face %d support does not have it in the cone", cell, face);
1644   PetscFunctionReturn(0);
1645 }
1646 
1647 /*@
1648   DMPlexLabelCohesiveComplete - Starting with a label marking points on an internal surface, we add all other mesh pieces
1649   to complete the surface
1650 
1651   Input Parameters:
1652 + dm     - The DM
1653 . label  - A DMLabel marking the surface
1654 . blabel - A DMLabel marking the vertices on the boundary which will not be duplicated, or NULL to find them automatically
1655 . flip   - Flag to flip the submesh normal and replace points on the other side
1656 - subdm  - The subDM associated with the label, or NULL
1657 
1658   Output Parameter:
1659 . label - A DMLabel marking all surface points
1660 
1661   Note: The vertices in blabel are called "unsplit" in the terminology from hybrid cell creation.
1662 
1663   Level: developer
1664 
1665 .seealso: DMPlexConstructCohesiveCells(), DMPlexLabelComplete()
1666 @*/
1667 PetscErrorCode DMPlexLabelCohesiveComplete(DM dm, DMLabel label, DMLabel blabel, PetscBool flip, DM subdm)
1668 {
1669   DMLabel         depthLabel;
1670   IS              dimIS, subpointIS = NULL, facePosIS, faceNegIS, crossEdgeIS = NULL;
1671   const PetscInt *points, *subpoints;
1672   const PetscInt  rev   = flip ? -1 : 1;
1673   PetscInt       *pMax;
1674   PetscInt        shift = 100, shift2 = 200, dim, depth, pSize, dep, cStart, cEnd, cMax, fStart, fEnd, vStart, vEnd, numPoints, numSubpoints, p, val;
1675   PetscErrorCode  ierr;
1676 
1677   PetscFunctionBegin;
1678   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
1679   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
1680   pSize = PetscMax(depth, dim) + 1;
1681   ierr = PetscMalloc1(pSize,&pMax);CHKERRQ(ierr);
1682   ierr = DMPlexGetHybridBounds(dm, depth >= 0 ? &pMax[depth] : NULL, depth>1 ? &pMax[depth-1] : NULL, depth>2 ? &pMax[1] : NULL, &pMax[0]);CHKERRQ(ierr);
1683   ierr = DMPlexGetDepthLabel(dm, &depthLabel);CHKERRQ(ierr);
1684   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
1685   if (subdm) {
1686     ierr = DMPlexCreateSubpointIS(subdm, &subpointIS);CHKERRQ(ierr);
1687     if (subpointIS) {
1688       ierr = ISGetLocalSize(subpointIS, &numSubpoints);CHKERRQ(ierr);
1689       ierr = ISGetIndices(subpointIS, &subpoints);CHKERRQ(ierr);
1690     }
1691   }
1692   /* Mark cell on the fault, and its faces which touch the fault: cell orientation for face gives the side of the fault */
1693   ierr = DMLabelGetStratumIS(label, dim-1, &dimIS);CHKERRQ(ierr);
1694   if (!dimIS) {
1695     ierr = PetscFree(pMax);CHKERRQ(ierr);
1696     ierr = ISDestroy(&subpointIS);CHKERRQ(ierr);
1697     PetscFunctionReturn(0);
1698   }
1699   ierr = ISGetLocalSize(dimIS, &numPoints);CHKERRQ(ierr);
1700   ierr = ISGetIndices(dimIS, &points);CHKERRQ(ierr);
1701   for (p = 0; p < numPoints; ++p) { /* Loop over fault faces */
1702     const PetscInt *support;
1703     PetscInt        supportSize, s;
1704 
1705     ierr = DMPlexGetSupportSize(dm, points[p], &supportSize);CHKERRQ(ierr);
1706 #if 0
1707     if (supportSize != 2) {
1708       const PetscInt *lp;
1709       PetscInt        Nlp, pind;
1710 
1711       /* Check that for a cell with a single support face, that face is in the SF */
1712       /*   THis check only works for the remote side. We would need root side information */
1713       ierr = PetscSFGetGraph(dm->sf, NULL, &Nlp, &lp, NULL);CHKERRQ(ierr);
1714       ierr = PetscFindInt(points[p], Nlp, lp, &pind);CHKERRQ(ierr);
1715       if (pind < 0) SETERRQ2(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Split face %d has %d != 2 supports, and the face is not shared with another process", points[p], supportSize);
1716     }
1717 #endif
1718     ierr = DMPlexGetSupport(dm, points[p], &support);CHKERRQ(ierr);
1719     for (s = 0; s < supportSize; ++s) {
1720       const PetscInt *cone;
1721       PetscInt        coneSize, c;
1722       PetscBool       pos;
1723 
1724       ierr = GetSurfaceSide_Static(dm, subdm, numSubpoints, subpoints, support[s], points[p], &pos);CHKERRQ(ierr);
1725       if (pos) {ierr = DMLabelSetValue(label, support[s],  rev*(shift+dim));CHKERRQ(ierr);}
1726       else     {ierr = DMLabelSetValue(label, support[s], -rev*(shift+dim));CHKERRQ(ierr);}
1727       if (rev < 0) pos = !pos ? PETSC_TRUE : PETSC_FALSE;
1728       /* Put faces touching the fault in the label */
1729       ierr = DMPlexGetConeSize(dm, support[s], &coneSize);CHKERRQ(ierr);
1730       ierr = DMPlexGetCone(dm, support[s], &cone);CHKERRQ(ierr);
1731       for (c = 0; c < coneSize; ++c) {
1732         const PetscInt point = cone[c];
1733 
1734         ierr = DMLabelGetValue(label, point, &val);CHKERRQ(ierr);
1735         if (val == -1) {
1736           PetscInt *closure = NULL;
1737           PetscInt  closureSize, cl;
1738 
1739           ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1740           for (cl = 0; cl < closureSize*2; cl += 2) {
1741             const PetscInt clp  = closure[cl];
1742             PetscInt       bval = -1;
1743 
1744             ierr = DMLabelGetValue(label, clp, &val);CHKERRQ(ierr);
1745             if (blabel) {ierr = DMLabelGetValue(blabel, clp, &bval);CHKERRQ(ierr);}
1746             if ((val >= 0) && (val < dim-1) && (bval < 0)) {
1747               ierr = DMLabelSetValue(label, point, pos == PETSC_TRUE ? shift+dim-1 : -(shift+dim-1));CHKERRQ(ierr);
1748               break;
1749             }
1750           }
1751           ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1752         }
1753       }
1754     }
1755   }
1756   ierr = ISRestoreIndices(dimIS, &points);CHKERRQ(ierr);
1757   ierr = ISDestroy(&dimIS);CHKERRQ(ierr);
1758   if (subpointIS) {ierr = ISRestoreIndices(subpointIS, &subpoints);CHKERRQ(ierr);}
1759   ierr = ISDestroy(&subpointIS);CHKERRQ(ierr);
1760   /* Mark boundary points as unsplit */
1761   if (blabel) {
1762     ierr = DMLabelGetStratumIS(blabel, 1, &dimIS);CHKERRQ(ierr);
1763     ierr = ISGetLocalSize(dimIS, &numPoints);CHKERRQ(ierr);
1764     ierr = ISGetIndices(dimIS, &points);CHKERRQ(ierr);
1765     for (p = 0; p < numPoints; ++p) {
1766       const PetscInt point = points[p];
1767       PetscInt       val, bval;
1768 
1769       ierr = DMLabelGetValue(blabel, point, &bval);CHKERRQ(ierr);
1770       if (bval >= 0) {
1771         ierr = DMLabelGetValue(label, point, &val);CHKERRQ(ierr);
1772         if ((val < 0) || (val > dim)) {
1773           /* This could be a point added from splitting a vertex on an adjacent fault, otherwise its just wrong */
1774           ierr = DMLabelClearValue(blabel, point, bval);CHKERRQ(ierr);
1775         }
1776       }
1777     }
1778     for (p = 0; p < numPoints; ++p) {
1779       const PetscInt point = points[p];
1780       PetscInt       val, bval;
1781 
1782       ierr = DMLabelGetValue(blabel, point, &bval);CHKERRQ(ierr);
1783       if (bval >= 0) {
1784         const PetscInt *cone,    *support;
1785         PetscInt        coneSize, supportSize, s, valA, valB, valE;
1786 
1787         /* Mark as unsplit */
1788         ierr = DMLabelGetValue(label, point, &val);CHKERRQ(ierr);
1789         if ((val < 0) || (val > dim)) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Point %d has label value %d, should be part of the fault", point, val);
1790         ierr = DMLabelClearValue(label, point, val);CHKERRQ(ierr);
1791         ierr = DMLabelSetValue(label, point, shift2+val);CHKERRQ(ierr);
1792         /* Check for cross-edge
1793              A cross-edge has endpoints which are both on the boundary of the surface, but the edge itself is not. */
1794         if (val != 0) continue;
1795         ierr = DMPlexGetSupport(dm, point, &support);CHKERRQ(ierr);
1796         ierr = DMPlexGetSupportSize(dm, point, &supportSize);CHKERRQ(ierr);
1797         for (s = 0; s < supportSize; ++s) {
1798           ierr = DMPlexGetCone(dm, support[s], &cone);CHKERRQ(ierr);
1799           ierr = DMPlexGetConeSize(dm, support[s], &coneSize);CHKERRQ(ierr);
1800           if (coneSize != 2) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Edge %D has %D vertices != 2", support[s], coneSize);
1801           ierr = DMLabelGetValue(blabel, cone[0], &valA);CHKERRQ(ierr);
1802           ierr = DMLabelGetValue(blabel, cone[1], &valB);CHKERRQ(ierr);
1803           ierr = DMLabelGetValue(blabel, support[s], &valE);CHKERRQ(ierr);
1804           if ((valE < 0) && (valA >= 0) && (valB >= 0) && (cone[0] != cone[1])) {ierr = DMLabelSetValue(blabel, support[s], 2);CHKERRQ(ierr);}
1805         }
1806       }
1807     }
1808     ierr = ISRestoreIndices(dimIS, &points);CHKERRQ(ierr);
1809     ierr = ISDestroy(&dimIS);CHKERRQ(ierr);
1810   }
1811   /* Search for other cells/faces/edges connected to the fault by a vertex */
1812   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
1813   ierr = DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd);CHKERRQ(ierr);
1814   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
1815   cMax = cMax < 0 ? cEnd : cMax;
1816   ierr = DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd);CHKERRQ(ierr);
1817   ierr = DMLabelGetStratumIS(label, 0, &dimIS);CHKERRQ(ierr);
1818   if (blabel) {ierr = DMLabelGetStratumIS(blabel, 2, &crossEdgeIS);CHKERRQ(ierr);}
1819   if (dimIS && crossEdgeIS) {
1820     IS vertIS = dimIS;
1821 
1822     ierr = ISExpand(vertIS, crossEdgeIS, &dimIS);CHKERRQ(ierr);
1823     ierr = ISDestroy(&crossEdgeIS);CHKERRQ(ierr);
1824     ierr = ISDestroy(&vertIS);CHKERRQ(ierr);
1825   }
1826   if (!dimIS) {
1827     ierr = PetscFree(pMax);CHKERRQ(ierr);
1828     PetscFunctionReturn(0);
1829   }
1830   ierr = ISGetLocalSize(dimIS, &numPoints);CHKERRQ(ierr);
1831   ierr = ISGetIndices(dimIS, &points);CHKERRQ(ierr);
1832   for (p = 0; p < numPoints; ++p) { /* Loop over fault vertices */
1833     PetscInt *star = NULL;
1834     PetscInt  starSize, s;
1835     PetscInt  again = 1;  /* 0: Finished 1: Keep iterating after a change 2: No change */
1836 
1837     /* All points connected to the fault are inside a cell, so at the top level we will only check cells */
1838     ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_FALSE, &starSize, &star);CHKERRQ(ierr);
1839     while (again) {
1840       if (again > 1) SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Could not classify all cells connected to the fault");
1841       again = 0;
1842       for (s = 0; s < starSize*2; s += 2) {
1843         const PetscInt  point = star[s];
1844         const PetscInt *cone;
1845         PetscInt        coneSize, c;
1846 
1847         if ((point < cStart) || (point >= cMax)) continue;
1848         ierr = DMLabelGetValue(label, point, &val);CHKERRQ(ierr);
1849         if (val != -1) continue;
1850         again = again == 1 ? 1 : 2;
1851         ierr  = DMPlexGetConeSize(dm, point, &coneSize);CHKERRQ(ierr);
1852         ierr  = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
1853         for (c = 0; c < coneSize; ++c) {
1854           ierr = DMLabelGetValue(label, cone[c], &val);CHKERRQ(ierr);
1855           if (val != -1) {
1856             const PetscInt *ccone;
1857             PetscInt        cconeSize, cc, side;
1858 
1859             if (PetscAbs(val) < shift) SETERRQ3(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Face %d on cell %d has an invalid label %d", cone[c], point, val);
1860             if (val > 0) side =  1;
1861             else         side = -1;
1862             ierr = DMLabelSetValue(label, point, side*(shift+dim));CHKERRQ(ierr);
1863             /* Mark cell faces which touch the fault */
1864             ierr = DMPlexGetConeSize(dm, point, &cconeSize);CHKERRQ(ierr);
1865             ierr = DMPlexGetCone(dm, point, &ccone);CHKERRQ(ierr);
1866             for (cc = 0; cc < cconeSize; ++cc) {
1867               PetscInt *closure = NULL;
1868               PetscInt  closureSize, cl;
1869 
1870               ierr = DMLabelGetValue(label, ccone[cc], &val);CHKERRQ(ierr);
1871               if (val != -1) continue;
1872               ierr = DMPlexGetTransitiveClosure(dm, ccone[cc], PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1873               for (cl = 0; cl < closureSize*2; cl += 2) {
1874                 const PetscInt clp = closure[cl];
1875 
1876                 ierr = DMLabelGetValue(label, clp, &val);CHKERRQ(ierr);
1877                 if (val == -1) continue;
1878                 ierr = DMLabelSetValue(label, ccone[cc], side*(shift+dim-1));CHKERRQ(ierr);
1879                 break;
1880               }
1881               ierr = DMPlexRestoreTransitiveClosure(dm, ccone[cc], PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1882             }
1883             again = 1;
1884             break;
1885           }
1886         }
1887       }
1888     }
1889     /* Classify the rest by cell membership */
1890     for (s = 0; s < starSize*2; s += 2) {
1891       const PetscInt point = star[s];
1892 
1893       ierr = DMLabelGetValue(label, point, &val);CHKERRQ(ierr);
1894       if (val == -1) {
1895         PetscInt  *sstar = NULL;
1896         PetscInt   sstarSize, ss;
1897         PetscBool  marked = PETSC_FALSE;
1898 
1899         ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_FALSE, &sstarSize, &sstar);CHKERRQ(ierr);
1900         for (ss = 0; ss < sstarSize*2; ss += 2) {
1901           const PetscInt spoint = sstar[ss];
1902 
1903           if ((spoint < cStart) || (spoint >= cMax)) continue;
1904           ierr = DMLabelGetValue(label, spoint, &val);CHKERRQ(ierr);
1905           if (val == -1) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Cell %d in star of %d does not have a valid label", spoint, point);
1906           ierr = DMLabelGetValue(depthLabel, point, &dep);CHKERRQ(ierr);
1907           if (val > 0) {
1908             ierr = DMLabelSetValue(label, point,   shift+dep);CHKERRQ(ierr);
1909           } else {
1910             ierr = DMLabelSetValue(label, point, -(shift+dep));CHKERRQ(ierr);
1911           }
1912           marked = PETSC_TRUE;
1913           break;
1914         }
1915         ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_FALSE, &sstarSize, &sstar);CHKERRQ(ierr);
1916         ierr = DMLabelGetValue(depthLabel, point, &dep);CHKERRQ(ierr);
1917         if (point < pMax[dep] && !marked) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Point %d could not be classified", point);
1918       }
1919     }
1920     ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_FALSE, &starSize, &star);CHKERRQ(ierr);
1921   }
1922   ierr = ISRestoreIndices(dimIS, &points);CHKERRQ(ierr);
1923   ierr = ISDestroy(&dimIS);CHKERRQ(ierr);
1924   /* If any faces touching the fault divide cells on either side, split them */
1925   ierr = DMLabelGetStratumIS(label,   shift+dim-1,  &facePosIS);CHKERRQ(ierr);
1926   ierr = DMLabelGetStratumIS(label, -(shift+dim-1), &faceNegIS);CHKERRQ(ierr);
1927   ierr = ISExpand(facePosIS, faceNegIS, &dimIS);CHKERRQ(ierr);
1928   ierr = ISDestroy(&facePosIS);CHKERRQ(ierr);
1929   ierr = ISDestroy(&faceNegIS);CHKERRQ(ierr);
1930   ierr = ISGetLocalSize(dimIS, &numPoints);CHKERRQ(ierr);
1931   ierr = ISGetIndices(dimIS, &points);CHKERRQ(ierr);
1932   for (p = 0; p < numPoints; ++p) {
1933     const PetscInt  point = points[p];
1934     const PetscInt *support;
1935     PetscInt        supportSize, valA, valB;
1936 
1937     ierr = DMPlexGetSupportSize(dm, point, &supportSize);CHKERRQ(ierr);
1938     if (supportSize != 2) continue;
1939     ierr = DMPlexGetSupport(dm, point, &support);CHKERRQ(ierr);
1940     ierr = DMLabelGetValue(label, support[0], &valA);CHKERRQ(ierr);
1941     ierr = DMLabelGetValue(label, support[1], &valB);CHKERRQ(ierr);
1942     if ((valA == -1) || (valB == -1)) continue;
1943     if (valA*valB > 0) continue;
1944     /* Split the face */
1945     ierr = DMLabelGetValue(label, point, &valA);CHKERRQ(ierr);
1946     ierr = DMLabelClearValue(label, point, valA);CHKERRQ(ierr);
1947     ierr = DMLabelSetValue(label, point, dim-1);CHKERRQ(ierr);
1948     /* Label its closure:
1949       unmarked: label as unsplit
1950       incident: relabel as split
1951       split:    do nothing
1952     */
1953     {
1954       PetscInt *closure = NULL;
1955       PetscInt  closureSize, cl;
1956 
1957       ierr = DMPlexGetTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1958       for (cl = 0; cl < closureSize*2; cl += 2) {
1959         ierr = DMLabelGetValue(label, closure[cl], &valA);CHKERRQ(ierr);
1960         if (valA == -1) { /* Mark as unsplit */
1961           ierr = DMLabelGetValue(depthLabel, closure[cl], &dep);CHKERRQ(ierr);
1962           ierr = DMLabelSetValue(label, closure[cl], shift2+dep);CHKERRQ(ierr);
1963         } else if (((valA >= shift) && (valA < shift2)) || ((valA <= -shift) && (valA > -shift2))) {
1964           ierr = DMLabelGetValue(depthLabel, closure[cl], &dep);CHKERRQ(ierr);
1965           ierr = DMLabelClearValue(label, closure[cl], valA);CHKERRQ(ierr);
1966           ierr = DMLabelSetValue(label, closure[cl], dep);CHKERRQ(ierr);
1967         }
1968       }
1969       ierr = DMPlexRestoreTransitiveClosure(dm, point, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
1970     }
1971   }
1972   ierr = ISRestoreIndices(dimIS, &points);CHKERRQ(ierr);
1973   ierr = ISDestroy(&dimIS);CHKERRQ(ierr);
1974   ierr = PetscFree(pMax);CHKERRQ(ierr);
1975   PetscFunctionReturn(0);
1976 }
1977 
1978 /* Check that no cell have all vertices on the fault */
1979 PetscErrorCode DMPlexCheckValidSubmesh_Private(DM dm, DMLabel label, DM subdm)
1980 {
1981   IS              subpointIS;
1982   const PetscInt *dmpoints;
1983   PetscInt        defaultValue, cStart, cEnd, c, vStart, vEnd;
1984   PetscErrorCode  ierr;
1985 
1986   PetscFunctionBegin;
1987   if (!label) PetscFunctionReturn(0);
1988   ierr = DMLabelGetDefaultValue(label, &defaultValue);CHKERRQ(ierr);
1989   ierr = DMPlexCreateSubpointIS(subdm, &subpointIS);CHKERRQ(ierr);
1990   if (!subpointIS) PetscFunctionReturn(0);
1991   ierr = DMPlexGetHeightStratum(subdm, 0, &cStart, &cEnd);CHKERRQ(ierr);
1992   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
1993   ierr = ISGetIndices(subpointIS, &dmpoints);CHKERRQ(ierr);
1994   for (c = cStart; c < cEnd; ++c) {
1995     PetscBool invalidCell = PETSC_TRUE;
1996     PetscInt *closure     = NULL;
1997     PetscInt  closureSize, cl;
1998 
1999     ierr = DMPlexGetTransitiveClosure(dm, dmpoints[c], PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
2000     for (cl = 0; cl < closureSize*2; cl += 2) {
2001       PetscInt value = 0;
2002 
2003       if ((closure[cl] < vStart) || (closure[cl] >= vEnd)) continue;
2004       ierr = DMLabelGetValue(label, closure[cl], &value);CHKERRQ(ierr);
2005       if (value == defaultValue) {invalidCell = PETSC_FALSE; break;}
2006     }
2007     ierr = DMPlexRestoreTransitiveClosure(dm, dmpoints[c], PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
2008     if (invalidCell) {
2009       ierr = ISRestoreIndices(subpointIS, &dmpoints);CHKERRQ(ierr);
2010       ierr = ISDestroy(&subpointIS);CHKERRQ(ierr);
2011       ierr = DMDestroy(&subdm);CHKERRQ(ierr);
2012       SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Ambiguous submesh. Cell %D has all of its vertices on the submesh.", dmpoints[c]);
2013     }
2014   }
2015   ierr = ISRestoreIndices(subpointIS, &dmpoints);CHKERRQ(ierr);
2016   ierr = ISDestroy(&subpointIS);CHKERRQ(ierr);
2017   PetscFunctionReturn(0);
2018 }
2019 
2020 
2021 /*@
2022   DMPlexCreateHybridMesh - Create a mesh with hybrid cells along an internal interface
2023 
2024   Collective on dm
2025 
2026   Input Parameters:
2027 + dm - The original DM
2028 . label - The label specifying the interface vertices
2029 - bdlabel - The optional label specifying the interface boundary vertices
2030 
2031   Output Parameters:
2032 + hybridLabel - The label fully marking the interface, or NULL if no output is desired
2033 . splitLabel - The label containing the split points, or NULL if no output is desired
2034 . dmInterface - The new interface DM, or NULL
2035 - dmHybrid - The new DM with cohesive cells
2036 
2037   Note: The hybridLabel indicates what parts of the original mesh impinged on the on division surface. For points
2038   directly on the division surface, they are labeled with their dimension, so an edge 7 on the division surface would be
2039   7 (1) in hybridLabel. For points that impinge from the positive side, they are labeled with 100+dim, so an edge 6 with
2040   one vertex 3 on the surface would be 6 (101) and 3 (0) in hybridLabel. If an edge 9 from the negative side of the
2041   surface also hits vertex 3, it would be 9 (-101) in hybridLabel.
2042 
2043   The splitLabel indicates what points in the new hybrid mesh were the result of splitting points in the original
2044   mesh. The label value is +=100+dim for each point. For example, if two edges 10 and 14 in the hybrid resulting from
2045   splitting an edge in the original mesh, you would have 10 (101) and 14 (-101) in the splitLabel.
2046 
2047   The dmInterface is a DM built from the original division surface. It has a label which can be retrieved using
2048   DMPlexGetSubpointMap() which maps each point back to the point in the surface of the original mesh.
2049 
2050   Level: developer
2051 
2052 .seealso: DMPlexConstructCohesiveCells(), DMPlexLabelCohesiveComplete(), DMPlexGetSubpointMap(), DMCreate()
2053 @*/
2054 PetscErrorCode DMPlexCreateHybridMesh(DM dm, DMLabel label, DMLabel bdlabel, DMLabel *hybridLabel, DMLabel *splitLabel, DM *dmInterface, DM *dmHybrid)
2055 {
2056   DM             idm;
2057   DMLabel        subpointMap, hlabel, slabel = NULL;
2058   PetscInt       dim;
2059   PetscErrorCode ierr;
2060 
2061   PetscFunctionBegin;
2062   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2063   if (bdlabel) PetscValidPointer(bdlabel, 3);
2064   if (hybridLabel) PetscValidPointer(hybridLabel, 4);
2065   if (splitLabel)  PetscValidPointer(splitLabel, 5);
2066   if (dmInterface) PetscValidPointer(dmInterface, 6);
2067   PetscValidPointer(dmHybrid, 7);
2068   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
2069   ierr = DMPlexCreateSubmesh(dm, label, 1, PETSC_FALSE, &idm);CHKERRQ(ierr);
2070   ierr = DMPlexCheckValidSubmesh_Private(dm, label, idm);CHKERRQ(ierr);
2071   ierr = DMPlexOrient(idm);CHKERRQ(ierr);
2072   ierr = DMPlexGetSubpointMap(idm, &subpointMap);CHKERRQ(ierr);
2073   ierr = DMLabelDuplicate(subpointMap, &hlabel);CHKERRQ(ierr);
2074   ierr = DMLabelClearStratum(hlabel, dim);CHKERRQ(ierr);
2075   if (splitLabel) {
2076     const char *name;
2077     char        sname[PETSC_MAX_PATH_LEN];
2078 
2079     ierr = PetscObjectGetName((PetscObject) hlabel, &name);CHKERRQ(ierr);
2080     ierr = PetscStrncpy(sname, name, PETSC_MAX_PATH_LEN);CHKERRQ(ierr);
2081     ierr = PetscStrcat(sname, " split");CHKERRQ(ierr);
2082     ierr = DMLabelCreate(PETSC_COMM_SELF, sname, &slabel);CHKERRQ(ierr);
2083   }
2084   ierr = DMPlexLabelCohesiveComplete(dm, hlabel, bdlabel, PETSC_FALSE, idm);CHKERRQ(ierr);
2085   if (dmInterface) {*dmInterface = idm;}
2086   else             {ierr = DMDestroy(&idm);CHKERRQ(ierr);}
2087   ierr = DMPlexConstructCohesiveCells(dm, hlabel, slabel, dmHybrid);CHKERRQ(ierr);
2088   if (hybridLabel) *hybridLabel = hlabel;
2089   else             {ierr = DMLabelDestroy(&hlabel);CHKERRQ(ierr);}
2090   if (splitLabel)  *splitLabel  = slabel;
2091   PetscFunctionReturn(0);
2092 }
2093 
2094 /* Here we need the explicit assumption that:
2095 
2096      For any marked cell, the marked vertices constitute a single face
2097 */
2098 static PetscErrorCode DMPlexMarkSubmesh_Uninterpolated(DM dm, DMLabel vertexLabel, PetscInt value, DMLabel subpointMap, PetscInt *numFaces, PetscInt *nFV, DM subdm)
2099 {
2100   IS               subvertexIS = NULL;
2101   const PetscInt  *subvertices;
2102   PetscInt        *pStart, *pEnd, *pMax, pSize;
2103   PetscInt         depth, dim, d, numSubVerticesInitial = 0, v;
2104   PetscErrorCode   ierr;
2105 
2106   PetscFunctionBegin;
2107   *numFaces = 0;
2108   *nFV      = 0;
2109   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
2110   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
2111   pSize = PetscMax(depth, dim) + 1;
2112   ierr = PetscMalloc3(pSize,&pStart,pSize,&pEnd,pSize,&pMax);CHKERRQ(ierr);
2113   ierr = DMPlexGetHybridBounds(dm, depth >= 0 ? &pMax[depth] : NULL, depth>1 ? &pMax[depth-1] : NULL, depth>2 ? &pMax[1] : NULL, &pMax[0]);CHKERRQ(ierr);
2114   for (d = 0; d <= depth; ++d) {
2115     ierr = DMPlexGetDepthStratum(dm, d, &pStart[d], &pEnd[d]);CHKERRQ(ierr);
2116     if (pMax[d] >= 0) pEnd[d] = PetscMin(pEnd[d], pMax[d]);
2117   }
2118   /* Loop over initial vertices and mark all faces in the collective star() */
2119   if (vertexLabel) {ierr = DMLabelGetStratumIS(vertexLabel, value, &subvertexIS);CHKERRQ(ierr);}
2120   if (subvertexIS) {
2121     ierr = ISGetSize(subvertexIS, &numSubVerticesInitial);CHKERRQ(ierr);
2122     ierr = ISGetIndices(subvertexIS, &subvertices);CHKERRQ(ierr);
2123   }
2124   for (v = 0; v < numSubVerticesInitial; ++v) {
2125     const PetscInt vertex = subvertices[v];
2126     PetscInt      *star   = NULL;
2127     PetscInt       starSize, s, numCells = 0, c;
2128 
2129     ierr = DMPlexGetTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star);CHKERRQ(ierr);
2130     for (s = 0; s < starSize*2; s += 2) {
2131       const PetscInt point = star[s];
2132       if ((point >= pStart[depth]) && (point < pEnd[depth])) star[numCells++] = point;
2133     }
2134     for (c = 0; c < numCells; ++c) {
2135       const PetscInt cell    = star[c];
2136       PetscInt      *closure = NULL;
2137       PetscInt       closureSize, cl;
2138       PetscInt       cellLoc, numCorners = 0, faceSize = 0;
2139 
2140       ierr = DMLabelGetValue(subpointMap, cell, &cellLoc);CHKERRQ(ierr);
2141       if (cellLoc == 2) continue;
2142       if (cellLoc >= 0) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Cell %d has dimension %d in the surface label", cell, cellLoc);
2143       ierr = DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
2144       for (cl = 0; cl < closureSize*2; cl += 2) {
2145         const PetscInt point = closure[cl];
2146         PetscInt       vertexLoc;
2147 
2148         if ((point >= pStart[0]) && (point < pEnd[0])) {
2149           ++numCorners;
2150           ierr = DMLabelGetValue(vertexLabel, point, &vertexLoc);CHKERRQ(ierr);
2151           if (vertexLoc == value) closure[faceSize++] = point;
2152         }
2153       }
2154       if (!(*nFV)) {ierr = DMPlexGetNumFaceVertices(dm, dim, numCorners, nFV);CHKERRQ(ierr);}
2155       if (faceSize > *nFV) SETERRQ1(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Invalid submesh: Too many vertices %d of an element on the surface", faceSize);
2156       if (faceSize == *nFV) {
2157         const PetscInt *cells = NULL;
2158         PetscInt        numCells, nc;
2159 
2160         ++(*numFaces);
2161         for (cl = 0; cl < faceSize; ++cl) {
2162           ierr = DMLabelSetValue(subpointMap, closure[cl], 0);CHKERRQ(ierr);
2163         }
2164         ierr = DMPlexGetJoin(dm, faceSize, closure, &numCells, &cells);CHKERRQ(ierr);
2165         for (nc = 0; nc < numCells; ++nc) {
2166           ierr = DMLabelSetValue(subpointMap, cells[nc], 2);CHKERRQ(ierr);
2167         }
2168         ierr = DMPlexRestoreJoin(dm, faceSize, closure, &numCells, &cells);CHKERRQ(ierr);
2169       }
2170       ierr = DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
2171     }
2172     ierr = DMPlexRestoreTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star);CHKERRQ(ierr);
2173   }
2174   if (subvertexIS) {
2175     ierr = ISRestoreIndices(subvertexIS, &subvertices);CHKERRQ(ierr);
2176   }
2177   ierr = ISDestroy(&subvertexIS);CHKERRQ(ierr);
2178   ierr = PetscFree3(pStart,pEnd,pMax);CHKERRQ(ierr);
2179   PetscFunctionReturn(0);
2180 }
2181 
2182 static PetscErrorCode DMPlexMarkSubmesh_Interpolated(DM dm, DMLabel vertexLabel, PetscInt value, PetscBool markedFaces, DMLabel subpointMap, DM subdm)
2183 {
2184   IS               subvertexIS = NULL;
2185   const PetscInt  *subvertices;
2186   PetscInt        *pStart, *pEnd, *pMax;
2187   PetscInt         dim, d, numSubVerticesInitial = 0, v;
2188   PetscErrorCode   ierr;
2189 
2190   PetscFunctionBegin;
2191   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
2192   ierr = PetscMalloc3(dim+1,&pStart,dim+1,&pEnd,dim+1,&pMax);CHKERRQ(ierr);
2193   ierr = DMPlexGetHybridBounds(dm, &pMax[dim], dim>1 ? &pMax[dim-1] : NULL, dim > 2 ? &pMax[1] : NULL, &pMax[0]);CHKERRQ(ierr);
2194   for (d = 0; d <= dim; ++d) {
2195     ierr = DMPlexGetDepthStratum(dm, d, &pStart[d], &pEnd[d]);CHKERRQ(ierr);
2196     if (pMax[d] >= 0) pEnd[d] = PetscMin(pEnd[d], pMax[d]);
2197   }
2198   /* Loop over initial vertices and mark all faces in the collective star() */
2199   if (vertexLabel) {
2200     ierr = DMLabelGetStratumIS(vertexLabel, value, &subvertexIS);CHKERRQ(ierr);
2201     if (subvertexIS) {
2202       ierr = ISGetSize(subvertexIS, &numSubVerticesInitial);CHKERRQ(ierr);
2203       ierr = ISGetIndices(subvertexIS, &subvertices);CHKERRQ(ierr);
2204     }
2205   }
2206   for (v = 0; v < numSubVerticesInitial; ++v) {
2207     const PetscInt vertex = subvertices[v];
2208     PetscInt      *star   = NULL;
2209     PetscInt       starSize, s, numFaces = 0, f;
2210 
2211     ierr = DMPlexGetTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star);CHKERRQ(ierr);
2212     for (s = 0; s < starSize*2; s += 2) {
2213       const PetscInt point = star[s];
2214       PetscInt       faceLoc;
2215 
2216       if ((point >= pStart[dim-1]) && (point < pEnd[dim-1])) {
2217         if (markedFaces) {
2218           ierr = DMLabelGetValue(vertexLabel, point, &faceLoc);CHKERRQ(ierr);
2219           if (faceLoc < 0) continue;
2220         }
2221         star[numFaces++] = point;
2222       }
2223     }
2224     for (f = 0; f < numFaces; ++f) {
2225       const PetscInt face    = star[f];
2226       PetscInt      *closure = NULL;
2227       PetscInt       closureSize, c;
2228       PetscInt       faceLoc;
2229 
2230       ierr = DMLabelGetValue(subpointMap, face, &faceLoc);CHKERRQ(ierr);
2231       if (faceLoc == dim-1) continue;
2232       if (faceLoc >= 0) SETERRQ2(PetscObjectComm((PetscObject)dm), PETSC_ERR_PLIB, "Face %d has dimension %d in the surface label", face, faceLoc);
2233       ierr = DMPlexGetTransitiveClosure(dm, face, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
2234       for (c = 0; c < closureSize*2; c += 2) {
2235         const PetscInt point = closure[c];
2236         PetscInt       vertexLoc;
2237 
2238         if ((point >= pStart[0]) && (point < pEnd[0])) {
2239           ierr = DMLabelGetValue(vertexLabel, point, &vertexLoc);CHKERRQ(ierr);
2240           if (vertexLoc != value) break;
2241         }
2242       }
2243       if (c == closureSize*2) {
2244         const PetscInt *support;
2245         PetscInt        supportSize, s;
2246 
2247         for (c = 0; c < closureSize*2; c += 2) {
2248           const PetscInt point = closure[c];
2249 
2250           for (d = 0; d < dim; ++d) {
2251             if ((point >= pStart[d]) && (point < pEnd[d])) {
2252               ierr = DMLabelSetValue(subpointMap, point, d);CHKERRQ(ierr);
2253               break;
2254             }
2255           }
2256         }
2257         ierr = DMPlexGetSupportSize(dm, face, &supportSize);CHKERRQ(ierr);
2258         ierr = DMPlexGetSupport(dm, face, &support);CHKERRQ(ierr);
2259         for (s = 0; s < supportSize; ++s) {
2260           ierr = DMLabelSetValue(subpointMap, support[s], dim);CHKERRQ(ierr);
2261         }
2262       }
2263       ierr = DMPlexRestoreTransitiveClosure(dm, face, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
2264     }
2265     ierr = DMPlexRestoreTransitiveClosure(dm, vertex, PETSC_FALSE, &starSize, &star);CHKERRQ(ierr);
2266   }
2267   if (subvertexIS) {ierr = ISRestoreIndices(subvertexIS, &subvertices);CHKERRQ(ierr);}
2268   ierr = ISDestroy(&subvertexIS);CHKERRQ(ierr);
2269   ierr = PetscFree3(pStart,pEnd,pMax);CHKERRQ(ierr);
2270   PetscFunctionReturn(0);
2271 }
2272 
2273 static PetscErrorCode DMPlexMarkCohesiveSubmesh_Uninterpolated(DM dm, PetscBool hasLagrange, const char labelname[], PetscInt value, DMLabel subpointMap, PetscInt *numFaces, PetscInt *nFV, PetscInt *subCells[], DM subdm)
2274 {
2275   DMLabel         label = NULL;
2276   const PetscInt *cone;
2277   PetscInt        dim, cMax, cEnd, c, subc = 0, p, coneSize = -1;
2278   PetscErrorCode  ierr;
2279 
2280   PetscFunctionBegin;
2281   *numFaces = 0;
2282   *nFV = 0;
2283   if (labelname) {ierr = DMGetLabel(dm, labelname, &label);CHKERRQ(ierr);}
2284   *subCells = NULL;
2285   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
2286   ierr = DMPlexGetHeightStratum(dm, 0, NULL, &cEnd);CHKERRQ(ierr);
2287   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
2288   if (cMax < 0) PetscFunctionReturn(0);
2289   if (label) {
2290     for (c = cMax; c < cEnd; ++c) {
2291       PetscInt val;
2292 
2293       ierr = DMLabelGetValue(label, c, &val);CHKERRQ(ierr);
2294       if (val == value) {
2295         ++(*numFaces);
2296         ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
2297       }
2298     }
2299   } else {
2300     *numFaces = cEnd - cMax;
2301     ierr = DMPlexGetConeSize(dm, cMax, &coneSize);CHKERRQ(ierr);
2302   }
2303   ierr = PetscMalloc1(*numFaces *2, subCells);CHKERRQ(ierr);
2304   if (!(*numFaces)) PetscFunctionReturn(0);
2305   *nFV = hasLagrange ? coneSize/3 : coneSize/2;
2306   for (c = cMax; c < cEnd; ++c) {
2307     const PetscInt *cells;
2308     PetscInt        numCells;
2309 
2310     if (label) {
2311       PetscInt val;
2312 
2313       ierr = DMLabelGetValue(label, c, &val);CHKERRQ(ierr);
2314       if (val != value) continue;
2315     }
2316     ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
2317     for (p = 0; p < *nFV; ++p) {
2318       ierr = DMLabelSetValue(subpointMap, cone[p], 0);CHKERRQ(ierr);
2319     }
2320     /* Negative face */
2321     ierr = DMPlexGetJoin(dm, *nFV, cone, &numCells, &cells);CHKERRQ(ierr);
2322     /* Not true in parallel
2323     if (numCells != 2) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive cells should separate two cells"); */
2324     for (p = 0; p < numCells; ++p) {
2325       ierr = DMLabelSetValue(subpointMap, cells[p], 2);CHKERRQ(ierr);
2326       (*subCells)[subc++] = cells[p];
2327     }
2328     ierr = DMPlexRestoreJoin(dm, *nFV, cone, &numCells, &cells);CHKERRQ(ierr);
2329     /* Positive face is not included */
2330   }
2331   PetscFunctionReturn(0);
2332 }
2333 
2334 static PetscErrorCode DMPlexMarkCohesiveSubmesh_Interpolated(DM dm, DMLabel label, PetscInt value, DMLabel subpointMap, DM subdm)
2335 {
2336   PetscInt      *pStart, *pEnd;
2337   PetscInt       dim, cMax, cEnd, c, d;
2338   PetscErrorCode ierr;
2339 
2340   PetscFunctionBegin;
2341   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
2342   ierr = DMPlexGetHeightStratum(dm, 0, NULL, &cEnd);CHKERRQ(ierr);
2343   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
2344   if (cMax < 0) PetscFunctionReturn(0);
2345   ierr = PetscMalloc2(dim+1,&pStart,dim+1,&pEnd);CHKERRQ(ierr);
2346   for (d = 0; d <= dim; ++d) {ierr = DMPlexGetDepthStratum(dm, d, &pStart[d], &pEnd[d]);CHKERRQ(ierr);}
2347   for (c = cMax; c < cEnd; ++c) {
2348     const PetscInt *cone;
2349     PetscInt       *closure = NULL;
2350     PetscInt        fconeSize, coneSize, closureSize, cl, val;
2351 
2352     if (label) {
2353       ierr = DMLabelGetValue(label, c, &val);CHKERRQ(ierr);
2354       if (val != value) continue;
2355     }
2356     ierr = DMPlexGetConeSize(dm, c, &coneSize);CHKERRQ(ierr);
2357     ierr = DMPlexGetCone(dm, c, &cone);CHKERRQ(ierr);
2358     ierr = DMPlexGetConeSize(dm, cone[0], &fconeSize);CHKERRQ(ierr);
2359     if (coneSize != (fconeSize ? fconeSize : 1) + 2) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive cells should separate two cells");
2360     /* Negative face */
2361     ierr = DMPlexGetTransitiveClosure(dm, cone[0], PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
2362     for (cl = 0; cl < closureSize*2; cl += 2) {
2363       const PetscInt point = closure[cl];
2364 
2365       for (d = 0; d <= dim; ++d) {
2366         if ((point >= pStart[d]) && (point < pEnd[d])) {
2367           ierr = DMLabelSetValue(subpointMap, point, d);CHKERRQ(ierr);
2368           break;
2369         }
2370       }
2371     }
2372     ierr = DMPlexRestoreTransitiveClosure(dm, cone[0], PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
2373     /* Cells -- positive face is not included */
2374     for (cl = 0; cl < 1; ++cl) {
2375       const PetscInt *support;
2376       PetscInt        supportSize, s;
2377 
2378       ierr = DMPlexGetSupportSize(dm, cone[cl], &supportSize);CHKERRQ(ierr);
2379       /* if (supportSize != 2) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive faces should separate two cells"); */
2380       ierr = DMPlexGetSupport(dm, cone[cl], &support);CHKERRQ(ierr);
2381       for (s = 0; s < supportSize; ++s) {
2382         ierr = DMLabelSetValue(subpointMap, support[s], dim);CHKERRQ(ierr);
2383       }
2384     }
2385   }
2386   ierr = PetscFree2(pStart, pEnd);CHKERRQ(ierr);
2387   PetscFunctionReturn(0);
2388 }
2389 
2390 static PetscErrorCode DMPlexGetFaceOrientation(DM dm, PetscInt cell, PetscInt numCorners, PetscInt indices[], PetscInt oppositeVertex, PetscInt origVertices[], PetscInt faceVertices[], PetscBool *posOriented)
2391 {
2392   MPI_Comm       comm;
2393   PetscBool      posOrient = PETSC_FALSE;
2394   const PetscInt debug     = 0;
2395   PetscInt       cellDim, faceSize, f;
2396   PetscErrorCode ierr;
2397 
2398   PetscFunctionBegin;
2399   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
2400   ierr = DMGetDimension(dm, &cellDim);CHKERRQ(ierr);
2401   if (debug) {ierr = PetscPrintf(comm, "cellDim: %d numCorners: %d\n", cellDim, numCorners);CHKERRQ(ierr);}
2402 
2403   if (cellDim == 1 && numCorners == 2) {
2404     /* Triangle */
2405     faceSize  = numCorners-1;
2406     posOrient = !(oppositeVertex%2) ? PETSC_TRUE : PETSC_FALSE;
2407   } else if (cellDim == 2 && numCorners == 3) {
2408     /* Triangle */
2409     faceSize  = numCorners-1;
2410     posOrient = !(oppositeVertex%2) ? PETSC_TRUE : PETSC_FALSE;
2411   } else if (cellDim == 3 && numCorners == 4) {
2412     /* Tetrahedron */
2413     faceSize  = numCorners-1;
2414     posOrient = (oppositeVertex%2) ? PETSC_TRUE : PETSC_FALSE;
2415   } else if (cellDim == 1 && numCorners == 3) {
2416     /* Quadratic line */
2417     faceSize  = 1;
2418     posOrient = PETSC_TRUE;
2419   } else if (cellDim == 2 && numCorners == 4) {
2420     /* Quads */
2421     faceSize = 2;
2422     if ((indices[1] > indices[0]) && (indices[1] - indices[0] == 1)) {
2423       posOrient = PETSC_TRUE;
2424     } else if ((indices[0] == 3) && (indices[1] == 0)) {
2425       posOrient = PETSC_TRUE;
2426     } else {
2427       if (((indices[0] > indices[1]) && (indices[0] - indices[1] == 1)) || ((indices[0] == 0) && (indices[1] == 3))) {
2428         posOrient = PETSC_FALSE;
2429       } else SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid quad crossedge");
2430     }
2431   } else if (cellDim == 2 && numCorners == 6) {
2432     /* Quadratic triangle (I hate this) */
2433     /* Edges are determined by the first 2 vertices (corners of edges) */
2434     const PetscInt faceSizeTri = 3;
2435     PetscInt       sortedIndices[3], i, iFace;
2436     PetscBool      found                    = PETSC_FALSE;
2437     PetscInt       faceVerticesTriSorted[9] = {
2438       0, 3,  4, /* bottom */
2439       1, 4,  5, /* right */
2440       2, 3,  5, /* left */
2441     };
2442     PetscInt       faceVerticesTri[9] = {
2443       0, 3,  4, /* bottom */
2444       1, 4,  5, /* right */
2445       2, 5,  3, /* left */
2446     };
2447 
2448     for (i = 0; i < faceSizeTri; ++i) sortedIndices[i] = indices[i];
2449     ierr = PetscSortInt(faceSizeTri, sortedIndices);CHKERRQ(ierr);
2450     for (iFace = 0; iFace < 3; ++iFace) {
2451       const PetscInt ii = iFace*faceSizeTri;
2452       PetscInt       fVertex, cVertex;
2453 
2454       if ((sortedIndices[0] == faceVerticesTriSorted[ii+0]) &&
2455           (sortedIndices[1] == faceVerticesTriSorted[ii+1])) {
2456         for (fVertex = 0; fVertex < faceSizeTri; ++fVertex) {
2457           for (cVertex = 0; cVertex < faceSizeTri; ++cVertex) {
2458             if (indices[cVertex] == faceVerticesTri[ii+fVertex]) {
2459               faceVertices[fVertex] = origVertices[cVertex];
2460               break;
2461             }
2462           }
2463         }
2464         found = PETSC_TRUE;
2465         break;
2466       }
2467     }
2468     if (!found) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid tri crossface");
2469     if (posOriented) *posOriented = PETSC_TRUE;
2470     PetscFunctionReturn(0);
2471   } else if (cellDim == 2 && numCorners == 9) {
2472     /* Quadratic quad (I hate this) */
2473     /* Edges are determined by the first 2 vertices (corners of edges) */
2474     const PetscInt faceSizeQuad = 3;
2475     PetscInt       sortedIndices[3], i, iFace;
2476     PetscBool      found                      = PETSC_FALSE;
2477     PetscInt       faceVerticesQuadSorted[12] = {
2478       0, 1,  4, /* bottom */
2479       1, 2,  5, /* right */
2480       2, 3,  6, /* top */
2481       0, 3,  7, /* left */
2482     };
2483     PetscInt       faceVerticesQuad[12] = {
2484       0, 1,  4, /* bottom */
2485       1, 2,  5, /* right */
2486       2, 3,  6, /* top */
2487       3, 0,  7, /* left */
2488     };
2489 
2490     for (i = 0; i < faceSizeQuad; ++i) sortedIndices[i] = indices[i];
2491     ierr = PetscSortInt(faceSizeQuad, sortedIndices);CHKERRQ(ierr);
2492     for (iFace = 0; iFace < 4; ++iFace) {
2493       const PetscInt ii = iFace*faceSizeQuad;
2494       PetscInt       fVertex, cVertex;
2495 
2496       if ((sortedIndices[0] == faceVerticesQuadSorted[ii+0]) &&
2497           (sortedIndices[1] == faceVerticesQuadSorted[ii+1])) {
2498         for (fVertex = 0; fVertex < faceSizeQuad; ++fVertex) {
2499           for (cVertex = 0; cVertex < faceSizeQuad; ++cVertex) {
2500             if (indices[cVertex] == faceVerticesQuad[ii+fVertex]) {
2501               faceVertices[fVertex] = origVertices[cVertex];
2502               break;
2503             }
2504           }
2505         }
2506         found = PETSC_TRUE;
2507         break;
2508       }
2509     }
2510     if (!found) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid quad crossface");
2511     if (posOriented) *posOriented = PETSC_TRUE;
2512     PetscFunctionReturn(0);
2513   } else if (cellDim == 3 && numCorners == 8) {
2514     /* Hexes
2515        A hex is two oriented quads with the normal of the first
2516        pointing up at the second.
2517 
2518           7---6
2519          /|  /|
2520         4---5 |
2521         | 1-|-2
2522         |/  |/
2523         0---3
2524 
2525         Faces are determined by the first 4 vertices (corners of faces) */
2526     const PetscInt faceSizeHex = 4;
2527     PetscInt       sortedIndices[4], i, iFace;
2528     PetscBool      found                     = PETSC_FALSE;
2529     PetscInt       faceVerticesHexSorted[24] = {
2530       0, 1, 2, 3,  /* bottom */
2531       4, 5, 6, 7,  /* top */
2532       0, 3, 4, 5,  /* front */
2533       2, 3, 5, 6,  /* right */
2534       1, 2, 6, 7,  /* back */
2535       0, 1, 4, 7,  /* left */
2536     };
2537     PetscInt       faceVerticesHex[24] = {
2538       1, 2, 3, 0,  /* bottom */
2539       4, 5, 6, 7,  /* top */
2540       0, 3, 5, 4,  /* front */
2541       3, 2, 6, 5,  /* right */
2542       2, 1, 7, 6,  /* back */
2543       1, 0, 4, 7,  /* left */
2544     };
2545 
2546     for (i = 0; i < faceSizeHex; ++i) sortedIndices[i] = indices[i];
2547     ierr = PetscSortInt(faceSizeHex, sortedIndices);CHKERRQ(ierr);
2548     for (iFace = 0; iFace < 6; ++iFace) {
2549       const PetscInt ii = iFace*faceSizeHex;
2550       PetscInt       fVertex, cVertex;
2551 
2552       if ((sortedIndices[0] == faceVerticesHexSorted[ii+0]) &&
2553           (sortedIndices[1] == faceVerticesHexSorted[ii+1]) &&
2554           (sortedIndices[2] == faceVerticesHexSorted[ii+2]) &&
2555           (sortedIndices[3] == faceVerticesHexSorted[ii+3])) {
2556         for (fVertex = 0; fVertex < faceSizeHex; ++fVertex) {
2557           for (cVertex = 0; cVertex < faceSizeHex; ++cVertex) {
2558             if (indices[cVertex] == faceVerticesHex[ii+fVertex]) {
2559               faceVertices[fVertex] = origVertices[cVertex];
2560               break;
2561             }
2562           }
2563         }
2564         found = PETSC_TRUE;
2565         break;
2566       }
2567     }
2568     if (!found) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid hex crossface");
2569     if (posOriented) *posOriented = PETSC_TRUE;
2570     PetscFunctionReturn(0);
2571   } else if (cellDim == 3 && numCorners == 10) {
2572     /* Quadratic tet */
2573     /* Faces are determined by the first 3 vertices (corners of faces) */
2574     const PetscInt faceSizeTet = 6;
2575     PetscInt       sortedIndices[6], i, iFace;
2576     PetscBool      found                     = PETSC_FALSE;
2577     PetscInt       faceVerticesTetSorted[24] = {
2578       0, 1, 2,  6, 7, 8, /* bottom */
2579       0, 3, 4,  6, 7, 9,  /* front */
2580       1, 4, 5,  7, 8, 9,  /* right */
2581       2, 3, 5,  6, 8, 9,  /* left */
2582     };
2583     PetscInt       faceVerticesTet[24] = {
2584       0, 1, 2,  6, 7, 8, /* bottom */
2585       0, 4, 3,  6, 7, 9,  /* front */
2586       1, 5, 4,  7, 8, 9,  /* right */
2587       2, 3, 5,  8, 6, 9,  /* left */
2588     };
2589 
2590     for (i = 0; i < faceSizeTet; ++i) sortedIndices[i] = indices[i];
2591     ierr = PetscSortInt(faceSizeTet, sortedIndices);CHKERRQ(ierr);
2592     for (iFace=0; iFace < 4; ++iFace) {
2593       const PetscInt ii = iFace*faceSizeTet;
2594       PetscInt       fVertex, cVertex;
2595 
2596       if ((sortedIndices[0] == faceVerticesTetSorted[ii+0]) &&
2597           (sortedIndices[1] == faceVerticesTetSorted[ii+1]) &&
2598           (sortedIndices[2] == faceVerticesTetSorted[ii+2]) &&
2599           (sortedIndices[3] == faceVerticesTetSorted[ii+3])) {
2600         for (fVertex = 0; fVertex < faceSizeTet; ++fVertex) {
2601           for (cVertex = 0; cVertex < faceSizeTet; ++cVertex) {
2602             if (indices[cVertex] == faceVerticesTet[ii+fVertex]) {
2603               faceVertices[fVertex] = origVertices[cVertex];
2604               break;
2605             }
2606           }
2607         }
2608         found = PETSC_TRUE;
2609         break;
2610       }
2611     }
2612     if (!found) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid tet crossface");
2613     if (posOriented) *posOriented = PETSC_TRUE;
2614     PetscFunctionReturn(0);
2615   } else if (cellDim == 3 && numCorners == 27) {
2616     /* Quadratic hexes (I hate this)
2617        A hex is two oriented quads with the normal of the first
2618        pointing up at the second.
2619 
2620          7---6
2621         /|  /|
2622        4---5 |
2623        | 3-|-2
2624        |/  |/
2625        0---1
2626 
2627        Faces are determined by the first 4 vertices (corners of faces) */
2628     const PetscInt faceSizeQuadHex = 9;
2629     PetscInt       sortedIndices[9], i, iFace;
2630     PetscBool      found                         = PETSC_FALSE;
2631     PetscInt       faceVerticesQuadHexSorted[54] = {
2632       0, 1, 2, 3,  8, 9, 10, 11,  24, /* bottom */
2633       4, 5, 6, 7,  12, 13, 14, 15,  25, /* top */
2634       0, 1, 4, 5,  8, 12, 16, 17,  22, /* front */
2635       1, 2, 5, 6,  9, 13, 17, 18,  21, /* right */
2636       2, 3, 6, 7,  10, 14, 18, 19,  23, /* back */
2637       0, 3, 4, 7,  11, 15, 16, 19,  20, /* left */
2638     };
2639     PetscInt       faceVerticesQuadHex[54] = {
2640       3, 2, 1, 0,  10, 9, 8, 11,  24, /* bottom */
2641       4, 5, 6, 7,  12, 13, 14, 15,  25, /* top */
2642       0, 1, 5, 4,  8, 17, 12, 16,  22, /* front */
2643       1, 2, 6, 5,  9, 18, 13, 17,  21, /* right */
2644       2, 3, 7, 6,  10, 19, 14, 18,  23, /* back */
2645       3, 0, 4, 7,  11, 16, 15, 19,  20 /* left */
2646     };
2647 
2648     for (i = 0; i < faceSizeQuadHex; ++i) sortedIndices[i] = indices[i];
2649     ierr = PetscSortInt(faceSizeQuadHex, sortedIndices);CHKERRQ(ierr);
2650     for (iFace = 0; iFace < 6; ++iFace) {
2651       const PetscInt ii = iFace*faceSizeQuadHex;
2652       PetscInt       fVertex, cVertex;
2653 
2654       if ((sortedIndices[0] == faceVerticesQuadHexSorted[ii+0]) &&
2655           (sortedIndices[1] == faceVerticesQuadHexSorted[ii+1]) &&
2656           (sortedIndices[2] == faceVerticesQuadHexSorted[ii+2]) &&
2657           (sortedIndices[3] == faceVerticesQuadHexSorted[ii+3])) {
2658         for (fVertex = 0; fVertex < faceSizeQuadHex; ++fVertex) {
2659           for (cVertex = 0; cVertex < faceSizeQuadHex; ++cVertex) {
2660             if (indices[cVertex] == faceVerticesQuadHex[ii+fVertex]) {
2661               faceVertices[fVertex] = origVertices[cVertex];
2662               break;
2663             }
2664           }
2665         }
2666         found = PETSC_TRUE;
2667         break;
2668       }
2669     }
2670     if (!found) SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Invalid hex crossface");
2671     if (posOriented) *posOriented = PETSC_TRUE;
2672     PetscFunctionReturn(0);
2673   } else SETERRQ(comm, PETSC_ERR_ARG_WRONG, "Unknown cell type for faceOrientation().");
2674   if (!posOrient) {
2675     if (debug) {ierr = PetscPrintf(comm, "  Reversing initial face orientation\n");CHKERRQ(ierr);}
2676     for (f = 0; f < faceSize; ++f) faceVertices[f] = origVertices[faceSize-1 - f];
2677   } else {
2678     if (debug) {ierr = PetscPrintf(comm, "  Keeping initial face orientation\n");CHKERRQ(ierr);}
2679     for (f = 0; f < faceSize; ++f) faceVertices[f] = origVertices[f];
2680   }
2681   if (posOriented) *posOriented = posOrient;
2682   PetscFunctionReturn(0);
2683 }
2684 
2685 /*@
2686   DMPlexGetOrientedFace - Given a cell and a face, as a set of vertices, return the oriented face, as a set of vertices,
2687   in faceVertices. The orientation is such that the face normal points out of the cell
2688 
2689   Not collective
2690 
2691   Input Parameters:
2692 + dm           - The original mesh
2693 . cell         - The cell mesh point
2694 . faceSize     - The number of vertices on the face
2695 . face         - The face vertices
2696 . numCorners   - The number of vertices on the cell
2697 . indices      - Local numbering of face vertices in cell cone
2698 - origVertices - Original face vertices
2699 
2700   Output Parameter:
2701 + faceVertices - The face vertices properly oriented
2702 - posOriented  - PETSC_TRUE if the face was oriented with outward normal
2703 
2704   Level: developer
2705 
2706 .seealso: DMPlexGetCone()
2707 @*/
2708 PetscErrorCode DMPlexGetOrientedFace(DM dm, PetscInt cell, PetscInt faceSize, const PetscInt face[], PetscInt numCorners, PetscInt indices[], PetscInt origVertices[], PetscInt faceVertices[], PetscBool *posOriented)
2709 {
2710   const PetscInt *cone = NULL;
2711   PetscInt        coneSize, v, f, v2;
2712   PetscInt        oppositeVertex = -1;
2713   PetscErrorCode  ierr;
2714 
2715   PetscFunctionBegin;
2716   ierr = DMPlexGetConeSize(dm, cell, &coneSize);CHKERRQ(ierr);
2717   ierr = DMPlexGetCone(dm, cell, &cone);CHKERRQ(ierr);
2718   for (v = 0, v2 = 0; v < coneSize; ++v) {
2719     PetscBool found = PETSC_FALSE;
2720 
2721     for (f = 0; f < faceSize; ++f) {
2722       if (face[f] == cone[v]) {
2723         found = PETSC_TRUE; break;
2724       }
2725     }
2726     if (found) {
2727       indices[v2]      = v;
2728       origVertices[v2] = cone[v];
2729       ++v2;
2730     } else {
2731       oppositeVertex = v;
2732     }
2733   }
2734   ierr = DMPlexGetFaceOrientation(dm, cell, numCorners, indices, oppositeVertex, origVertices, faceVertices, posOriented);CHKERRQ(ierr);
2735   PetscFunctionReturn(0);
2736 }
2737 
2738 /*
2739   DMPlexInsertFace_Internal - Puts a face into the mesh
2740 
2741   Not collective
2742 
2743   Input Parameters:
2744   + dm              - The DMPlex
2745   . numFaceVertex   - The number of vertices in the face
2746   . faceVertices    - The vertices in the face for dm
2747   . subfaceVertices - The vertices in the face for subdm
2748   . numCorners      - The number of vertices in the cell
2749   . cell            - A cell in dm containing the face
2750   . subcell         - A cell in subdm containing the face
2751   . firstFace       - First face in the mesh
2752   - newFacePoint    - Next face in the mesh
2753 
2754   Output Parameters:
2755   . newFacePoint - Contains next face point number on input, updated on output
2756 
2757   Level: developer
2758 */
2759 static PetscErrorCode DMPlexInsertFace_Internal(DM dm, DM subdm, PetscInt numFaceVertices, const PetscInt faceVertices[], const PetscInt subfaceVertices[], PetscInt numCorners, PetscInt cell, PetscInt subcell, PetscInt firstFace, PetscInt *newFacePoint)
2760 {
2761   MPI_Comm        comm;
2762   DM_Plex        *submesh = (DM_Plex*) subdm->data;
2763   const PetscInt *faces;
2764   PetscInt        numFaces, coneSize;
2765   PetscErrorCode  ierr;
2766 
2767   PetscFunctionBegin;
2768   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
2769   ierr = DMPlexGetConeSize(subdm, subcell, &coneSize);CHKERRQ(ierr);
2770   if (coneSize != 1) SETERRQ2(comm, PETSC_ERR_ARG_OUTOFRANGE, "Cone size of cell %d is %d != 1", cell, coneSize);
2771 #if 0
2772   /* Cannot use this because support() has not been constructed yet */
2773   ierr = DMPlexGetJoin(subdm, numFaceVertices, subfaceVertices, &numFaces, &faces);CHKERRQ(ierr);
2774 #else
2775   {
2776     PetscInt f;
2777 
2778     numFaces = 0;
2779     ierr     = DMGetWorkArray(subdm, 1, MPIU_INT, (void **) &faces);CHKERRQ(ierr);
2780     for (f = firstFace; f < *newFacePoint; ++f) {
2781       PetscInt dof, off, d;
2782 
2783       ierr = PetscSectionGetDof(submesh->coneSection, f, &dof);CHKERRQ(ierr);
2784       ierr = PetscSectionGetOffset(submesh->coneSection, f, &off);CHKERRQ(ierr);
2785       /* Yes, I know this is quadratic, but I expect the sizes to be <5 */
2786       for (d = 0; d < dof; ++d) {
2787         const PetscInt p = submesh->cones[off+d];
2788         PetscInt       v;
2789 
2790         for (v = 0; v < numFaceVertices; ++v) {
2791           if (subfaceVertices[v] == p) break;
2792         }
2793         if (v == numFaceVertices) break;
2794       }
2795       if (d == dof) {
2796         numFaces               = 1;
2797         ((PetscInt*) faces)[0] = f;
2798       }
2799     }
2800   }
2801 #endif
2802   if (numFaces > 1) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Vertex set had %d faces, not one", numFaces);
2803   else if (numFaces == 1) {
2804     /* Add the other cell neighbor for this face */
2805     ierr = DMPlexSetCone(subdm, subcell, faces);CHKERRQ(ierr);
2806   } else {
2807     PetscInt *indices, *origVertices, *orientedVertices, *orientedSubVertices, v, ov;
2808     PetscBool posOriented;
2809 
2810     ierr                = DMGetWorkArray(subdm, 4*numFaceVertices * sizeof(PetscInt), MPIU_INT, &orientedVertices);CHKERRQ(ierr);
2811     origVertices        = &orientedVertices[numFaceVertices];
2812     indices             = &orientedVertices[numFaceVertices*2];
2813     orientedSubVertices = &orientedVertices[numFaceVertices*3];
2814     ierr                = DMPlexGetOrientedFace(dm, cell, numFaceVertices, faceVertices, numCorners, indices, origVertices, orientedVertices, &posOriented);CHKERRQ(ierr);
2815     /* TODO: I know that routine should return a permutation, not the indices */
2816     for (v = 0; v < numFaceVertices; ++v) {
2817       const PetscInt vertex = faceVertices[v], subvertex = subfaceVertices[v];
2818       for (ov = 0; ov < numFaceVertices; ++ov) {
2819         if (orientedVertices[ov] == vertex) {
2820           orientedSubVertices[ov] = subvertex;
2821           break;
2822         }
2823       }
2824       if (ov == numFaceVertices) SETERRQ1(comm, PETSC_ERR_PLIB, "Could not find face vertex %d in orientated set", vertex);
2825     }
2826     ierr = DMPlexSetCone(subdm, *newFacePoint, orientedSubVertices);CHKERRQ(ierr);
2827     ierr = DMPlexSetCone(subdm, subcell, newFacePoint);CHKERRQ(ierr);
2828     ierr = DMRestoreWorkArray(subdm, 4*numFaceVertices * sizeof(PetscInt), MPIU_INT, &orientedVertices);CHKERRQ(ierr);
2829     ++(*newFacePoint);
2830   }
2831 #if 0
2832   ierr = DMPlexRestoreJoin(subdm, numFaceVertices, subfaceVertices, &numFaces, &faces);CHKERRQ(ierr);
2833 #else
2834   ierr = DMRestoreWorkArray(subdm, 1, MPIU_INT, (void **) &faces);CHKERRQ(ierr);
2835 #endif
2836   PetscFunctionReturn(0);
2837 }
2838 
2839 static PetscErrorCode DMPlexCreateSubmesh_Uninterpolated(DM dm, DMLabel vertexLabel, PetscInt value, DM subdm)
2840 {
2841   MPI_Comm        comm;
2842   DMLabel         subpointMap;
2843   IS              subvertexIS,  subcellIS;
2844   const PetscInt *subVertices, *subCells;
2845   PetscInt        numSubVertices, firstSubVertex, numSubCells;
2846   PetscInt       *subface, maxConeSize, numSubFaces = 0, firstSubFace, newFacePoint, nFV = 0;
2847   PetscInt        vStart, vEnd, c, f;
2848   PetscErrorCode  ierr;
2849 
2850   PetscFunctionBegin;
2851   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
2852   /* Create subpointMap which marks the submesh */
2853   ierr = DMLabelCreate(PETSC_COMM_SELF, "subpoint_map", &subpointMap);CHKERRQ(ierr);
2854   ierr = DMPlexSetSubpointMap(subdm, subpointMap);CHKERRQ(ierr);
2855   ierr = DMLabelDestroy(&subpointMap);CHKERRQ(ierr);
2856   if (vertexLabel) {ierr = DMPlexMarkSubmesh_Uninterpolated(dm, vertexLabel, value, subpointMap, &numSubFaces, &nFV, subdm);CHKERRQ(ierr);}
2857   /* Setup chart */
2858   ierr = DMLabelGetStratumSize(subpointMap, 0, &numSubVertices);CHKERRQ(ierr);
2859   ierr = DMLabelGetStratumSize(subpointMap, 2, &numSubCells);CHKERRQ(ierr);
2860   ierr = DMPlexSetChart(subdm, 0, numSubCells+numSubFaces+numSubVertices);CHKERRQ(ierr);
2861   ierr = DMPlexSetVTKCellHeight(subdm, 1);CHKERRQ(ierr);
2862   /* Set cone sizes */
2863   firstSubVertex = numSubCells;
2864   firstSubFace   = numSubCells+numSubVertices;
2865   newFacePoint   = firstSubFace;
2866   ierr = DMLabelGetStratumIS(subpointMap, 0, &subvertexIS);CHKERRQ(ierr);
2867   if (subvertexIS) {ierr = ISGetIndices(subvertexIS, &subVertices);CHKERRQ(ierr);}
2868   ierr = DMLabelGetStratumIS(subpointMap, 2, &subcellIS);CHKERRQ(ierr);
2869   if (subcellIS) {ierr = ISGetIndices(subcellIS, &subCells);CHKERRQ(ierr);}
2870   for (c = 0; c < numSubCells; ++c) {
2871     ierr = DMPlexSetConeSize(subdm, c, 1);CHKERRQ(ierr);
2872   }
2873   for (f = firstSubFace; f < firstSubFace+numSubFaces; ++f) {
2874     ierr = DMPlexSetConeSize(subdm, f, nFV);CHKERRQ(ierr);
2875   }
2876   ierr = DMSetUp(subdm);CHKERRQ(ierr);
2877   /* Create face cones */
2878   ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
2879   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, NULL);CHKERRQ(ierr);
2880   ierr = DMGetWorkArray(subdm, maxConeSize, MPIU_INT, (void**) &subface);CHKERRQ(ierr);
2881   for (c = 0; c < numSubCells; ++c) {
2882     const PetscInt cell    = subCells[c];
2883     const PetscInt subcell = c;
2884     PetscInt      *closure = NULL;
2885     PetscInt       closureSize, cl, numCorners = 0, faceSize = 0;
2886 
2887     ierr = DMPlexGetTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
2888     for (cl = 0; cl < closureSize*2; cl += 2) {
2889       const PetscInt point = closure[cl];
2890       PetscInt       subVertex;
2891 
2892       if ((point >= vStart) && (point < vEnd)) {
2893         ++numCorners;
2894         ierr = PetscFindInt(point, numSubVertices, subVertices, &subVertex);CHKERRQ(ierr);
2895         if (subVertex >= 0) {
2896           closure[faceSize] = point;
2897           subface[faceSize] = firstSubVertex+subVertex;
2898           ++faceSize;
2899         }
2900       }
2901     }
2902     if (faceSize > nFV) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Invalid submesh: Too many vertices %d of an element on the surface", faceSize);
2903     if (faceSize == nFV) {
2904       ierr = DMPlexInsertFace_Internal(dm, subdm, faceSize, closure, subface, numCorners, cell, subcell, firstSubFace, &newFacePoint);CHKERRQ(ierr);
2905     }
2906     ierr = DMPlexRestoreTransitiveClosure(dm, cell, PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
2907   }
2908   ierr = DMRestoreWorkArray(subdm, maxConeSize, MPIU_INT, (void**) &subface);CHKERRQ(ierr);
2909   ierr = DMPlexSymmetrize(subdm);CHKERRQ(ierr);
2910   ierr = DMPlexStratify(subdm);CHKERRQ(ierr);
2911   /* Build coordinates */
2912   {
2913     PetscSection coordSection, subCoordSection;
2914     Vec          coordinates, subCoordinates;
2915     PetscScalar *coords, *subCoords;
2916     PetscInt     numComp, coordSize, v;
2917     const char  *name;
2918 
2919     ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
2920     ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
2921     ierr = DMGetCoordinateSection(subdm, &subCoordSection);CHKERRQ(ierr);
2922     ierr = PetscSectionSetNumFields(subCoordSection, 1);CHKERRQ(ierr);
2923     ierr = PetscSectionGetFieldComponents(coordSection, 0, &numComp);CHKERRQ(ierr);
2924     ierr = PetscSectionSetFieldComponents(subCoordSection, 0, numComp);CHKERRQ(ierr);
2925     ierr = PetscSectionSetChart(subCoordSection, firstSubVertex, firstSubVertex+numSubVertices);CHKERRQ(ierr);
2926     for (v = 0; v < numSubVertices; ++v) {
2927       const PetscInt vertex    = subVertices[v];
2928       const PetscInt subvertex = firstSubVertex+v;
2929       PetscInt       dof;
2930 
2931       ierr = PetscSectionGetDof(coordSection, vertex, &dof);CHKERRQ(ierr);
2932       ierr = PetscSectionSetDof(subCoordSection, subvertex, dof);CHKERRQ(ierr);
2933       ierr = PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof);CHKERRQ(ierr);
2934     }
2935     ierr = PetscSectionSetUp(subCoordSection);CHKERRQ(ierr);
2936     ierr = PetscSectionGetStorageSize(subCoordSection, &coordSize);CHKERRQ(ierr);
2937     ierr = VecCreate(PETSC_COMM_SELF, &subCoordinates);CHKERRQ(ierr);
2938     ierr = PetscObjectGetName((PetscObject)coordinates,&name);CHKERRQ(ierr);
2939     ierr = PetscObjectSetName((PetscObject)subCoordinates,name);CHKERRQ(ierr);
2940     ierr = VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
2941     ierr = VecSetType(subCoordinates,VECSTANDARD);CHKERRQ(ierr);
2942     if (coordSize) {
2943       ierr = VecGetArray(coordinates,    &coords);CHKERRQ(ierr);
2944       ierr = VecGetArray(subCoordinates, &subCoords);CHKERRQ(ierr);
2945       for (v = 0; v < numSubVertices; ++v) {
2946         const PetscInt vertex    = subVertices[v];
2947         const PetscInt subvertex = firstSubVertex+v;
2948         PetscInt       dof, off, sdof, soff, d;
2949 
2950         ierr = PetscSectionGetDof(coordSection, vertex, &dof);CHKERRQ(ierr);
2951         ierr = PetscSectionGetOffset(coordSection, vertex, &off);CHKERRQ(ierr);
2952         ierr = PetscSectionGetDof(subCoordSection, subvertex, &sdof);CHKERRQ(ierr);
2953         ierr = PetscSectionGetOffset(subCoordSection, subvertex, &soff);CHKERRQ(ierr);
2954         if (dof != sdof) SETERRQ4(comm, PETSC_ERR_PLIB, "Coordinate dimension %d on subvertex %d, vertex %d should be %d", sdof, subvertex, vertex, dof);
2955         for (d = 0; d < dof; ++d) subCoords[soff+d] = coords[off+d];
2956       }
2957       ierr = VecRestoreArray(coordinates,    &coords);CHKERRQ(ierr);
2958       ierr = VecRestoreArray(subCoordinates, &subCoords);CHKERRQ(ierr);
2959     }
2960     ierr = DMSetCoordinatesLocal(subdm, subCoordinates);CHKERRQ(ierr);
2961     ierr = VecDestroy(&subCoordinates);CHKERRQ(ierr);
2962   }
2963   /* Cleanup */
2964   if (subvertexIS) {ierr = ISRestoreIndices(subvertexIS, &subVertices);CHKERRQ(ierr);}
2965   ierr = ISDestroy(&subvertexIS);CHKERRQ(ierr);
2966   if (subcellIS) {ierr = ISRestoreIndices(subcellIS, &subCells);CHKERRQ(ierr);}
2967   ierr = ISDestroy(&subcellIS);CHKERRQ(ierr);
2968   PetscFunctionReturn(0);
2969 }
2970 
2971 PETSC_STATIC_INLINE PetscInt DMPlexFilterPoint_Internal(PetscInt point, PetscInt firstSubPoint, PetscInt numSubPoints, const PetscInt subPoints[])
2972 {
2973   PetscInt       subPoint;
2974   PetscErrorCode ierr;
2975 
2976   ierr = PetscFindInt(point, numSubPoints, subPoints, &subPoint); if (ierr < 0) return ierr;
2977   return subPoint < 0 ? subPoint : firstSubPoint+subPoint;
2978 }
2979 
2980 static PetscErrorCode DMPlexCreateSubmeshGeneric_Interpolated(DM dm, DMLabel label, PetscInt value, PetscBool markedFaces, PetscBool isCohesive, PetscInt cellHeight, DM subdm)
2981 {
2982   MPI_Comm         comm;
2983   DMLabel          subpointMap;
2984   IS              *subpointIS;
2985   const PetscInt **subpoints;
2986   PetscInt        *numSubPoints, *firstSubPoint, *coneNew, *orntNew;
2987   PetscInt         totSubPoints = 0, maxConeSize, cMax, cEnd, dim, p, d, v;
2988   PetscMPIInt      rank;
2989   PetscErrorCode   ierr;
2990 
2991   PetscFunctionBegin;
2992   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
2993   ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
2994   /* Create subpointMap which marks the submesh */
2995   ierr = DMLabelCreate(PETSC_COMM_SELF, "subpoint_map", &subpointMap);CHKERRQ(ierr);
2996   ierr = DMPlexSetSubpointMap(subdm, subpointMap);CHKERRQ(ierr);
2997   if (cellHeight) {
2998     if (isCohesive) {ierr = DMPlexMarkCohesiveSubmesh_Interpolated(dm, label, value, subpointMap, subdm);CHKERRQ(ierr);}
2999     else            {ierr = DMPlexMarkSubmesh_Interpolated(dm, label, value, markedFaces, subpointMap, subdm);CHKERRQ(ierr);}
3000   } else {
3001     DMLabel         depth;
3002     IS              pointIS;
3003     const PetscInt *points;
3004     PetscInt        numPoints;
3005 
3006     ierr = DMPlexGetDepthLabel(dm, &depth);CHKERRQ(ierr);
3007     ierr = DMLabelGetStratumSize(label, value, &numPoints);CHKERRQ(ierr);
3008     ierr = DMLabelGetStratumIS(label, value, &pointIS);CHKERRQ(ierr);
3009     ierr = ISGetIndices(pointIS, &points);CHKERRQ(ierr);
3010     for (p = 0; p < numPoints; ++p) {
3011       PetscInt *closure = NULL;
3012       PetscInt  closureSize, c, pdim;
3013 
3014       ierr = DMPlexGetTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
3015       for (c = 0; c < closureSize*2; c += 2) {
3016         ierr = DMLabelGetValue(depth, closure[c], &pdim);CHKERRQ(ierr);
3017         ierr = DMLabelSetValue(subpointMap, closure[c], pdim);CHKERRQ(ierr);
3018       }
3019       ierr = DMPlexRestoreTransitiveClosure(dm, points[p], PETSC_TRUE, &closureSize, &closure);CHKERRQ(ierr);
3020     }
3021     ierr = ISRestoreIndices(pointIS, &points);CHKERRQ(ierr);
3022     ierr = ISDestroy(&pointIS);CHKERRQ(ierr);
3023   }
3024   ierr = DMPlexGetHeightStratum(dm, 0, NULL, &cEnd);CHKERRQ(ierr);
3025   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
3026   cMax = (cMax < 0) ? cEnd : cMax;
3027   /* Setup chart */
3028   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
3029   ierr = PetscMalloc4(dim+1,&numSubPoints,dim+1,&firstSubPoint,dim+1,&subpointIS,dim+1,&subpoints);CHKERRQ(ierr);
3030   for (d = 0; d <= dim; ++d) {
3031     ierr = DMLabelGetStratumSize(subpointMap, d, &numSubPoints[d]);CHKERRQ(ierr);
3032     totSubPoints += numSubPoints[d];
3033   }
3034   ierr = DMPlexSetChart(subdm, 0, totSubPoints);CHKERRQ(ierr);
3035   ierr = DMPlexSetVTKCellHeight(subdm, cellHeight);CHKERRQ(ierr);
3036   /* Set cone sizes */
3037   firstSubPoint[dim] = 0;
3038   firstSubPoint[0]   = firstSubPoint[dim] + numSubPoints[dim];
3039   if (dim > 1) {firstSubPoint[dim-1] = firstSubPoint[0]     + numSubPoints[0];}
3040   if (dim > 2) {firstSubPoint[dim-2] = firstSubPoint[dim-1] + numSubPoints[dim-1];}
3041   for (d = 0; d <= dim; ++d) {
3042     ierr = DMLabelGetStratumIS(subpointMap, d, &subpointIS[d]);CHKERRQ(ierr);
3043     if (subpointIS[d]) {ierr = ISGetIndices(subpointIS[d], &subpoints[d]);CHKERRQ(ierr);}
3044   }
3045   for (d = 0; d <= dim; ++d) {
3046     for (p = 0; p < numSubPoints[d]; ++p) {
3047       const PetscInt  point    = subpoints[d][p];
3048       const PetscInt  subpoint = firstSubPoint[d] + p;
3049       const PetscInt *cone;
3050       PetscInt        coneSize, coneSizeNew, c, val;
3051 
3052       ierr = DMPlexGetConeSize(dm, point, &coneSize);CHKERRQ(ierr);
3053       ierr = DMPlexSetConeSize(subdm, subpoint, coneSize);CHKERRQ(ierr);
3054       if (cellHeight && (d == dim)) {
3055         ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
3056         for (c = 0, coneSizeNew = 0; c < coneSize; ++c) {
3057           ierr = DMLabelGetValue(subpointMap, cone[c], &val);CHKERRQ(ierr);
3058           if (val >= 0) coneSizeNew++;
3059         }
3060         ierr = DMPlexSetConeSize(subdm, subpoint, coneSizeNew);CHKERRQ(ierr);
3061       }
3062     }
3063   }
3064   ierr = DMLabelDestroy(&subpointMap);CHKERRQ(ierr);
3065   ierr = DMSetUp(subdm);CHKERRQ(ierr);
3066   /* Set cones */
3067   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, NULL);CHKERRQ(ierr);
3068   ierr = PetscMalloc2(maxConeSize,&coneNew,maxConeSize,&orntNew);CHKERRQ(ierr);
3069   for (d = 0; d <= dim; ++d) {
3070     for (p = 0; p < numSubPoints[d]; ++p) {
3071       const PetscInt  point    = subpoints[d][p];
3072       const PetscInt  subpoint = firstSubPoint[d] + p;
3073       const PetscInt *cone, *ornt;
3074       PetscInt        coneSize, subconeSize, coneSizeNew, c, subc, fornt = 0;
3075 
3076       if (d == dim-1) {
3077         const PetscInt *support, *cone, *ornt;
3078         PetscInt        supportSize, coneSize, s, subc;
3079 
3080         ierr = DMPlexGetSupport(dm, point, &support);CHKERRQ(ierr);
3081         ierr = DMPlexGetSupportSize(dm, point, &supportSize);CHKERRQ(ierr);
3082         for (s = 0; s < supportSize; ++s) {
3083           if ((support[s] < cMax) || (support[s] >= cEnd)) continue;
3084           ierr = PetscFindInt(support[s], numSubPoints[d+1], subpoints[d+1], &subc);CHKERRQ(ierr);
3085           if (subc >= 0) {
3086             const PetscInt ccell = subpoints[d+1][subc];
3087 
3088             ierr = DMPlexGetCone(dm, ccell, &cone);CHKERRQ(ierr);
3089             ierr = DMPlexGetConeSize(dm, ccell, &coneSize);CHKERRQ(ierr);
3090             ierr = DMPlexGetConeOrientation(dm, ccell, &ornt);CHKERRQ(ierr);
3091             for (c = 0; c < coneSize; ++c) {
3092               if (cone[c] == point) {
3093                 fornt = ornt[c];
3094                 break;
3095               }
3096             }
3097             break;
3098           }
3099         }
3100       }
3101       ierr = DMPlexGetConeSize(dm, point, &coneSize);CHKERRQ(ierr);
3102       ierr = DMPlexGetConeSize(subdm, subpoint, &subconeSize);CHKERRQ(ierr);
3103       ierr = DMPlexGetCone(dm, point, &cone);CHKERRQ(ierr);
3104       ierr = DMPlexGetConeOrientation(dm, point, &ornt);CHKERRQ(ierr);
3105       for (c = 0, coneSizeNew = 0; c < coneSize; ++c) {
3106         ierr = PetscFindInt(cone[c], numSubPoints[d-1], subpoints[d-1], &subc);CHKERRQ(ierr);
3107         if (subc >= 0) {
3108           coneNew[coneSizeNew] = firstSubPoint[d-1] + subc;
3109           orntNew[coneSizeNew] = ornt[c];
3110           ++coneSizeNew;
3111         }
3112       }
3113       if (coneSizeNew != subconeSize) SETERRQ2(comm, PETSC_ERR_PLIB, "Number of cone points located %d does not match subcone size %d", coneSizeNew, subconeSize);
3114       if (fornt < 0) {
3115         /* This should be replaced by a call to DMPlexReverseCell() */
3116 #if 0
3117         ierr = DMPlexReverseCell(subdm, subpoint);CHKERRQ(ierr);
3118 #else
3119         for (c = 0; c < coneSizeNew/2 + coneSizeNew%2; ++c) {
3120           PetscInt faceSize, tmp;
3121 
3122           tmp        = coneNew[c];
3123           coneNew[c] = coneNew[coneSizeNew-1-c];
3124           coneNew[coneSizeNew-1-c] = tmp;
3125           ierr = DMPlexGetConeSize(dm, cone[c], &faceSize);CHKERRQ(ierr);
3126           tmp        = orntNew[c] >= 0 ? -(faceSize-orntNew[c]) : faceSize+orntNew[c];
3127           orntNew[c] = orntNew[coneSizeNew-1-c] >= 0 ? -(faceSize-orntNew[coneSizeNew-1-c]) : faceSize+orntNew[coneSizeNew-1-c];
3128           orntNew[coneSizeNew-1-c] = tmp;
3129         }
3130       }
3131       ierr = DMPlexSetCone(subdm, subpoint, coneNew);CHKERRQ(ierr);
3132       ierr = DMPlexSetConeOrientation(subdm, subpoint, orntNew);CHKERRQ(ierr);
3133 #endif
3134     }
3135   }
3136   ierr = PetscFree2(coneNew,orntNew);CHKERRQ(ierr);
3137   ierr = DMPlexSymmetrize(subdm);CHKERRQ(ierr);
3138   ierr = DMPlexStratify(subdm);CHKERRQ(ierr);
3139   /* Build coordinates */
3140   {
3141     PetscSection coordSection, subCoordSection;
3142     Vec          coordinates, subCoordinates;
3143     PetscScalar *coords, *subCoords;
3144     PetscInt     cdim, numComp, coordSize;
3145     const char  *name;
3146 
3147     ierr = DMGetCoordinateDim(dm, &cdim);CHKERRQ(ierr);
3148     ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
3149     ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
3150     ierr = DMGetCoordinateSection(subdm, &subCoordSection);CHKERRQ(ierr);
3151     ierr = PetscSectionSetNumFields(subCoordSection, 1);CHKERRQ(ierr);
3152     ierr = PetscSectionGetFieldComponents(coordSection, 0, &numComp);CHKERRQ(ierr);
3153     ierr = PetscSectionSetFieldComponents(subCoordSection, 0, numComp);CHKERRQ(ierr);
3154     ierr = PetscSectionSetChart(subCoordSection, firstSubPoint[0], firstSubPoint[0]+numSubPoints[0]);CHKERRQ(ierr);
3155     for (v = 0; v < numSubPoints[0]; ++v) {
3156       const PetscInt vertex    = subpoints[0][v];
3157       const PetscInt subvertex = firstSubPoint[0]+v;
3158       PetscInt       dof;
3159 
3160       ierr = PetscSectionGetDof(coordSection, vertex, &dof);CHKERRQ(ierr);
3161       ierr = PetscSectionSetDof(subCoordSection, subvertex, dof);CHKERRQ(ierr);
3162       ierr = PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof);CHKERRQ(ierr);
3163     }
3164     ierr = PetscSectionSetUp(subCoordSection);CHKERRQ(ierr);
3165     ierr = PetscSectionGetStorageSize(subCoordSection, &coordSize);CHKERRQ(ierr);
3166     ierr = VecCreate(PETSC_COMM_SELF, &subCoordinates);CHKERRQ(ierr);
3167     ierr = PetscObjectGetName((PetscObject)coordinates,&name);CHKERRQ(ierr);
3168     ierr = PetscObjectSetName((PetscObject)subCoordinates,name);CHKERRQ(ierr);
3169     ierr = VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
3170     ierr = VecSetBlockSize(subCoordinates, cdim);CHKERRQ(ierr);
3171     ierr = VecSetType(subCoordinates,VECSTANDARD);CHKERRQ(ierr);
3172     ierr = VecGetArray(coordinates,    &coords);CHKERRQ(ierr);
3173     ierr = VecGetArray(subCoordinates, &subCoords);CHKERRQ(ierr);
3174     for (v = 0; v < numSubPoints[0]; ++v) {
3175       const PetscInt vertex    = subpoints[0][v];
3176       const PetscInt subvertex = firstSubPoint[0]+v;
3177       PetscInt dof, off, sdof, soff, d;
3178 
3179       ierr = PetscSectionGetDof(coordSection, vertex, &dof);CHKERRQ(ierr);
3180       ierr = PetscSectionGetOffset(coordSection, vertex, &off);CHKERRQ(ierr);
3181       ierr = PetscSectionGetDof(subCoordSection, subvertex, &sdof);CHKERRQ(ierr);
3182       ierr = PetscSectionGetOffset(subCoordSection, subvertex, &soff);CHKERRQ(ierr);
3183       if (dof != sdof) SETERRQ4(comm, PETSC_ERR_PLIB, "Coordinate dimension %d on subvertex %d, vertex %d should be %d", sdof, subvertex, vertex, dof);
3184       for (d = 0; d < dof; ++d) subCoords[soff+d] = coords[off+d];
3185     }
3186     ierr = VecRestoreArray(coordinates,    &coords);CHKERRQ(ierr);
3187     ierr = VecRestoreArray(subCoordinates, &subCoords);CHKERRQ(ierr);
3188     ierr = DMSetCoordinatesLocal(subdm, subCoordinates);CHKERRQ(ierr);
3189     ierr = VecDestroy(&subCoordinates);CHKERRQ(ierr);
3190   }
3191   /* Build SF: We need this complexity because subpoints might not be selected on the owning process */
3192   {
3193     PetscSF            sfPoint, sfPointSub;
3194     IS                 subpIS;
3195     const PetscSFNode *remotePoints;
3196     PetscSFNode       *sremotePoints, *newLocalPoints, *newOwners;
3197     const PetscInt    *localPoints, *subpoints;
3198     PetscInt          *slocalPoints;
3199     PetscInt           numRoots, numLeaves, numSubpoints = 0, numSubroots, numSubleaves = 0, l, sl, ll, pStart, pEnd, p;
3200     PetscMPIInt        rank;
3201 
3202     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRQ(ierr);
3203     ierr = DMGetPointSF(dm, &sfPoint);CHKERRQ(ierr);
3204     ierr = DMGetPointSF(subdm, &sfPointSub);CHKERRQ(ierr);
3205     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
3206     ierr = DMPlexGetChart(subdm, NULL, &numSubroots);CHKERRQ(ierr);
3207     ierr = DMPlexCreateSubpointIS(subdm, &subpIS);CHKERRQ(ierr);
3208     if (subpIS) {
3209       ierr = ISGetIndices(subpIS, &subpoints);CHKERRQ(ierr);
3210       ierr = ISGetLocalSize(subpIS, &numSubpoints);CHKERRQ(ierr);
3211     }
3212     ierr = PetscSFGetGraph(sfPoint, &numRoots, &numLeaves, &localPoints, &remotePoints);CHKERRQ(ierr);
3213     if (numRoots >= 0) {
3214       ierr = PetscMalloc2(pEnd-pStart,&newLocalPoints,numRoots,&newOwners);CHKERRQ(ierr);
3215       for (p = 0; p < pEnd-pStart; ++p) {
3216         newLocalPoints[p].rank  = -2;
3217         newLocalPoints[p].index = -2;
3218       }
3219       /* Set subleaves */
3220       for (l = 0; l < numLeaves; ++l) {
3221         const PetscInt point    = localPoints[l];
3222         const PetscInt subpoint = DMPlexFilterPoint_Internal(point, 0, numSubpoints, subpoints);
3223 
3224         if (subpoint < 0) continue;
3225         newLocalPoints[point-pStart].rank  = rank;
3226         newLocalPoints[point-pStart].index = subpoint;
3227         ++numSubleaves;
3228       }
3229       /* Must put in owned subpoints */
3230       for (p = pStart; p < pEnd; ++p) {
3231         const PetscInt subpoint = DMPlexFilterPoint_Internal(p, 0, numSubpoints, subpoints);
3232 
3233         if (subpoint < 0) {
3234           newOwners[p-pStart].rank  = -3;
3235           newOwners[p-pStart].index = -3;
3236         } else {
3237           newOwners[p-pStart].rank  = rank;
3238           newOwners[p-pStart].index = subpoint;
3239         }
3240       }
3241       ierr = PetscSFReduceBegin(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC);CHKERRQ(ierr);
3242       ierr = PetscSFReduceEnd(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC);CHKERRQ(ierr);
3243       ierr = PetscSFBcastBegin(sfPoint, MPIU_2INT, newOwners, newLocalPoints);CHKERRQ(ierr);
3244       ierr = PetscSFBcastEnd(sfPoint, MPIU_2INT, newOwners, newLocalPoints);CHKERRQ(ierr);
3245       ierr = PetscMalloc1(numSubleaves, &slocalPoints);CHKERRQ(ierr);
3246       ierr = PetscMalloc1(numSubleaves, &sremotePoints);CHKERRQ(ierr);
3247       for (l = 0, sl = 0, ll = 0; l < numLeaves; ++l) {
3248         const PetscInt point    = localPoints[l];
3249         const PetscInt subpoint = DMPlexFilterPoint_Internal(point, 0, numSubpoints, subpoints);
3250 
3251         if (subpoint < 0) continue;
3252         if (newLocalPoints[point].rank == rank) {++ll; continue;}
3253         slocalPoints[sl]        = subpoint;
3254         sremotePoints[sl].rank  = newLocalPoints[point].rank;
3255         sremotePoints[sl].index = newLocalPoints[point].index;
3256         if (sremotePoints[sl].rank  < 0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote rank for local point %d", point);
3257         if (sremotePoints[sl].index < 0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote subpoint for local point %d", point);
3258         ++sl;
3259       }
3260       if (sl + ll != numSubleaves) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatch in number of subleaves %d + %d != %d", sl, ll, numSubleaves);
3261       ierr = PetscFree2(newLocalPoints,newOwners);CHKERRQ(ierr);
3262       ierr = PetscSFSetGraph(sfPointSub, numSubroots, sl, slocalPoints, PETSC_OWN_POINTER, sremotePoints, PETSC_OWN_POINTER);CHKERRQ(ierr);
3263     }
3264     if (subpIS) {
3265       ierr = ISRestoreIndices(subpIS, &subpoints);CHKERRQ(ierr);
3266       ierr = ISDestroy(&subpIS);CHKERRQ(ierr);
3267     }
3268   }
3269   /* Cleanup */
3270   for (d = 0; d <= dim; ++d) {
3271     if (subpointIS[d]) {ierr = ISRestoreIndices(subpointIS[d], &subpoints[d]);CHKERRQ(ierr);}
3272     ierr = ISDestroy(&subpointIS[d]);CHKERRQ(ierr);
3273   }
3274   ierr = PetscFree4(numSubPoints,firstSubPoint,subpointIS,subpoints);CHKERRQ(ierr);
3275   PetscFunctionReturn(0);
3276 }
3277 
3278 static PetscErrorCode DMPlexCreateSubmesh_Interpolated(DM dm, DMLabel vertexLabel, PetscInt value, PetscBool markedFaces, DM subdm)
3279 {
3280   PetscErrorCode ierr;
3281 
3282   PetscFunctionBegin;
3283   ierr = DMPlexCreateSubmeshGeneric_Interpolated(dm, vertexLabel, value, markedFaces, PETSC_FALSE, 1, subdm);CHKERRQ(ierr);
3284   PetscFunctionReturn(0);
3285 }
3286 
3287 /*@
3288   DMPlexCreateSubmesh - Extract a hypersurface from the mesh using vertices defined by a label
3289 
3290   Input Parameters:
3291 + dm           - The original mesh
3292 . vertexLabel  - The DMLabel marking points contained in the surface
3293 . value        - The label value to use
3294 - markedFaces  - PETSC_TRUE if surface faces are marked in addition to vertices, PETSC_FALSE if only vertices are marked
3295 
3296   Output Parameter:
3297 . subdm - The surface mesh
3298 
3299   Note: This function produces a DMLabel mapping original points in the submesh to their depth. This can be obtained using DMPlexGetSubpointMap().
3300 
3301   Level: developer
3302 
3303 .seealso: DMPlexGetSubpointMap(), DMGetLabel(), DMLabelSetValue()
3304 @*/
3305 PetscErrorCode DMPlexCreateSubmesh(DM dm, DMLabel vertexLabel, PetscInt value, PetscBool markedFaces, DM *subdm)
3306 {
3307   DMPlexInterpolatedFlag interpolated;
3308   PetscInt       dim, cdim;
3309   PetscErrorCode ierr;
3310 
3311   PetscFunctionBegin;
3312   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3313   PetscValidPointer(subdm, 3);
3314   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
3315   ierr = DMCreate(PetscObjectComm((PetscObject)dm), subdm);CHKERRQ(ierr);
3316   ierr = DMSetType(*subdm, DMPLEX);CHKERRQ(ierr);
3317   ierr = DMSetDimension(*subdm, dim-1);CHKERRQ(ierr);
3318   ierr = DMGetCoordinateDim(dm, &cdim);CHKERRQ(ierr);
3319   ierr = DMSetCoordinateDim(*subdm, cdim);CHKERRQ(ierr);
3320   ierr = DMPlexIsInterpolated(dm, &interpolated);CHKERRQ(ierr);
3321   if (interpolated == DMPLEX_INTERPOLATED_PARTIAL) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Not for partially interpolated meshes");
3322   if (interpolated) {
3323     ierr = DMPlexCreateSubmesh_Interpolated(dm, vertexLabel, value, markedFaces, *subdm);CHKERRQ(ierr);
3324   } else {
3325     ierr = DMPlexCreateSubmesh_Uninterpolated(dm, vertexLabel, value, *subdm);CHKERRQ(ierr);
3326   }
3327   PetscFunctionReturn(0);
3328 }
3329 
3330 static PetscErrorCode DMPlexCreateCohesiveSubmesh_Uninterpolated(DM dm, PetscBool hasLagrange, const char label[], PetscInt value, DM subdm)
3331 {
3332   MPI_Comm        comm;
3333   DMLabel         subpointMap;
3334   IS              subvertexIS;
3335   const PetscInt *subVertices;
3336   PetscInt        numSubVertices, firstSubVertex, numSubCells, *subCells = NULL;
3337   PetscInt       *subface, maxConeSize, numSubFaces, firstSubFace, newFacePoint, nFV;
3338   PetscInt        cMax, c, f;
3339   PetscErrorCode  ierr;
3340 
3341   PetscFunctionBegin;
3342   ierr = PetscObjectGetComm((PetscObject)dm, &comm);CHKERRQ(ierr);
3343   /* Create subpointMap which marks the submesh */
3344   ierr = DMLabelCreate(PETSC_COMM_SELF, "subpoint_map", &subpointMap);CHKERRQ(ierr);
3345   ierr = DMPlexSetSubpointMap(subdm, subpointMap);CHKERRQ(ierr);
3346   ierr = DMLabelDestroy(&subpointMap);CHKERRQ(ierr);
3347   ierr = DMPlexMarkCohesiveSubmesh_Uninterpolated(dm, hasLagrange, label, value, subpointMap, &numSubFaces, &nFV, &subCells, subdm);CHKERRQ(ierr);
3348   /* Setup chart */
3349   ierr = DMLabelGetStratumSize(subpointMap, 0, &numSubVertices);CHKERRQ(ierr);
3350   ierr = DMLabelGetStratumSize(subpointMap, 2, &numSubCells);CHKERRQ(ierr);
3351   ierr = DMPlexSetChart(subdm, 0, numSubCells+numSubFaces+numSubVertices);CHKERRQ(ierr);
3352   ierr = DMPlexSetVTKCellHeight(subdm, 1);CHKERRQ(ierr);
3353   /* Set cone sizes */
3354   firstSubVertex = numSubCells;
3355   firstSubFace   = numSubCells+numSubVertices;
3356   newFacePoint   = firstSubFace;
3357   ierr = DMLabelGetStratumIS(subpointMap, 0, &subvertexIS);CHKERRQ(ierr);
3358   if (subvertexIS) {ierr = ISGetIndices(subvertexIS, &subVertices);CHKERRQ(ierr);}
3359   for (c = 0; c < numSubCells; ++c) {
3360     ierr = DMPlexSetConeSize(subdm, c, 1);CHKERRQ(ierr);
3361   }
3362   for (f = firstSubFace; f < firstSubFace+numSubFaces; ++f) {
3363     ierr = DMPlexSetConeSize(subdm, f, nFV);CHKERRQ(ierr);
3364   }
3365   ierr = DMSetUp(subdm);CHKERRQ(ierr);
3366   /* Create face cones */
3367   ierr = DMPlexGetMaxSizes(dm, &maxConeSize, NULL);CHKERRQ(ierr);
3368   ierr = DMPlexGetHybridBounds(dm, &cMax, NULL, NULL, NULL);CHKERRQ(ierr);
3369   ierr = DMGetWorkArray(subdm, maxConeSize, MPIU_INT, (void**) &subface);CHKERRQ(ierr);
3370   for (c = 0; c < numSubCells; ++c) {
3371     const PetscInt  cell    = subCells[c];
3372     const PetscInt  subcell = c;
3373     const PetscInt *cone, *cells;
3374     PetscInt        numCells, subVertex, p, v;
3375 
3376     if (cell < cMax) continue;
3377     ierr = DMPlexGetCone(dm, cell, &cone);CHKERRQ(ierr);
3378     for (v = 0; v < nFV; ++v) {
3379       ierr = PetscFindInt(cone[v], numSubVertices, subVertices, &subVertex);CHKERRQ(ierr);
3380       subface[v] = firstSubVertex+subVertex;
3381     }
3382     ierr = DMPlexSetCone(subdm, newFacePoint, subface);CHKERRQ(ierr);
3383     ierr = DMPlexSetCone(subdm, subcell, &newFacePoint);CHKERRQ(ierr);
3384     ierr = DMPlexGetJoin(dm, nFV, cone, &numCells, &cells);CHKERRQ(ierr);
3385     /* Not true in parallel
3386     if (numCells != 2) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Cohesive cells should separate two cells"); */
3387     for (p = 0; p < numCells; ++p) {
3388       PetscInt negsubcell;
3389 
3390       if (cells[p] >= cMax) continue;
3391       /* I know this is a crap search */
3392       for (negsubcell = 0; negsubcell < numSubCells; ++negsubcell) {
3393         if (subCells[negsubcell] == cells[p]) break;
3394       }
3395       if (negsubcell == numSubCells) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not find negative face neighbor for cohesive cell %d", cell);
3396       ierr = DMPlexSetCone(subdm, negsubcell, &newFacePoint);CHKERRQ(ierr);
3397     }
3398     ierr = DMPlexRestoreJoin(dm, nFV, cone, &numCells, &cells);CHKERRQ(ierr);
3399     ++newFacePoint;
3400   }
3401   ierr = DMRestoreWorkArray(subdm, maxConeSize, MPIU_INT, (void**) &subface);CHKERRQ(ierr);
3402   ierr = DMPlexSymmetrize(subdm);CHKERRQ(ierr);
3403   ierr = DMPlexStratify(subdm);CHKERRQ(ierr);
3404   /* Build coordinates */
3405   {
3406     PetscSection coordSection, subCoordSection;
3407     Vec          coordinates, subCoordinates;
3408     PetscScalar *coords, *subCoords;
3409     PetscInt     cdim, numComp, coordSize, v;
3410     const char  *name;
3411 
3412     ierr = DMGetCoordinateDim(dm, &cdim);CHKERRQ(ierr);
3413     ierr = DMGetCoordinateSection(dm, &coordSection);CHKERRQ(ierr);
3414     ierr = DMGetCoordinatesLocal(dm, &coordinates);CHKERRQ(ierr);
3415     ierr = DMGetCoordinateSection(subdm, &subCoordSection);CHKERRQ(ierr);
3416     ierr = PetscSectionSetNumFields(subCoordSection, 1);CHKERRQ(ierr);
3417     ierr = PetscSectionGetFieldComponents(coordSection, 0, &numComp);CHKERRQ(ierr);
3418     ierr = PetscSectionSetFieldComponents(subCoordSection, 0, numComp);CHKERRQ(ierr);
3419     ierr = PetscSectionSetChart(subCoordSection, firstSubVertex, firstSubVertex+numSubVertices);CHKERRQ(ierr);
3420     for (v = 0; v < numSubVertices; ++v) {
3421       const PetscInt vertex    = subVertices[v];
3422       const PetscInt subvertex = firstSubVertex+v;
3423       PetscInt       dof;
3424 
3425       ierr = PetscSectionGetDof(coordSection, vertex, &dof);CHKERRQ(ierr);
3426       ierr = PetscSectionSetDof(subCoordSection, subvertex, dof);CHKERRQ(ierr);
3427       ierr = PetscSectionSetFieldDof(subCoordSection, subvertex, 0, dof);CHKERRQ(ierr);
3428     }
3429     ierr = PetscSectionSetUp(subCoordSection);CHKERRQ(ierr);
3430     ierr = PetscSectionGetStorageSize(subCoordSection, &coordSize);CHKERRQ(ierr);
3431     ierr = VecCreate(PETSC_COMM_SELF, &subCoordinates);CHKERRQ(ierr);
3432     ierr = PetscObjectGetName((PetscObject)coordinates,&name);CHKERRQ(ierr);
3433     ierr = PetscObjectSetName((PetscObject)subCoordinates,name);CHKERRQ(ierr);
3434     ierr = VecSetSizes(subCoordinates, coordSize, PETSC_DETERMINE);CHKERRQ(ierr);
3435     ierr = VecSetBlockSize(subCoordinates, cdim);CHKERRQ(ierr);
3436     ierr = VecSetType(subCoordinates,VECSTANDARD);CHKERRQ(ierr);
3437     ierr = VecGetArray(coordinates,    &coords);CHKERRQ(ierr);
3438     ierr = VecGetArray(subCoordinates, &subCoords);CHKERRQ(ierr);
3439     for (v = 0; v < numSubVertices; ++v) {
3440       const PetscInt vertex    = subVertices[v];
3441       const PetscInt subvertex = firstSubVertex+v;
3442       PetscInt       dof, off, sdof, soff, d;
3443 
3444       ierr = PetscSectionGetDof(coordSection, vertex, &dof);CHKERRQ(ierr);
3445       ierr = PetscSectionGetOffset(coordSection, vertex, &off);CHKERRQ(ierr);
3446       ierr = PetscSectionGetDof(subCoordSection, subvertex, &sdof);CHKERRQ(ierr);
3447       ierr = PetscSectionGetOffset(subCoordSection, subvertex, &soff);CHKERRQ(ierr);
3448       if (dof != sdof) SETERRQ4(comm, PETSC_ERR_PLIB, "Coordinate dimension %d on subvertex %d, vertex %d should be %d", sdof, subvertex, vertex, dof);
3449       for (d = 0; d < dof; ++d) subCoords[soff+d] = coords[off+d];
3450     }
3451     ierr = VecRestoreArray(coordinates,    &coords);CHKERRQ(ierr);
3452     ierr = VecRestoreArray(subCoordinates, &subCoords);CHKERRQ(ierr);
3453     ierr = DMSetCoordinatesLocal(subdm, subCoordinates);CHKERRQ(ierr);
3454     ierr = VecDestroy(&subCoordinates);CHKERRQ(ierr);
3455   }
3456   /* Build SF */
3457   CHKMEMQ;
3458   {
3459     PetscSF            sfPoint, sfPointSub;
3460     const PetscSFNode *remotePoints;
3461     PetscSFNode       *sremotePoints, *newLocalPoints, *newOwners;
3462     const PetscInt    *localPoints;
3463     PetscInt          *slocalPoints;
3464     PetscInt           numRoots, numLeaves, numSubRoots = numSubCells+numSubFaces+numSubVertices, numSubLeaves = 0, l, sl, ll, pStart, pEnd, p, vStart, vEnd;
3465     PetscMPIInt        rank;
3466 
3467     ierr = MPI_Comm_rank(PetscObjectComm((PetscObject) dm), &rank);CHKERRQ(ierr);
3468     ierr = DMGetPointSF(dm, &sfPoint);CHKERRQ(ierr);
3469     ierr = DMGetPointSF(subdm, &sfPointSub);CHKERRQ(ierr);
3470     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
3471     ierr = DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd);CHKERRQ(ierr);
3472     ierr = PetscSFGetGraph(sfPoint, &numRoots, &numLeaves, &localPoints, &remotePoints);CHKERRQ(ierr);
3473     if (numRoots >= 0) {
3474       /* Only vertices should be shared */
3475       ierr = PetscMalloc2(pEnd-pStart,&newLocalPoints,numRoots,&newOwners);CHKERRQ(ierr);
3476       for (p = 0; p < pEnd-pStart; ++p) {
3477         newLocalPoints[p].rank  = -2;
3478         newLocalPoints[p].index = -2;
3479       }
3480       /* Set subleaves */
3481       for (l = 0; l < numLeaves; ++l) {
3482         const PetscInt point    = localPoints[l];
3483         const PetscInt subPoint = DMPlexFilterPoint_Internal(point, firstSubVertex, numSubVertices, subVertices);
3484 
3485         if ((point < vStart) && (point >= vEnd)) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Should not be mapping anything but vertices, %d", point);
3486         if (subPoint < 0) continue;
3487         newLocalPoints[point-pStart].rank  = rank;
3488         newLocalPoints[point-pStart].index = subPoint;
3489         ++numSubLeaves;
3490       }
3491       /* Must put in owned subpoints */
3492       for (p = pStart; p < pEnd; ++p) {
3493         const PetscInt subPoint = DMPlexFilterPoint_Internal(p, firstSubVertex, numSubVertices, subVertices);
3494 
3495         if (subPoint < 0) {
3496           newOwners[p-pStart].rank  = -3;
3497           newOwners[p-pStart].index = -3;
3498         } else {
3499           newOwners[p-pStart].rank  = rank;
3500           newOwners[p-pStart].index = subPoint;
3501         }
3502       }
3503       ierr = PetscSFReduceBegin(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC);CHKERRQ(ierr);
3504       ierr = PetscSFReduceEnd(sfPoint, MPIU_2INT, newLocalPoints, newOwners, MPI_MAXLOC);CHKERRQ(ierr);
3505       ierr = PetscSFBcastBegin(sfPoint, MPIU_2INT, newOwners, newLocalPoints);CHKERRQ(ierr);
3506       ierr = PetscSFBcastEnd(sfPoint, MPIU_2INT, newOwners, newLocalPoints);CHKERRQ(ierr);
3507       ierr = PetscMalloc1(numSubLeaves,    &slocalPoints);CHKERRQ(ierr);
3508       ierr = PetscMalloc1(numSubLeaves, &sremotePoints);CHKERRQ(ierr);
3509       for (l = 0, sl = 0, ll = 0; l < numLeaves; ++l) {
3510         const PetscInt point    = localPoints[l];
3511         const PetscInt subPoint = DMPlexFilterPoint_Internal(point, firstSubVertex, numSubVertices, subVertices);
3512 
3513         if (subPoint < 0) continue;
3514         if (newLocalPoints[point].rank == rank) {++ll; continue;}
3515         slocalPoints[sl]        = subPoint;
3516         sremotePoints[sl].rank  = newLocalPoints[point].rank;
3517         sremotePoints[sl].index = newLocalPoints[point].index;
3518         if (sremotePoints[sl].rank  < 0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote rank for local point %d", point);
3519         if (sremotePoints[sl].index < 0) SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid remote subpoint for local point %d", point);
3520         ++sl;
3521       }
3522       ierr = PetscFree2(newLocalPoints,newOwners);CHKERRQ(ierr);
3523       if (sl + ll != numSubLeaves) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_PLIB, "Mismatch in number of subleaves %d + %d != %d", sl, ll, numSubLeaves);
3524       ierr = PetscSFSetGraph(sfPointSub, numSubRoots, sl, slocalPoints, PETSC_OWN_POINTER, sremotePoints, PETSC_OWN_POINTER);CHKERRQ(ierr);
3525     }
3526   }
3527   CHKMEMQ;
3528   /* Cleanup */
3529   if (subvertexIS) {ierr = ISRestoreIndices(subvertexIS, &subVertices);CHKERRQ(ierr);}
3530   ierr = ISDestroy(&subvertexIS);CHKERRQ(ierr);
3531   ierr = PetscFree(subCells);CHKERRQ(ierr);
3532   PetscFunctionReturn(0);
3533 }
3534 
3535 static PetscErrorCode DMPlexCreateCohesiveSubmesh_Interpolated(DM dm, const char labelname[], PetscInt value, DM subdm)
3536 {
3537   DMLabel        label = NULL;
3538   PetscErrorCode ierr;
3539 
3540   PetscFunctionBegin;
3541   if (labelname) {ierr = DMGetLabel(dm, labelname, &label);CHKERRQ(ierr);}
3542   ierr = DMPlexCreateSubmeshGeneric_Interpolated(dm, label, value, PETSC_FALSE, PETSC_TRUE, 1, subdm);CHKERRQ(ierr);
3543   PetscFunctionReturn(0);
3544 }
3545 
3546 /*@C
3547   DMPlexCreateCohesiveSubmesh - Extract from a mesh with cohesive cells the hypersurface defined by one face of the cells. Optionally, a Label an be given to restrict the cells.
3548 
3549   Input Parameters:
3550 + dm          - The original mesh
3551 . hasLagrange - The mesh has Lagrange unknowns in the cohesive cells
3552 . label       - A label name, or NULL
3553 - value  - A label value
3554 
3555   Output Parameter:
3556 . subdm - The surface mesh
3557 
3558   Note: This function produces a DMLabel mapping original points in the submesh to their depth. This can be obtained using DMPlexGetSubpointMap().
3559 
3560   Level: developer
3561 
3562 .seealso: DMPlexGetSubpointMap(), DMPlexCreateSubmesh()
3563 @*/
3564 PetscErrorCode DMPlexCreateCohesiveSubmesh(DM dm, PetscBool hasLagrange, const char label[], PetscInt value, DM *subdm)
3565 {
3566   PetscInt       dim, cdim, depth;
3567   PetscErrorCode ierr;
3568 
3569   PetscFunctionBegin;
3570   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3571   PetscValidPointer(subdm, 5);
3572   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
3573   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3574   ierr = DMCreate(PetscObjectComm((PetscObject)dm), subdm);CHKERRQ(ierr);
3575   ierr = DMSetType(*subdm, DMPLEX);CHKERRQ(ierr);
3576   ierr = DMSetDimension(*subdm, dim-1);CHKERRQ(ierr);
3577   ierr = DMGetCoordinateDim(dm, &cdim);CHKERRQ(ierr);
3578   ierr = DMSetCoordinateDim(*subdm, cdim);CHKERRQ(ierr);
3579   if (depth == dim) {
3580     ierr = DMPlexCreateCohesiveSubmesh_Interpolated(dm, label, value, *subdm);CHKERRQ(ierr);
3581   } else {
3582     ierr = DMPlexCreateCohesiveSubmesh_Uninterpolated(dm, hasLagrange, label, value, *subdm);CHKERRQ(ierr);
3583   }
3584   PetscFunctionReturn(0);
3585 }
3586 
3587 /*@
3588   DMPlexFilter - Extract a subset of mesh cells defined by a label as a separate mesh
3589 
3590   Input Parameters:
3591 + dm        - The original mesh
3592 . cellLabel - The DMLabel marking cells contained in the new mesh
3593 - value     - The label value to use
3594 
3595   Output Parameter:
3596 . subdm - The new mesh
3597 
3598   Note: This function produces a DMLabel mapping original points in the submesh to their depth. This can be obtained using DMPlexGetSubpointMap().
3599 
3600   Level: developer
3601 
3602 .seealso: DMPlexGetSubpointMap(), DMGetLabel(), DMLabelSetValue()
3603 @*/
3604 PetscErrorCode DMPlexFilter(DM dm, DMLabel cellLabel, PetscInt value, DM *subdm)
3605 {
3606   PetscInt       dim;
3607   PetscErrorCode ierr;
3608 
3609   PetscFunctionBegin;
3610   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3611   PetscValidPointer(subdm, 3);
3612   ierr = DMGetDimension(dm, &dim);CHKERRQ(ierr);
3613   ierr = DMCreate(PetscObjectComm((PetscObject) dm), subdm);CHKERRQ(ierr);
3614   ierr = DMSetType(*subdm, DMPLEX);CHKERRQ(ierr);
3615   ierr = DMSetDimension(*subdm, dim);CHKERRQ(ierr);
3616   /* Extract submesh in place, could be empty on some procs, could have inconsistency if procs do not both extract a shared cell */
3617   ierr = DMPlexCreateSubmeshGeneric_Interpolated(dm, cellLabel, value, PETSC_FALSE, PETSC_FALSE, 0, *subdm);CHKERRQ(ierr);
3618   PetscFunctionReturn(0);
3619 }
3620 
3621 /*@
3622   DMPlexGetSubpointMap - Returns a DMLabel with point dimension as values
3623 
3624   Input Parameter:
3625 . dm - The submesh DM
3626 
3627   Output Parameter:
3628 . subpointMap - The DMLabel of all the points from the original mesh in this submesh, or NULL if this is not a submesh
3629 
3630   Level: developer
3631 
3632 .seealso: DMPlexCreateSubmesh(), DMPlexCreateSubpointIS()
3633 @*/
3634 PetscErrorCode DMPlexGetSubpointMap(DM dm, DMLabel *subpointMap)
3635 {
3636   PetscFunctionBegin;
3637   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3638   PetscValidPointer(subpointMap, 2);
3639   *subpointMap = ((DM_Plex*) dm->data)->subpointMap;
3640   PetscFunctionReturn(0);
3641 }
3642 
3643 /*@
3644   DMPlexSetSubpointMap - Sets the DMLabel with point dimension as values
3645 
3646   Input Parameters:
3647 + dm - The submesh DM
3648 - subpointMap - The DMLabel of all the points from the original mesh in this submesh
3649 
3650   Note: Should normally not be called by the user, since it is set in DMPlexCreateSubmesh()
3651 
3652   Level: developer
3653 
3654 .seealso: DMPlexCreateSubmesh(), DMPlexCreateSubpointIS()
3655 @*/
3656 PetscErrorCode DMPlexSetSubpointMap(DM dm, DMLabel subpointMap)
3657 {
3658   DM_Plex       *mesh = (DM_Plex *) dm->data;
3659   DMLabel        tmp;
3660   PetscErrorCode ierr;
3661 
3662   PetscFunctionBegin;
3663   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3664   tmp  = mesh->subpointMap;
3665   mesh->subpointMap = subpointMap;
3666   ierr = PetscObjectReference((PetscObject) mesh->subpointMap);CHKERRQ(ierr);
3667   ierr = DMLabelDestroy(&tmp);CHKERRQ(ierr);
3668   PetscFunctionReturn(0);
3669 }
3670 
3671 /*@
3672   DMPlexCreateSubpointIS - Creates an IS covering the entire subdm chart with the original points as data
3673 
3674   Input Parameter:
3675 . dm - The submesh DM
3676 
3677   Output Parameter:
3678 . subpointIS - The IS of all the points from the original mesh in this submesh, or NULL if this is not a submesh
3679 
3680   Note: This IS is guaranteed to be sorted by the construction of the submesh
3681 
3682   Level: developer
3683 
3684 .seealso: DMPlexCreateSubmesh(), DMPlexGetSubpointMap()
3685 @*/
3686 PetscErrorCode DMPlexCreateSubpointIS(DM dm, IS *subpointIS)
3687 {
3688   MPI_Comm        comm;
3689   DMLabel         subpointMap;
3690   IS              is;
3691   const PetscInt *opoints;
3692   PetscInt       *points, *depths;
3693   PetscInt        depth, depStart, depEnd, d, pStart, pEnd, p, n, off;
3694   PetscErrorCode  ierr;
3695 
3696   PetscFunctionBegin;
3697   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3698   PetscValidPointer(subpointIS, 2);
3699   ierr = PetscObjectGetComm((PetscObject)dm,&comm);CHKERRQ(ierr);
3700   *subpointIS = NULL;
3701   ierr = DMPlexGetSubpointMap(dm, &subpointMap);CHKERRQ(ierr);
3702   ierr = DMPlexGetDepth(dm, &depth);CHKERRQ(ierr);
3703   if (subpointMap && depth >= 0) {
3704     ierr = DMPlexGetChart(dm, &pStart, &pEnd);CHKERRQ(ierr);
3705     if (pStart) SETERRQ1(comm, PETSC_ERR_ARG_WRONG, "Submeshes must start the point numbering at 0, not %d", pStart);
3706     ierr = DMGetWorkArray(dm, depth+1, MPIU_INT, &depths);CHKERRQ(ierr);
3707     depths[0] = depth;
3708     depths[1] = 0;
3709     for(d = 2; d <= depth; ++d) {depths[d] = depth+1 - d;}
3710     ierr = PetscMalloc1(pEnd, &points);CHKERRQ(ierr);
3711     for(d = 0, off = 0; d <= depth; ++d) {
3712       const PetscInt dep = depths[d];
3713 
3714       ierr = DMPlexGetDepthStratum(dm, dep, &depStart, &depEnd);CHKERRQ(ierr);
3715       ierr = DMLabelGetStratumSize(subpointMap, dep, &n);CHKERRQ(ierr);
3716       if (((d < 2) && (depth > 1)) || (d == 1)) { /* Only check vertices and cells for now since the map is broken for others */
3717         if (n != depEnd-depStart) SETERRQ3(comm, PETSC_ERR_ARG_WRONG, "The number of mapped submesh points %d at depth %d should be %d", n, dep, depEnd-depStart);
3718       } else {
3719         if (!n) {
3720           if (d == 0) {
3721             /* Missing cells */
3722             for(p = 0; p < depEnd-depStart; ++p, ++off) points[off] = -1;
3723           } else {
3724             /* Missing faces */
3725             for(p = 0; p < depEnd-depStart; ++p, ++off) points[off] = PETSC_MAX_INT;
3726           }
3727         }
3728       }
3729       if (n) {
3730         ierr = DMLabelGetStratumIS(subpointMap, dep, &is);CHKERRQ(ierr);
3731         ierr = ISGetIndices(is, &opoints);CHKERRQ(ierr);
3732         for(p = 0; p < n; ++p, ++off) points[off] = opoints[p];
3733         ierr = ISRestoreIndices(is, &opoints);CHKERRQ(ierr);
3734         ierr = ISDestroy(&is);CHKERRQ(ierr);
3735       }
3736     }
3737     ierr = DMRestoreWorkArray(dm, depth+1, MPIU_INT, &depths);CHKERRQ(ierr);
3738     if (off != pEnd) SETERRQ2(comm, PETSC_ERR_ARG_WRONG, "The number of mapped submesh points %d should be %d", off, pEnd);
3739     ierr = ISCreateGeneral(PETSC_COMM_SELF, pEnd, points, PETSC_OWN_POINTER, subpointIS);CHKERRQ(ierr);
3740   }
3741   PetscFunctionReturn(0);
3742 }
3743 
3744 /*@
3745   DMGetEnclosureRelation - Get the relationship between dmA and dmB
3746 
3747   Input Parameters:
3748 + dmA - The first DM
3749 - dmB - The second DM
3750 
3751   Output Parameter:
3752 . rel - The relation of dmA to dmB
3753 
3754   Level: intermediate
3755 
3756 .seealso: DMPlexGetEnclosurePoint()
3757 @*/
3758 PetscErrorCode DMGetEnclosureRelation(DM dmA, DM dmB, DMEnclosureType *rel)
3759 {
3760   DM             plexA, plexB, sdm;
3761   DMLabel        spmap;
3762   PetscInt       pStartA, pEndA, pStartB, pEndB, NpA, NpB;
3763   PetscErrorCode ierr;
3764 
3765   PetscFunctionBegin;
3766   PetscValidPointer(rel, 3);
3767   *rel = DM_ENC_NONE;
3768   if (!dmA || !dmB) PetscFunctionReturn(0);
3769   PetscValidHeaderSpecific(dmA, DM_CLASSID, 1);
3770   PetscValidHeaderSpecific(dmB, DM_CLASSID, 1);
3771   if (dmA == dmB) {*rel = DM_ENC_EQUALITY; PetscFunctionReturn(0);}
3772   ierr = DMConvert(dmA, DMPLEX, &plexA);CHKERRQ(ierr);
3773   ierr = DMConvert(dmB, DMPLEX, &plexB);CHKERRQ(ierr);
3774   ierr = DMPlexGetChart(plexA, &pStartA, &pEndA);CHKERRQ(ierr);
3775   ierr = DMPlexGetChart(plexB, &pStartB, &pEndB);CHKERRQ(ierr);
3776   /* Assumption 1: subDMs have smaller charts than the DMs that they originate from
3777     - The degenerate case of a subdomain which includes all of the domain on some process can be treated as equality */
3778   if ((pStartA == pStartB) && (pEndA == pEndB)) {
3779     *rel = DM_ENC_EQUALITY;
3780     goto end;
3781   }
3782   NpA = pEndA - pStartA;
3783   NpB = pEndB - pStartB;
3784   if (NpA == NpB) goto end;
3785   sdm = NpA > NpB ? plexB : plexA; /* The other is the original, enclosing dm */
3786   ierr = DMPlexGetSubpointMap(sdm, &spmap);CHKERRQ(ierr);
3787   if (!spmap) goto end;
3788   /* TODO Check the space mapped to by subpointMap is same size as dm */
3789   if (NpA > NpB) {
3790     *rel = DM_ENC_SUPERMESH;
3791   } else {
3792     *rel = DM_ENC_SUBMESH;
3793   }
3794   end:
3795   ierr = DMDestroy(&plexA);CHKERRQ(ierr);
3796   ierr = DMDestroy(&plexB);CHKERRQ(ierr);
3797   PetscFunctionReturn(0);
3798 }
3799 
3800 /*@
3801   DMGetEnclosurePoint - Get the point pA in dmA which corresponds to the point pB in dmB
3802 
3803   Input Parameters:
3804 + dmA   - The first DM
3805 . dmB   - The second DM
3806 . etype - The type of enclosure relation that dmA has to dmB
3807 - pB    - A point of dmB
3808 
3809   Output Parameter:
3810 . pA    - The corresponding point of dmA
3811 
3812   Level: intermediate
3813 
3814 .seealso: DMGetEnclosureRelation()
3815 @*/
3816 PetscErrorCode DMGetEnclosurePoint(DM dmA, DM dmB, DMEnclosureType etype, PetscInt pB, PetscInt *pA)
3817 {
3818   DM              sdm;
3819   DMLabel         spmap;
3820   IS              subpointIS;
3821   const PetscInt *subpoints;
3822   PetscInt        numSubpoints;
3823   PetscErrorCode  ierr;
3824 
3825   PetscFunctionBegin;
3826   /* TODO Cache the IS, making it look like an index */
3827   switch (etype) {
3828     case DM_ENC_SUPERMESH:
3829     sdm  = dmB;
3830     ierr = DMPlexGetSubpointMap(sdm, &spmap);CHKERRQ(ierr);
3831     ierr = DMPlexCreateSubpointIS(sdm, &subpointIS);CHKERRQ(ierr);
3832     ierr = ISGetIndices(subpointIS, &subpoints);CHKERRQ(ierr);
3833     *pA  = subpoints[pB];
3834     ierr = ISRestoreIndices(subpointIS, &subpoints);CHKERRQ(ierr);
3835     ierr = ISDestroy(&subpointIS);CHKERRQ(ierr);
3836     break;
3837     case DM_ENC_SUBMESH:
3838     sdm  = dmA;
3839     ierr = DMPlexGetSubpointMap(sdm, &spmap);CHKERRQ(ierr);
3840     ierr = DMPlexCreateSubpointIS(sdm, &subpointIS);CHKERRQ(ierr);
3841     ierr = ISGetLocalSize(subpointIS, &numSubpoints);CHKERRQ(ierr);
3842     ierr = ISGetIndices(subpointIS, &subpoints);CHKERRQ(ierr);
3843     ierr = PetscFindInt(pB, numSubpoints, subpoints, pA);CHKERRQ(ierr);
3844     if (*pA < 0) {
3845       ierr = DMViewFromOptions(dmA, NULL, "-dm_enc_A_view");CHKERRQ(ierr);
3846       ierr = DMViewFromOptions(dmB, NULL, "-dm_enc_B_view");CHKERRQ(ierr);
3847       SETERRQ1(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Point %d not found in submesh", pB);
3848     }
3849     ierr = ISRestoreIndices(subpointIS, &subpoints);CHKERRQ(ierr);
3850     ierr = ISDestroy(&subpointIS);CHKERRQ(ierr);
3851     break;
3852     case DM_ENC_EQUALITY:
3853     case DM_ENC_NONE:
3854     *pA = pB;break;
3855     case DM_ENC_UNKNOWN:
3856     {
3857       DMEnclosureType enc;
3858 
3859       ierr = DMGetEnclosureRelation(dmA, dmB, &enc);CHKERRQ(ierr);
3860       ierr = DMGetEnclosurePoint(dmA, dmB, enc, pB, pA);CHKERRQ(ierr);
3861     }
3862     break;
3863     default: SETERRQ1(PetscObjectComm((PetscObject) dmA), PETSC_ERR_ARG_OUTOFRANGE, "Invalid enclosure type %d", (int) etype);
3864   }
3865   PetscFunctionReturn(0);
3866 }
3867